Data commit

Task/Tokenize-a-string-with-escaping/Lingo/tokenize-a-string-with-escaping-1.lingo

@@ -0,0 +1,43 @@
+-- in some movie script
+
+on tokenize (str, sep, esc)
+  l = []
+  _player.itemDelimiter = sep
+  cnt = str.item.count
+  repeat with i = 1 to cnt
+    prev = l.getLast() -- can be VOID
+    if _trailEscCount(prev, esc) mod 2 then
+      l[l.count] = prev.char[1..prev.length-1]&sep&str.item[i]
+    else
+      l.add(str.item[i])
+    end if
+  end repeat
+  -- remove escape characters from tokens
+  cnt = l.count
+  repeat with i = 1 to cnt
+    l[i] = _removeEsc(l[i], esc)
+  end repeat
+  return l
+end
+
+-- counts number of trailing escape characters
+on _trailEscCount (str, esc)
+  n = 0
+  repeat with i = str.length down to 1
+    if str.char[i]=esc then n=n+1
+    else exit repeat
+  end repeat
+  return n
+end
+
+-- could be implemented more efficiently by using offset()
+on _removeEsc (str, esc)
+  cnt = str.length-1
+  repeat with i = 1 to cnt
+    if str.char[i]=esc then
+      delete char i of str
+      cnt = cnt-1
+    end if
+  end repeat
+  return str
+end

Task/Tokenize-a-string-with-escaping/Lingo/tokenize-a-string-with-escaping-2.lingo

@@ -0,0 +1,5 @@
+str = "one^|uno||three^^^^|four^^^|^cuatro|"
+sep = "|"
+esc = "^"
+put tokenize(str, sep, esc)
+-- ["one|uno", "", "three^^", "four^|cuatro", ""]