Data commit

Task/Tokenize-a-string-with-escaping/J/tokenize-a-string-with-escaping-1.j

@@ -0,0 +1,22 @@
+tokenize1=: tokenize =: '^|'&$: :(4 : 0)
+ 'ESC SEP' =. x
+ STATE =. 0
+ RESULT =. 0 $ a:
+ TOKEN =. ''
+ for_C. y do.
+  if. STATE do.
+   TOKEN =. TOKEN , C
+   STATE =. 0
+  else.
+   if. C = ESC do.
+    STATE =. 1
+   elseif. C = SEP do.
+    RESULT =. RESULT , < TOKEN
+    TOKEN =. ''
+   elseif. do.
+    TOKEN =. TOKEN , C
+   end.
+  end.
+ end.
+ RESULT =. RESULT , < TOKEN
+)

Task/Tokenize-a-string-with-escaping/J/tokenize-a-string-with-escaping-2.j

@@ -0,0 +1,10 @@
+tokenize2=: tokenize=:3 :0
+  '^|' tokenize2 y  NB. task default escape and separator
+:
+  'ESC SEP'=. x
+  E=. 18 b./\.&.|.ESC=y NB. escape positions
+  S=. (SEP=y)>_1}.0,E NB. separator positions
+  K=. -.E+.S NB. keep positions
+  T=. (#y){. 1,}.S NB. token beginnings
+  (T<;.1 K)#&.>T<;.1 y
+)

Task/Tokenize-a-string-with-escaping/J/tokenize-a-string-with-escaping-3.j

@@ -0,0 +1,4 @@
+   '^|' tokenize 'one^|uno||three^^^^|four^^^|^cuatro|'
+┌───────┬┬───────┬────────────┬┐
+│one|uno││three^^│four^|cuatro││
+└───────┴┴───────┴────────────┴┘

Task/Tokenize-a-string-with-escaping/J/tokenize-a-string-with-escaping-4.j

@@ -0,0 +1,4 @@
+charTokens =: (0;(3 2 2$(2 1 1 1 2 2 1 2 1 0 1 0));<<'^')&;:  NB. sequential machine
+splitTokens =: ((<,'|')&= <;._1 ])@:((<,'|'),])
+removeExtra =: (}.^:(1<#)) L:0
+tokenize3=: tokenize=: ; each @: (removeExtra @: splitTokens @: charTokens)

Task/Tokenize-a-string-with-escaping/J/tokenize-a-string-with-escaping-5.j

@@ -0,0 +1,9 @@
+   t=: 'one^|uno||three^^^^|four^^^|^cuatro|'
+
+  tokenize t
+┌───────┬┬───────┬────────────┬┐
+│one|uno││three^^│four^|cuatro││
+└───────┴┴───────┴────────────┴┘
+
+   $tokenize t
+5

Task/Tokenize-a-string-with-escaping/J/tokenize-a-string-with-escaping-6.j

@@ -0,0 +1,6 @@
+   txt=: 1e5$'one^|uno||three^^^^|four^^^|^cuatro|'
+
+   (%"1 <./) timespacex every 'tokenize1 txt';'tokenize2 txt';'tokenize3 txt'
+132.856       1
+      1 7.73534
+8.29568 19.9766