Data commit

Task/Tokenize-a-string-with-escaping/Mathematica/tokenize-a-string-with-escaping.math

@@ -0,0 +1,31 @@
+ClearAll[Tokenize]
+Tokenize[str_String, escape_String : "^", sep_String : "|"] :=
+ Module[{results = {}, token = "", state = 0, a},
+  a = Characters[str];
+  Do[
+   If[state == 0,
+    Switch[c,
+     escape,
+     state = 1
+     ,
+     sep,
+     AppendTo[results, token];
+     token = "";
+     ,
+     _,
+     token = token <> c;
+     ]
+    ,
+    If[state == 1,
+     token = token <> c;
+     state = 0;
+     ]
+    ]
+
+   ,
+   {c, a}
+   ];
+  AppendTo[results, token];
+  results
+ ]
+Tokenize["one^|uno||three^^^^|four^^^|^cuatro|"]