32 lines
524 B
Text
32 lines
524 B
Text
|
ClearAll[Tokenize]
|
||
|
|
Tokenize[str_String, escape_String : "^", sep_String : "|"] :=
|
||
|
|
Module[{results = {}, token = "", state = 0, a},
|
||
|
|
a = Characters[str];
|
||
|
|
Do[
|
||
|
|
If[state == 0,
|
||
|
|
Switch[c,
|
||
|
|
escape,
|
||
|
|
state = 1
|
||
|
|
,
|
||
|
|
sep,
|
||
|
|
AppendTo[results, token];
|
||
|
|
token = "";
|
||
|
|
,
|
||
|
|
_,
|
||
|
|
token = token <> c;
|
||
|
|
]
|
||
|
|
,
|
||
|
|
If[state == 1,
|
||
|
|
token = token <> c;
|
||
|
|
state = 0;
|
||
|
|
]
|
||
|
|
]
|
||
|
|
|
||
|
|
,
|
||
|
|
{c, a}
|
||
|
|
];
|
||
|
|
AppendTo[results, token];
|
||
|
|
results
|
||
|
|
]
|
||
|
|
Tokenize["one^|uno||three^^^^|four^^^|^cuatro|"]
|