29 lines
517 B
Text
29 lines
517 B
Text
REM >tokenizer
|
|
PROC_tokenize("one^|uno||three^^^^|four^^^|^cuatro|", "|", "^")
|
|
END
|
|
:
|
|
DEF PROC_tokenize(src$, sep$, esc$)
|
|
LOCAL field%, char$, escaping%, i%
|
|
field% = 1
|
|
escaping% = FALSE
|
|
PRINT field%; " ";
|
|
FOR i% = 1 TO LEN src$
|
|
char$ = MID$(src$, i%, 1)
|
|
IF escaping% THEN
|
|
PRINT char$;
|
|
escaping% = FALSE
|
|
ELSE
|
|
CASE char$ OF
|
|
WHEN sep$
|
|
PRINT
|
|
field% += 1
|
|
PRINT field%; " ";
|
|
WHEN esc$
|
|
escaping% = TRUE
|
|
OTHERWISE
|
|
PRINT char$;
|
|
ENDCASE
|
|
ENDIF
|
|
NEXT
|
|
PRINT
|
|
ENDPROC
|