43 lines
945 B
Text
43 lines
945 B
Text
-- in some movie script
|
|
|
|
on tokenize (str, sep, esc)
|
|
l = []
|
|
_player.itemDelimiter = sep
|
|
cnt = str.item.count
|
|
repeat with i = 1 to cnt
|
|
prev = l.getLast() -- can be VOID
|
|
if _trailEscCount(prev, esc) mod 2 then
|
|
l[l.count] = prev.char[1..prev.length-1]&sep&str.item[i]
|
|
else
|
|
l.add(str.item[i])
|
|
end if
|
|
end repeat
|
|
-- remove escape characters from tokens
|
|
cnt = l.count
|
|
repeat with i = 1 to cnt
|
|
l[i] = _removeEsc(l[i], esc)
|
|
end repeat
|
|
return l
|
|
end
|
|
|
|
-- counts number of trailing escape characters
|
|
on _trailEscCount (str, esc)
|
|
n = 0
|
|
repeat with i = str.length down to 1
|
|
if str.char[i]=esc then n=n+1
|
|
else exit repeat
|
|
end repeat
|
|
return n
|
|
end
|
|
|
|
-- could be implemented more efficiently by using offset()
|
|
on _removeEsc (str, esc)
|
|
cnt = str.length-1
|
|
repeat with i = 1 to cnt
|
|
if str.char[i]=esc then
|
|
delete char i of str
|
|
cnt = cnt-1
|
|
end if
|
|
end repeat
|
|
return str
|
|
end
|