-- in some movie script

on tokenize (str, sep, esc)
  l = []
  _player.itemDelimiter = sep
  cnt = str.item.count
  repeat with i = 1 to cnt
    prev = l.getLast() -- can be VOID
    if _trailEscCount(prev, esc) mod 2 then
      l[l.count] = prev.char[1..prev.length-1]&sep&str.item[i]
    else
      l.add(str.item[i])
    end if
  end repeat
  -- remove escape characters from tokens
  cnt = l.count
  repeat with i = 1 to cnt
    l[i] = _removeEsc(l[i], esc)
  end repeat
  return l
end

-- counts number of trailing escape characters
on _trailEscCount (str, esc)
  n = 0
  repeat with i = str.length down to 1
    if str.char[i]=esc then n=n+1
    else exit repeat
  end repeat
  return n
end

-- could be implemented more efficiently by using offset()
on _removeEsc (str, esc)
  cnt = str.length-1
  repeat with i = 1 to cnt
    if str.char[i]=esc then
      delete char i of str
      cnt = cnt-1
    end if
  end repeat
  return str
end