Data commit

Task/Tokenize-a-string-with-escaping/Dyalect/tokenize-a-string-with-escaping.dyalect

@@ -0,0 +1,31 @@
+func String.Tokenize(separator, escape) {
+    var buffer = []
+    var escaping = false
+    for c in this {
+        if escaping {
+            buffer.Add(c)
+            escaping = false
+        } else if c == escape {
+            escaping = true
+        } else if c == separator {
+            yield buffer.Flush();
+        } else {
+            buffer.Add(c);
+        }
+    }
+
+    if buffer.Length() > 0 || this[this.Length() - 1] == separator {
+        yield buffer.Flush()
+    }
+}
+
+func Array.Flush() {
+    var str = String.Concat(values: this)
+    this.Clear()
+    str
+}
+
+let testcase = "one^|uno||three^^^^|four^^^|^cuatro|";
+for token in testcase.Tokenize(separator: '|', escape: '^') {
+    print(": \(token)")
+}