Data commit

Task/Tokenize-a-string/CLU/tokenize-a-string.clu

@@ -0,0 +1,23 @@
+% This iterator splits the string on a given character,
+% and returns each substring in order.
+tokenize = iter (s: string, c: char) yields (string)
+    while ~string$empty(s) do
+        next: int := string$indexc(c, s)
+        if next = 0 then
+            yield(s)
+            break
+        else
+            yield(string$substr(s, 1, next-1))
+            s := string$rest(s, next+1)
+        end
+    end
+end tokenize
+
+start_up = proc ()
+    po: stream := stream$primary_output()
+    str: string := "Hello,How,Are,You,Today"
+
+    for part: string in tokenize(str, ',') do
+        stream$putl(po, part || ".")
+    end
+end start_up