Data update

Task/Tokenize-a-string-with-escaping/Ada/tokenize-a-string-with-escaping.adb

@@ -0,0 +1,57 @@
+with Ada.Text_Io;
+with Ada.Containers.Indefinite_Vectors;
+with Ada.Strings.Unbounded;
+
+procedure Tokenize is
+
+   package String_Vectors is
+     new Ada.Containers.Indefinite_Vectors (Positive, String);
+   use String_Vectors;
+
+   function Split (Text      : String;
+                   Separator : Character := '|';
+                   Escape    : Character := '^') return Vector
+   is
+      use Ada.Strings.Unbounded;
+      Result  : Vector;
+      Escaped : Boolean := False;
+      Accu    : Unbounded_String;
+   begin
+
+      for Char of Text loop
+
+         case Escaped is
+
+            when False =>
+               if Char = Escape then
+                  Escaped := True;
+               elsif Char = Separator then
+                  Append (Result, To_String (Accu));
+                  Accu := Null_Unbounded_String;
+               else
+                  Append (Accu, Char);
+               end if;
+
+            when True =>
+               Append (Accu, Char);
+               Escaped := False;
+
+         end case;
+
+      end loop;
+      Append (Result, To_String (Accu));
+
+      return Result;
+   end Split;
+
+   procedure Put_Vector (List : Vector) is
+      use Ada.Text_Io;
+   begin
+      for Element of List loop
+         Put ("'"); Put (Element); Put ("'"); New_Line;
+      end loop;
+   end Put_Vector;
+
+begin
+   Put_Vector (Split ("one^|uno||three^^^^|four^^^|^cuatro|"));
+end Tokenize;

Task/Tokenize-a-string-with-escaping/COBOL/tokenize-a-string-with-escaping.cob

@@ -0,0 +1,135 @@
+       >>SOURCE FORMAT FREE
+identification division.
+program-id. 'tokenizewithescaping'.
+environment division.
+configuration section.
+repository.
+    function all intrinsic.
+data division.
+working-storage section.
+
+01 escape-char pic x value '^'.
+01 separator-char pic x value '|'.
+01 reference-string pic x(64) value
+   'one^|uno||three^^^^|four^^^|^cuatro|'.
+
+01 input-string pic x(64).
+01 c pic 99.
+01 escaped pic x.
+
+01 t pic 99.
+01 t-max pic 99.
+01 t-lim pic 99 value 32.
+01 token-entry occurs 32.
+   03  token-len pic 99.
+   03  token pic x(16).
+
+01 l pic 99.
+01 l-lim pic 99 value 16.
+
+01 error-found pic x.
+
+procedure division.
+start-tokenize-with-escaping.
+
+    move reference-string to input-string
+    perform tokenize
+
+    move 'token' to input-string
+    perform tokenize
+
+    move '^^^^^^^^' to input-string
+    perform tokenize
+
+    move '||||||||' to input-string
+    perform tokenize
+
+    move all 'token' to input-string
+    perform tokenize
+
+    move all 't|' to input-string
+    perform tokenize
+
+    move spaces to input-string
+    perform tokenize
+
+    display space
+
+    stop run
+    .
+tokenize.
+    display space
+    display 'string:'
+    display input-string
+
+    move 'N' to escaped error-found
+    move 1 to t-max
+    initialize token-entry(t-max)
+    move 0 to l
+
+    perform varying c from 1 by 1 until
+    c > length(input-string)
+    or input-string(c:) = spaces
+
+        evaluate escaped also input-string(c:1)
+        when 'N' also escape-char
+            move 'Y' to escaped
+        when 'N' also separator-char
+            perform increment-t-max
+            if error-found = 'Y'
+                exit paragraph
+            end-if
+        when 'N' also any
+            perform move-c
+            if error-found = 'Y'
+                exit paragraph
+            end-if
+        when 'Y' also any
+            perform move-c
+            if error-found = 'Y'
+                exit paragraph
+            end-if
+            move 'N' to escaped
+        end-evaluate
+
+    end-perform
+    if l > 0
+        move l to token-len(t-max)
+    end-if
+
+    if c = 1
+        display 'no tokens'
+    else
+        display 'tokens:'
+        perform varying t from 1 by 1 until t > t-max
+            if token-len(t) > 0
+                display t ': ' token-len(t) space token(t)
+            else
+                display t ': ' token-len(t)
+            end-if
+        end-perform
+    end-if
+    .
+increment-t-max.
+    if t-max >= t-lim
+        display 'error: at ' c ' number of tokens exceeds ' t-lim
+        move 'Y' to error-found
+    else
+        move l to token-len(t-max)
+        add 1 to t-max
+        initialize token-entry(t-max)
+        move 0 to l
+        move 'N' to error-found
+    end-if
+    .
+move-c.
+    if l >= l-lim
+        display 'error: at ' c ' token length exceeds ' l-lim
+        move 'Y' to error-found
+    else
+        add 1 to l
+        move input-string(c:1) to token(t-max)(l:1)
+        move 'N' to error-found
+    end-if
+    .
+end program 'tokenizewithescaping'.

Task/Tokenize-a-string-with-escaping/Pluto/tokenize-a-string-with-escaping.pluto

@@ -0,0 +1,14 @@
+$define SPE = "\u{fffe}"  -- unused unicode character in Specials block
+$define SPF = "\u{ffff}"  -- ditto
+
+local function tokenize(str, sep, esc)
+    str = str:replace(esc .. esc, SPE):replace(esc .. sep, SPF)
+    str = (str[-1] == esc) ? (str:sub(1, -2):replace(esc, "") .. esc) : (str:replace(esc, ""))
+    return str:split(sep):map(|s| -> s:replace(SPE, esc):replace(SPF, sep))
+end
+
+local str = "one^|uno||three^^^^|four^^^|^cuatro|"
+local sep = "|"
+local esc = "^"
+local items = tokenize(str, sep, esc)
+for items as item do print((item == "") ? "(empty)" : item) end

Task/Tokenize-a-string-with-escaping/PowerShell/tokenize-a-string-with-escaping-1.ps1

@@ -0,0 +1,29 @@
+function Split-String ([string]$String, [char]$Separator, [char]$Escape)
+{
+    if ($String -notmatch "\$Separator|\$Escape") {return $String}
+
+    [bool]$escaping = $false
+    [string]$output = ""
+
+    for ($i = 0; $i -lt $String.Length; $i++)
+    {
+        [char]$character = $String.Substring($i,1)
+
+        if ($escaping)
+        {
+            $output += $character
+            $escaping = $false
+        }
+        else
+        {
+            switch ($character)
+            {
+                {$_ -eq $Separator} {$output; $output = ""; break}
+                {$_ -eq $Escape}    {$escaping = $true    ; break}
+                Default             {$output += $character}
+            }
+        }
+    }
+
+    if ($String[-1] -eq $Separator) {[String]::Empty}
+}

Task/Tokenize-a-string-with-escaping/PowerShell/tokenize-a-string-with-escaping-2.ps1

@@ -0,0 +1,3 @@
+Split-String "one^|uno||three^^^^|four^^^|^cuatro|" -Separator "|" -Escape "^" | ForEach-Object `
+                                                                                        -Begin   {$n = 0} `
+                                                                                        -Process {$n+= 1; "{0}: {1}" -f $n, $_}

Task/Tokenize-a-string-with-escaping/V-(Vlang)/tokenize-a-string-with-escaping.v

@@ -1,4 +1,4 @@
-fn tokenize_string(s string, sep u8, escape u8) ?[]string {
+fn tokenize_string(s string, sep u8, escape u8) ![]string {
     mut tokens := []string{}
 	mut runes := []u8{}
 	mut in_escape := false
@@ -10,7 +10,7 @@ fn tokenize_string(s string, sep u8, escape u8) ?[]string {
 			in_escape = true
         } else if r == sep {
 			tokens << runes.bytestr()
-			runes = runes[..0]
+			runes = runes[..0].clone()
 		} else {
             runes << r
         }
@@ -27,6 +27,6 @@ const separator = `|`
 const escape = `^`
 fn main() {
 	println("Input:   $sample")
-	tokens := tokenize_string(sample, separator, escape)?
+	tokens := tokenize_string(sample, separator, escape)!
 	println("Tokens: $tokens")
 }