Data commit

Task/UTF-8-encode-and-decode/Tcl/utf-8-encode-and-decode-1.tcl

@@ -0,0 +1,20 @@
+proc encoder int {
+   set u [format %c $int]
+   set bytes {}
+   foreach byte [split [encoding convertto utf-8 $u] ""] {
+      lappend bytes [format %02X [scan $byte %c]]
+   }
+   return $bytes
+}
+proc decoder bytes {
+   set str {}
+   foreach byte $bytes {
+      append str [format %c [scan $byte %x]]
+   }
+   return [encoding convertfrom utf-8 $str]
+}
+foreach test {0x0041 0x00f6 0x0416 0x20ac 0x1d11e} {
+   set res $test
+   lappend res [encoder $test] -> [decoder [encoder $test]]
+   puts $res
+}

Task/UTF-8-encode-and-decode/Tcl/utf-8-encode-and-decode-2.tcl

@@ -0,0 +1,28 @@
+proc utf8 {codepoint} {
+    scan $codepoint %llx cp
+    if {$cp < 0x10000} {
+        set str [subst \\u$codepoint]               ;# substitute per Tcl backslash rule
+        set bytes [encoding convertto utf-8 $str]   ;# encode
+    } else {                                        ;# codepoints beyond the BMP need manual approach
+        set bits [format %021b $cp]                 ;# format as binary string
+        set unibits    11110[string range $bits 0 2];# insert extra bits for utf-8 4-byte encoding
+        append unibits 10[string range $bits 3 8]
+        append unibits 10[string range $bits 9 14]
+        append unibits 10[string range $bits 15 20]
+        set bytes [binary format B* $unibits]       ;# turn into a sequence of bytes
+    }
+    return $bytes
+}
+
+proc hexchars {s} {
+    binary scan $s H* hex
+    regsub -all .. $hex {\0 }
+}
+
+# for the test, we assume the tty is in utf-8 mode and can handle beyond-BMP chars
+# so set output mode to binary so we can write raw bytes!
+chan configure stdout -encoding binary
+foreach codepoint { 41 F6 416 20AC 1D11E } {
+    set utf8 [utf8 $codepoint]
+    puts "[format U+%04s $codepoint]\t$utf8\t[hexchars $utf8]"
+}