Data update
This commit is contained in:
Ingy döt Net
parent
4bb20c9b71
commit
cbaf4c4b64
12390 changed files with 318560 additions and 27248 deletions
49
Task/UTF-8-encode-and-decode/Ada/utf-8-encode-and-decode.adb
Normal file
49
Task/UTF-8-encode-and-decode/Ada/utf-8-encode-and-decode.adb
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
with Ada.Strings.Fixed; use Ada.Strings.Fixed;
|
||||
with Ada.Strings.UTF_Encoding.Wide_Wide_Strings;
|
||||
with Ada.Integer_Text_IO;
|
||||
with Ada.Text_IO;
|
||||
with Ada.Wide_Wide_Text_IO;
|
||||
|
||||
procedure UTF8_Encode_And_Decode
|
||||
is
|
||||
package TIO renames Ada.Text_IO;
|
||||
package WWTIO renames Ada.Wide_Wide_Text_IO;
|
||||
package WWS renames Ada.Strings.UTF_Encoding.Wide_Wide_Strings;
|
||||
|
||||
function To_Hex
|
||||
(i : in Integer;
|
||||
width : in Natural := 0;
|
||||
fill : in Character := '0') return String
|
||||
is
|
||||
holder : String(1 .. 20);
|
||||
begin
|
||||
Ada.Integer_Text_IO.Put(holder, i, 16);
|
||||
declare
|
||||
hex : constant String := holder(Index(holder, "#")+1 .. holder'Last-1);
|
||||
filled : String := Natural'Max(width, hex'Length) * fill;
|
||||
begin
|
||||
filled(filled'Last - hex'Length + 1 .. filled'Last) := hex;
|
||||
return filled;
|
||||
end;
|
||||
end To_Hex;
|
||||
|
||||
input : constant Wide_Wide_String := "AöЖ€𝄞";
|
||||
begin
|
||||
TIO.Put_Line("Character Unicode UTF-8 encoding (hex)");
|
||||
TIO.Put_Line(43 * '-');
|
||||
for WWC of input loop
|
||||
WWTIO.Put(WWC & " ");
|
||||
declare
|
||||
filled : String := 11 * ' ';
|
||||
unicode : constant String := "U+" & To_Hex(Wide_Wide_Character'Pos(WWC), width => 4);
|
||||
utf8_string : constant String := WWS.Encode((1 => WWC));
|
||||
begin
|
||||
filled(filled'First .. filled'First + unicode'Length - 1) := unicode;
|
||||
TIO.Put(filled);
|
||||
for C of utf8_string loop
|
||||
TIO.Put(To_Hex(Character'Pos(C)) & " ");
|
||||
end loop;
|
||||
TIO.New_Line;
|
||||
end;
|
||||
end loop;
|
||||
end UTF8_Encode_And_Decode;
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
include "cowgol.coh";
|
||||
|
||||
sub UTF8Encode(char: uint32, buf: [uint8]): (len: uint8) is
|
||||
if char < 0x80 then
|
||||
[buf] := char as uint8;
|
||||
len := 1;
|
||||
elseif char < 0x800 then
|
||||
[buf] := (0xC0 | (char >> 6)) as uint8;
|
||||
buf := @next buf;
|
||||
[buf] := (0x80 | (char & 0x3F)) as uint8;
|
||||
len := 2;
|
||||
elseif char < 0x10000 then
|
||||
[buf] := (0xE0 | (char >> 12)) as uint8;
|
||||
buf := @next buf;
|
||||
[buf] := (0x80 | ((char >> 6) & 0x3F)) as uint8;
|
||||
buf := @next buf;
|
||||
[buf] := (0x80 | (char & 0x3F)) as uint8;
|
||||
len := 3;
|
||||
elseif char < 0x110000 then
|
||||
[buf] := (0xF0 | (char >> 18)) as uint8;
|
||||
buf := @next buf;
|
||||
[buf] := (0x80 | ((char >> 12) & 0x3F)) as uint8;
|
||||
buf := @next buf;
|
||||
[buf] := (0x80 | ((char >> 6) & 0x3F)) as uint8;
|
||||
buf := @next buf;
|
||||
[buf] := (0x80 | (char & 0x3F)) as uint8;
|
||||
len := 4;
|
||||
else
|
||||
len := 0;
|
||||
end if;
|
||||
end sub;
|
||||
|
||||
sub UTF8Decode(buf: [uint8]): (len: uint8, char: uint32) is
|
||||
len := 1;
|
||||
var start := [buf];
|
||||
var mask: uint8 := 0;
|
||||
while start & 0x80 == 0x80 loop
|
||||
start := start << 1;
|
||||
mask := (mask >> 1) | 0x80;
|
||||
len := len + 1;
|
||||
end loop;
|
||||
|
||||
char := ([buf] & ~mask) as uint32;
|
||||
|
||||
var steps := len - 2;
|
||||
while steps > 0 loop
|
||||
buf := @next buf;
|
||||
char := (char << 6) | ([buf] & 0x3F) as uint32;
|
||||
steps := steps - 1;
|
||||
end loop;
|
||||
end sub;
|
||||
|
||||
sub Test(char: uint32) is
|
||||
var buf: uint8[5];
|
||||
|
||||
print_hex_i32(char);
|
||||
print(" ->");
|
||||
var len := UTF8Encode(char, &buf[0]);
|
||||
|
||||
var i: uint8 := 0;
|
||||
while i<len loop
|
||||
print_char(' ');
|
||||
print_hex_i8(buf[i]);
|
||||
i := i+1;
|
||||
end loop;
|
||||
|
||||
print(" (");
|
||||
buf[len] := 0;
|
||||
print(&buf[0]);
|
||||
print(") -> ");
|
||||
(len, char) := UTF8Decode(&buf[0]);
|
||||
print_hex_i32(char);
|
||||
print_nl();
|
||||
end sub;
|
||||
|
||||
Test(0x00041);
|
||||
Test(0x000F6);
|
||||
Test(0x00416);
|
||||
Test(0x020AC);
|
||||
Test(0x1D11E);
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
codepoints = [0x41, 0xF6, 0x416, 0x20AC, 0x1D11E]
|
||||
|
||||
codepoints.each do |codepoint1|
|
||||
# Int#chr returns a Char with that unicode codepoint
|
||||
char = codepoint1.chr
|
||||
|
||||
# Char#bytes returns an array of bytes with itself encoded in UTF-8
|
||||
bytes = char.bytes
|
||||
|
||||
# A String can be created from a Slice of bytes encoded in UTF-8
|
||||
string = String.new(Slice.new(bytes.to_unsafe, bytes.size))
|
||||
|
||||
# Indexing a string, the first Char can be extracted
|
||||
char2 = string[0]
|
||||
|
||||
# Char#ord returns its codepoint as an integer. The roundtrip is complete
|
||||
codepoint2 = char2.ord
|
||||
|
||||
puts "U+%06x -> %-20s -> U+%06x (%s)" % {codepoint1, bytes, codepoint2, char}
|
||||
end
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
sysconf hex_numbers
|
||||
func$ utf8dec b[] .
|
||||
ind = 1
|
||||
while ind <= len b[]
|
||||
n = b[ind]
|
||||
if n < 0x80
|
||||
cnt = 0
|
||||
elif n >= 0xf0
|
||||
cnt = 3
|
||||
n = bitand n 0x7
|
||||
elif n >= 0xe0
|
||||
cnt = 2
|
||||
n = bitand n 0xf
|
||||
elif n >= 0xc0
|
||||
cnt = 1
|
||||
n = bitand n 0x1f
|
||||
else
|
||||
return ""
|
||||
.
|
||||
for i = 1 to cnt
|
||||
h = b[ind + i]
|
||||
if bitand h 0xc0 <> 0x80 : return ""
|
||||
h = bitand h 0x3f
|
||||
n = n * 64 + h
|
||||
.
|
||||
ind += cnt + 1
|
||||
res$ &= strchar n
|
||||
.
|
||||
return res$
|
||||
.
|
||||
func[] utf8enc s$ .
|
||||
for c$ in strchars s$
|
||||
c = strcode c$
|
||||
if c < 0x80
|
||||
cnt = 0
|
||||
pre = 0
|
||||
elif c < 0x800
|
||||
cnt = 1
|
||||
pre = 0xc0
|
||||
elif c < 0x10000
|
||||
cnt = 2
|
||||
pre = 0xe0
|
||||
elif c < 0x200000
|
||||
cnt = 3
|
||||
pre = 0xf0
|
||||
else
|
||||
return [ ]
|
||||
.
|
||||
for i to cnt + 1 : r[] &= 0
|
||||
for i = 0 to cnt - 1
|
||||
l = c mod 0x40 + 0x80
|
||||
c = c div 0x40
|
||||
r[$ - i] = l
|
||||
.
|
||||
r[$ - i] = c + pre
|
||||
.
|
||||
return r[]
|
||||
.
|
||||
for c$ in [ "A" "ö" "Ж" "€" "𝄞" "你好 😊" ]
|
||||
b[] = utf8enc c$
|
||||
print c$ & " -> " & b[] & " -> " & utf8dec b[]
|
||||
.
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
link hexcvt # for hexstring
|
||||
|
||||
record testitem(ch,code,u8str)
|
||||
|
||||
procedure main()
|
||||
testdata := [
|
||||
testitem("A", 16r0041, "\x41" ),
|
||||
testitem("ö", 16r00F6, "\xC3\xB6" ),
|
||||
testitem("Ж", 16r0416, "\xD0\x96" ),
|
||||
testitem("€", 16r20AC, "\xE2\x82\xAC" ),
|
||||
testitem("𝄞", 16r1D11E, "\xF0\x9D\x84\x9E")
|
||||
]
|
||||
every item := !testdata do
|
||||
if u8char(item.code) == item.ch == item.u8str &
|
||||
u8ord(item.u8str) = item.code then
|
||||
write(u8char(item.code)," (U+",hexstring(u8ord(item.u8str),4),
|
||||
") <-> ",hexordstr(u8char(item.code))," passed")
|
||||
else
|
||||
write(item.ch," (U+",hexstring(item.code,4),") failed")
|
||||
end
|
||||
|
||||
procedure u8char(i) # Codepoint to utf-8 encoding
|
||||
local s
|
||||
if not ((i := integer(i)) >= 0) then runerr(101,i) # not an integer >= 0
|
||||
if i < 16r80 then{ # 1 byte
|
||||
s := char(i)
|
||||
}else if i < 16r800 then{ # 2 byte
|
||||
s := char(ior(2r11000000,ishift(i,-6)))
|
||||
s ||:= char(ior(2r10000000,iand(i,2r111111)))
|
||||
}else if i < 16r10000 then{ # 3 byte
|
||||
if 16rD800 <= i <= 16rDFFF then runerr(205,i) # surrogates are not codepoints
|
||||
s := char(ior(2r11100000,ishift(i,-12)))
|
||||
s ||:= char(ior(2r10000000,iand(ishift(i,-6),2r111111)))
|
||||
s ||:= char(ior(2r10000000,iand(i,2r111111)))
|
||||
}else if i < 16r110000 then{ # 4 byte
|
||||
s := char(ior(2r11110000,ishift(i,-18)))
|
||||
s ||:= char(ior(2r10000000,iand(ishift(i,-12),2r111111)))
|
||||
s ||:= char(ior(2r10000000,iand(ishift(i,-6),2r111111)))
|
||||
s ||:= char(ior(2r10000000,iand(i,2r111111)))
|
||||
}else{ # beyond range
|
||||
runerr(101,i)
|
||||
}
|
||||
return s
|
||||
end
|
||||
|
||||
procedure u8ord(u8ch) # utf-8 encoding to code point
|
||||
local b1,b2,b3,b4,i
|
||||
|
||||
type(u8ch) == "string" | runerr(103,u8ch)
|
||||
b1 := ord(u8ch[1])
|
||||
case *u8ch of{
|
||||
1:{ b1 < 16r80 | fail
|
||||
i := b1
|
||||
}
|
||||
2:{ 16rC1 < b1 < 16rE0 | fail
|
||||
b2 := ord(u8ch[2])
|
||||
iand(b2,2r11000000) = 2r10000000 | fail
|
||||
i := ishift(iand(b1,2r11111),6)
|
||||
i := ior(i,iand(b2,2r111111))
|
||||
}
|
||||
3:{ 16rDF < b1 < 16rF0 | fail
|
||||
b2 := ord(u8ch[2]) ; b3 := ord(u8ch[3])
|
||||
iand(b2,2r11000000) = iand(b3,2r11000000) = 2r10000000 | fail
|
||||
i := ishift(iand(b1,2r1111),12)
|
||||
i := ior(i,ishift(iand(b2,2r111111),6))
|
||||
i := ior(i,iand(b3,2r111111))
|
||||
16r0800 <= i < 16rD800 | i > 16rDFFF | fail
|
||||
}
|
||||
4:{ 16rEF < b1 < 16rF5 | fail
|
||||
b2 := ord(u8ch[2]) ; b3 := ord(u8ch[3]) ; b4 := ord(u8ch[4])
|
||||
iand(b2,2r11000000) = iand(b3,2r11000000) = iand(b4,2r11000000) =
|
||||
2r10000000 | fail
|
||||
i := ishift(iand(b1,2r1111),18)
|
||||
i := ior(i,ishift(iand(b2,2r111111),12))
|
||||
i := ior(i,ishift(iand(b3,2r111111), 6))
|
||||
i := ior(i,iand(b4,2r111111))
|
||||
16r010000 <= i <= 16r10FFFF | fail
|
||||
}
|
||||
default: runerr(205,u8ch)
|
||||
}
|
||||
return i
|
||||
end
|
||||
|
||||
procedure hexordstr(str)
|
||||
s := ""
|
||||
every s ||:= hexstring(ord(!str)) || " "
|
||||
return s[1:-1]
|
||||
end
|
||||
|
|
@ -3,6 +3,6 @@ writeln "character Unicode UTF-8 encoding (hex)"
|
|||
for cp in "AöЖ€𝄞" {
|
||||
val utf8 = cp -> cp2s -> s2b
|
||||
val cpstr = utf8 -> b2s
|
||||
val utf8rep = join(map(utf8, by=fn b:"{{b:X02}}"), delim=" ")
|
||||
val utf8rep = join(map(utf8, by=fn(b) { "{{b:X02}}" }), delim=" ")
|
||||
writeln "{{cpstr:-11}} U+{{cp:X04:-8}} {{utf8rep}}"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import encoding.hex
|
||||
fn decode(s string) ?[]u8 {
|
||||
fn decode(s string) ![]u8 {
|
||||
return hex.decode(s)
|
||||
}
|
||||
|
||||
|
|
@ -7,7 +7,7 @@ fn main() {
|
|||
println("${'Char':-7} ${'Unicode':7}\tUTF-8 encoded\tDecoded")
|
||||
for codepoint in [`A`, `ö`, `Ж`, `€`, `𝄞`] {
|
||||
encoded := codepoint.bytes().hex()
|
||||
decoded := decode(encoded)?
|
||||
decoded := decode(encoded)!
|
||||
println("${codepoint:-7} U+${codepoint:04X}\t${encoded:-12}\t${decoded.bytestr()}")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,86 @@
|
|||
Option Explicit
|
||||
Dim m_1,m_2,m_3,m_4
|
||||
Dim d_2,d_3,d_4
|
||||
Dim h_0,h_2,h_3,h_4
|
||||
Dim mc_0,mc_2,mc_3,mc_4
|
||||
|
||||
m_1=&h3F
|
||||
d_2=m_1+1
|
||||
m_2=m_1 * d_2
|
||||
d_3= (m_2 Or m_1)+1
|
||||
m_3= m_2* d_2
|
||||
d_4=(m_3 Or m_2 Or m_1)+1
|
||||
|
||||
h_0=&h80
|
||||
h_2=&hC0
|
||||
h_3=&hE0
|
||||
h_4=&hF0
|
||||
|
||||
mc_0=&h3f
|
||||
mc_2=&h1F
|
||||
mc_3=&hF
|
||||
mc_4=&h7
|
||||
|
||||
Function cp2utf8(cp) 'cp as long, returns string
|
||||
If cp<&h80 Then
|
||||
cp2utf8=Chr(cp)
|
||||
ElseIf (cp <=&H7FF) Then
|
||||
cp2utf8=Chr(h_2 or (cp \ d_2) )&Chr(h_0 Or (cp And m_1))
|
||||
ElseIf (cp <=&Hffff&) Then
|
||||
cp2utf8= Chr(h_3 Or (cp\ d_3)) & Chr(h_0 Or (cp And m_2)\d_2) & Chr(h_0 Or (cp And m_1))
|
||||
Else
|
||||
cp2utf8= Chr(h_4 Or (cp\d_4))& Chr(h_0 Or ((cp And m_3) \d_3))& Chr(h_0 Or ((cp And m_2)\d_2)) & Chr(h_0 Or (cp And m_1))
|
||||
End if
|
||||
End Function
|
||||
|
||||
Function utf82cp(utf) 'utf as string, returns long
|
||||
Dim a,b,m
|
||||
m=strreverse(utf)
|
||||
b= Len(utf)
|
||||
a=asc(mid(m,1,1))
|
||||
utf82cp=a And &h7f
|
||||
if b=1 Then Exit Function
|
||||
a=asc(mid(m,2,1))
|
||||
If b=2 Then utf82cp= utf82cp Or (a And mc_2)*d_2 :Exit function
|
||||
utf82cp= utf82cp Or (a And m_1)*d_2
|
||||
a=asc(mid(m,3,1))
|
||||
If b=3 Then utf82cp= utf82cp Or (a And mc_3)*d_3 :Exit function
|
||||
utf82cp= utf82cp Or (a And m_1)*d_3 Or (a=asc(mid(m,4,1)) And mc_4)*d_4
|
||||
End Function
|
||||
|
||||
Sub print(s):
|
||||
On Error Resume Next
|
||||
WScript.stdout.Write (s)
|
||||
If err= &h80070006& Then WScript.Echo " Please run this script with CScript": WScript.quit
|
||||
End Sub
|
||||
|
||||
Function utf8displ(utf)
|
||||
Dim s,i
|
||||
s=""
|
||||
For i=1 To Len(utf)
|
||||
s=s &" "& Hex(Asc(Mid(utf,i,1)))
|
||||
Next
|
||||
utf8displ= pad(s,12)
|
||||
End function
|
||||
|
||||
function pad(s,n) if n<0 then pad= right(space(-n) & s ,-n) else pad= left(s& space(n),n) end if :end function
|
||||
|
||||
Sub check(i)
|
||||
Dim c,c0,c1,c2,u
|
||||
c=b(i):c0=pad(c(0),29) :c1=c(1) :c2=pad(c(2),12):u=cp2utf8(c1)
|
||||
print c0 & " CP:" & pad("U+" & Hex(c1),-8) & " my utf8:" & utf8displ (u) & " should be:" & c2 & " back to CP:" & pad("U+" & Hex(utf82cp(u)),-8)& vbCrLf
|
||||
End Sub
|
||||
|
||||
Dim b
|
||||
b=Array(_
|
||||
Array("LATIN CAPITAL LETTER A ",&h41," 41"),_
|
||||
Array("LATIN SMALL LETTER O WITH DIAERESIS ",&hF6," C3 B6"),_
|
||||
Array("CYRILLIC CAPITAL LETTER ZHE ",&h416," D0 96"),_
|
||||
Array("EURO SIGN",&h20AC," E2 82 AC "),_
|
||||
Array("MUSICAL SYMBOL G CLEF ",&h1D11E," F0 9D 84 9E"))
|
||||
|
||||
check 0
|
||||
check 1
|
||||
check 2
|
||||
check 3
|
||||
check 4
|
||||
Loading…
Add table
Add a link
Reference in a new issue