Data update

This commit is contained in:
Ingy döt Net 2026-04-30 12:34:36 -04:00
parent 4bb20c9b71
commit cbaf4c4b64
12390 changed files with 318560 additions and 27248 deletions

View file

@ -0,0 +1,92 @@
with Ada.Containers.Vectors;
with Ada.Text_IO;
with Ada.Unchecked_Conversion;
procedure VLQ is
package Nat_IO is new Ada.Text_IO.Integer_IO (Natural);
type Byte is mod 2**8;
package Byte_IO is new Ada.Text_IO.Modular_IO (Byte);
type Int7 is mod 2**7;
package Int7_IO is new Ada.Text_IO.Modular_IO (Int7);
type VLQ_Octet is record
Value : Int7 := 0;
Next : Boolean := True;
end record;
pragma Pack (VLQ_Octet);
for VLQ_Octet'Size use 8;
function VLQ_To_Byte is new Ada.Unchecked_Conversion (VLQ_Octet, Byte);
function Byte_To_VLQ is new Ada.Unchecked_Conversion (Byte, VLQ_Octet);
package VLQ_Vectors is new Ada.Containers.Vectors (Natural, VLQ_Octet);
procedure Hex_Print (Position : in VLQ_Vectors.Cursor) is
Value : Byte := VLQ_To_Byte (VLQ_Vectors.Element (Position));
begin
Ada.Text_IO.Put (':');
Byte_IO.Put (Item => Value, Width => 6, Base => 16);
end Hex_Print;
procedure Print (X : VLQ_Vectors.Vector) is
begin
X.Iterate (Hex_Print'Access);
Ada.Text_IO.New_Line;
end Print;
function To_VLQ (From : Natural) return VLQ_Vectors.Vector is
Result : VLQ_Vectors.Vector;
Current : Natural := From;
Element : VLQ_Octet;
begin
loop
Element.Value := Int7 (Current mod 2**7);
Result.Prepend (Element);
Current := Current / 2**7;
exit when Current = 0;
end loop;
Element := Result.Last_Element;
Element.Next := False;
VLQ_Vectors.Replace_Element (Result, Result.Last, Element);
return Result;
end To_VLQ;
function To_Int (From : VLQ_Vectors.Vector) return Natural is
use type VLQ_Vectors.Cursor;
Result : Natural := 0;
Iterator : VLQ_Vectors.Cursor := From.First;
begin
while Iterator /= VLQ_Vectors.No_Element loop
Result := Result * 2**7;
Result := Result + Natural(VLQ_Vectors.Element (Iterator).Value);
VLQ_Vectors.Next (Iterator);
end loop;
return Result;
end To_Int;
Test : VLQ_Vectors.Vector;
begin
Test := To_VLQ (16#7f#);
Nat_IO.Put (To_Int (Test), 10, 16); Ada.Text_IO.Put (" = ");
Print (Test);
Test := To_VLQ (16#4000#);
Nat_IO.Put (To_Int (Test), 10, 16); Ada.Text_IO.Put (" = ");
Print (Test);
Test := To_VLQ (16#0#);
Nat_IO.Put (To_Int (Test), 10, 16); Ada.Text_IO.Put (" = ");
Print (Test);
Test := To_VLQ (16#3FFFFE#);
Nat_IO.Put (To_Int (Test), 10, 16); Ada.Text_IO.Put (" = ");
Print (Test);
Test := To_VLQ (16#1FFFFF#);
Nat_IO.Put (To_Int (Test), 10, 16); Ada.Text_IO.Put (" = ");
Print (Test);
Test := To_VLQ (16#200000#);
Nat_IO.Put (To_Int (Test), 10, 16); Ada.Text_IO.Put (" = ");
Print (Test);
end VLQ;

View file

@ -0,0 +1,23 @@
def vl_encode (n)
raise "negative numbers aren't vl-encodeable" unless n >= 0
bytes_needed = n == 0 ? 1 : (n.bit_length - 1) // 7 + 1
bytes = Bytes.new(bytes_needed)
(0...bytes_needed).reverse_each do |i|
bytes[i] = (n & 127).to_u8 | 128
n >>= 7
end
bytes[-1] &= 127
bytes
end
def vl_decode (bytes)
size = (bytes.size - 1) * 7 + (bytes[0] & 127).bit_length
n = size > 63 ? 0.to_i128 : size > 31 ? 0.to_i64 : 0
bytes.reduce(n) {|result, byte| (result << 7) + (byte & 127) }
end
[0x200000, 0x1fffff].each do |n|
encoded = vl_encode(n)
decoded = vl_decode(encoded)
puts "%8d %12s %8d" % { n, encoded.map {|b| "%02x" % b }.join(":"), decoded }
end

View file

@ -21,8 +21,8 @@ struct VLQ {
"Too large for ulong or invalid format.");
else
value = (t << 7) | v[idx];
return idx + 1;
}
return cast(uint) idx + 1;
}
VLQ from(in ubyte[] v) pure {
extract(v);
@ -54,7 +54,6 @@ struct VLQ {
}
}
void main() { // VLQ demo code.
VLQ a = VLQ(0x7f),
b = VLQ(0x4000),

View file

@ -0,0 +1,41 @@
function vlq_encode(integer n)
sequence s
s = {}
while n > 0 do
s = prepend(s, #80 * (length(s) > 0) + and_bits(n, #7F))
n = floor(n / #80)
end while
if length(s) = 0 then
s = {0}
end if
return s
end function
function vlq_decode(sequence s)
integer n
n = 0
for i = 1 to length(s) do
n *= #80
n += and_bits(s[i], #7F)
if not and_bits(s[i], #80) then
exit
end if
end for
return n
end function
function svlg(sequence s)
sequence out
out = ""
for i = 1 to length(s) do
out &= sprintf("#%02x:", {s[i]})
end for
return out[1..$-1]
end function
constant testNumbers = { #200000, #1FFFFF, 1, 127, 128 }
sequence s
for i = 1 to length(testNumbers) do
s = vlq_encode(testNumbers[i])
printf(1, "#%02x -> %s -> #%02x\n", {testNumbers[i], svlg(s), vlq_decode(s)})
end for

View file

@ -0,0 +1,17 @@
let to_vlq n =
let a, b = n >>> 7, n &&& 0x7F
let rec aux n acc =
let x,xs = (n &&& 0x7F) ||| 0x80, n >>> 7
if xs > 0 then aux xs (x::acc) else x::acc
aux a [b]
let to_num = List.fold (fun n x -> (n <<< 7) + (x &&& 0x7F)) 0
let v_rep n =
let seq = to_vlq n
printf "%d ->" n
List.iter (printf " 0x%02X") seq
printfn " -> %d" (to_num seq)
v_rep 0x200000
v_rep 0x1FFFFF

View file

@ -1,33 +1,35 @@
using Printf
mutable struct VLQ
quant::Vector{UInt8}
quant::Vector{UInt8}
end
function VLQ(n::T) where T <: Integer
quant = UInt8.(digits(n, 128))
@inbounds for i in 2:length(quant) quant[i] |= 0x80 end
VLQ(reverse(quant))
quant = UInt8.(digits(n, base = 128))
@inbounds for i in 2:lastindex(quant)
quant[i] |= 0x80
end
VLQ(reverse(quant))
end
import Base.UInt64
function Base.UInt64(vlq::VLQ)
quant = reverse(vlq.quant)
n = shift!(quant)
p = one(UInt64)
for i in quant
p *= 0x80
n += p * ( i & 0x7f)
end
return n
quant = reverse(vlq.quant)
n = popfirst!(quant)
p = one(UInt64)
for i in quant
p *= 0x80
n += p * (i & 0x7f)
end
return n
end
const test = [0x00200000, 0x001fffff, 0x00000000, 0x0000007f,
0x00000080, 0x00002000, 0x00003fff, 0x00004000,
0x08000000, 0x0fffffff]
0x00000080, 0x00002000, 0x00003fff, 0x00004000,
0x08000000, 0x0fffffff]
for i in test
vlq = VLQ(i)
j = UInt(vlq)
@printf "0x%-8x => [%-25s] => 0x%x\n" i join(("0x" * hex(r, 2) for r in vlq.quant), ", ") j
vlq = VLQ(i)
j = UInt(vlq)
@printf "0x%-8x => [%-25s] => 0x%x\n" i join(("0x" * string(r, base = 16, pad = 2) for r in vlq.quant), ", ") j
end

View file

@ -1,43 +1,38 @@
-->
<span style="color: #008080;">function</span> <span style="color: #000000;">vlq_encode</span><span style="color: #0000FF;">(</span><span style="color: #004080;">sequence</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">s</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">to</span> <span style="color: #000000;">1</span> <span style="color: #008080;">by</span> <span style="color: #0000FF;">-</span><span style="color: #000000;">1</span> <span style="color: #008080;">do</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">n</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">],</span> <span style="color: #000000;">msb</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">n</span><span style="color: #0000FF;"><</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span> <span style="color: #7060A8;">crash</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"unsigned integers only!"</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">while</span> <span style="color: #000000;">1</span> <span style="color: #008080;">do</span>
<span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">prepend</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">,</span><span style="color: #000000;">msb</span><span style="color: #0000FF;">+</span><span style="color: #7060A8;">and_bits</span><span style="color: #0000FF;">(</span><span style="color: #000000;">n</span><span style="color: #0000FF;">,</span><span style="color: #000000;">#7F</span><span style="color: #0000FF;">))</span>
<span style="color: #000000;">n</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">floor</span><span style="color: #0000FF;">(</span><span style="color: #000000;">n</span><span style="color: #0000FF;">/</span><span style="color: #000000;">#80</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">n</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #000000;">msb</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">#80</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">while</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">res</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
with javascript_semantics
<span style="color: #008080;">function</span> <span style="color: #000000;">vlq_decode</span><span style="color: #0000FF;">(</span><span style="color: #004080;">sequence</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">s</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">do</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">si</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">],</span>
<span style="color: #000000;">n</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">n</span><span style="color: #0000FF;">*</span><span style="color: #000000;">#80</span><span style="color: #0000FF;">+</span><span style="color: #7060A8;">and_bits</span><span style="color: #0000FF;">(</span><span style="color: #004080;">byte</span><span style="color: #0000FF;">,</span><span style="color: #000000;">#7F</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #7060A8;">and_bits</span><span style="color: #0000FF;">(</span><span style="color: #000000;">si</span><span style="color: #0000FF;">,</span><span style="color: #000000;">#80</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">append</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">,</span><span style="color: #000000;">n</span><span style="color: #0000FF;">)</span>
<span style="color: #000000;">n</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">res</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
function vlq_encode(sequence s)
sequence res = {}
for i=length(s) to 1 by -1 do
integer n = s[i], msb = 0
if n<0 then crash("unsigned integers only!") end if
while 1 do
res = prepend(res,msb+and_bits(n,#7F))
n = floor(n/#80)
if n=0 then exit end if
msb = #80
end while
end for
return res
end function
<span style="color: #008080;">function</span> <span style="color: #000000;">svlg</span><span style="color: #0000FF;">(</span><span style="color: #004080;">sequence</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">""</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">s</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">do</span>
<span style="color: #000000;">res</span> <span style="color: #0000FF;">&=</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"#%02x:"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">s</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]})</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">1</span><span style="color: #0000FF;">..$-</span><span style="color: #000000;">1</span><span style="color: #0000FF;">]</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
function vlq_decode(sequence s)
sequence res = {}
integer n = 0
for si in s do
n = n*#80+and_bits(si,#7F)
if not and_bits(si,#80) then
res = append(res,n)
n = 0
end if
end for
return res
end function
<span style="color: #008080;">constant</span> <span style="color: #000000;">testNumbers</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{</span> <span style="color: #000000;">#200000</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">#1FFFFF</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">1</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">127</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">128</span> <span style="color: #0000FF;">}</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">s</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">vlq_encode</span><span style="color: #0000FF;">(</span><span style="color: #000000;">testNumbers</span><span style="color: #0000FF;">),</span>
<span style="color: #000000;">decoded</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">vlq_decode</span><span style="color: #0000FF;">(</span><span style="color: #000000;">s</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">printf</span><span style="color: #0000FF;">(</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%s -> %s -> %s\n"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">svlg</span><span style="color: #0000FF;">(</span><span style="color: #000000;">testNumbers</span><span style="color: #0000FF;">),</span><span style="color: #000000;">svlg</span><span style="color: #0000FF;">(</span><span style="color: #000000;">s</span><span style="color: #0000FF;">),</span><span style="color: #000000;">svlg</span><span style="color: #0000FF;">(</span><span style="color: #000000;">decoded</span><span style="color: #0000FF;">)})</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">decoded</span><span style="color: #0000FF;">!=</span><span style="color: #000000;">testNumbers</span> <span style="color: #008080;">then</span> <span style="color: #7060A8;">crash</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"something wrong"</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<!--
function svlg(sequence s)
return join(s,':',fmt:="#%02x")
end function
constant testNumbers = { #200000, #1FFFFF, #00, #01, #7F, #80 }
sequence s = vlq_encode(testNumbers), decoded = vlq_decode(s)
printf(1,"%s -> %s -> %s\n",{svlg(testNumbers),svlg(s),svlg(decoded)})
if decoded!=testNumbers then crash("something wrong") end if

View file

@ -0,0 +1,34 @@
local fmt = require "fmt"
local function to_octets(n)
local s = fmt.bin(n)
local le = #s
local r = le % 7
local d = le // 7
if r > 0 then
d += 1
s = fmt.lpad(s, 7 * d, "0")
end
local chunks = s:split(""):chunk(7):map(|ch| -> ch:concat(""))
local last = "0" .. chunks:back()
s = chunks:slice(1, -2):map(|ch| -> "1" .. ch):concat("") .. last
return s:split(""):chunk(8):map(|ch| -> tonumber(ch:concat(""), 2))
end
local function from_octets(octets)
local s = ""
for octets as oct do
local bin = fmt.bin(oct)
bin = fmt.lpad(bin, 7, "0")
s ..= bin:sub(-7)
end
return tonumber(s, 2)
end
local tests = {2097152, 2097151}
for tests as test do
local octets = to_octets(test)
local display = octets:mapped(|oct| -> "Ox" .. string.format("%02x", oct))
io.write($"{test} -> {fmt.swrite(display, " ", "")} -> ")
print(from_octets(octets))
end

View file

@ -0,0 +1,33 @@
fn vlq_encode_4bytes(n u32) []u8 {
mut bytes := []u8{len: 4}
bytes[0] = u8((n & 0x7f) | 0x80)
bytes[1] = u8(((n >> 7) & 0x7f) | 0x80)
bytes[2] = u8(((n >> 14) & 0x7f) | 0x80)
bytes[3] = u8((n >> 21) & 0x7f) // cleared on last byte
return bytes
}
fn vlq_decode_4bytes(bytes []u8) u32 {
mut n := u32(0)
n |= u32(bytes[0] & 0x7f) << 0
n |= u32(bytes[1] & 0x7f) << 7
n |= u32(bytes[2] & 0x7f) << 14
n |= u32(bytes[3] & 0x7f) << 21
return n
}
fn byte_to_hex(b u8) string {
hex := b.hex()
return if hex.len == 1 { '0' + hex } else { hex }
}
fn main() {
values := [u32(0x200000), u32(0x1fffff)]
for x in values {
encoded := vlq_encode_4bytes(x)
hex_str := encoded.map(byte_to_hex).join('')
println('$x encodes into ${encoded.len} bytes: $hex_str')
decoded := vlq_decode_4bytes(encoded)
println('$decoded decoded')
}
}