Data update

This commit is contained in:
Ingy döt Net 2025-08-11 18:05:26 -07:00
parent 4d5544505c
commit 4924dd0264
3073 changed files with 55820 additions and 4408 deletions

View file

@ -0,0 +1,40 @@
(defun entropy (str / log2 eCalc strList len ent cnt n new end cPrev)
(defun log2 (x)
(if (> x 0)
(/ (log x) (log 2.0))
)
)
(defun eCalc (cnt len / p)
(setq p (/ (float cnt) len))
(* p (log2 p))
)
(setq
strList (acad_strlsort (mapcar 'chr (vl-string->list str)))
len (length strList)
ent 0.0
cnt 0
n 0
)
(foreach c strList
(setq
n (1+ n)
new (and cPrev (not (eq c cPrev)))
end (= n len)
)
(if (or new end)
(setq
cnt (if (and end (not new)) (1+ cnt) cnt)
ent (+ ent (eCalc cnt len))
cnt 0
)
)
(if (and new end)
(setq ent (+ ent (eCalc 1 len)))
)
(setq
cnt (1+ cnt)
cPrev c
)
)
(* -1 ent)
)

View file

@ -0,0 +1,16 @@
# h should be a DuckDB histogram and n should be map_values(h).list_sum()
create or replace function h2e(h, n) as (
- map_values(h).list_transform( x -> x * log2(x/n)).list_sum() / n
);
# str is normally a string but if not, will be cast to one
create or replace function shannon_entropy(str) as (
with cte as (
from unnest( string_split(str::VARCHAR,'')) _(s)
),
n as (select count(*) as n from cte),
h as (select histogram(s) as h from cte)
select h2e(h,n) from h, n
);
select shannon_entropy(1223334444);

View file

@ -38,5 +38,5 @@ public program()
infoC *= -1;
console.printLine("The Entropy of ", input, " is ", infoC)
Console.printLine("The Entropy of ", input, " is ", infoC)
}