September 2017 Update

This commit is contained in:
Ingy döt Net 2017-09-23 10:01:46 +02:00
parent bba7bfd280
commit ba8067c3b7
14570 changed files with 153136 additions and 63871 deletions

View file

@ -1,21 +0,0 @@
' ': 000
'a': 1000
'c': 01101
'd': 01100
'e': 0101
'f': 0010
'g': 010000
'h': 1101
'i': 0011
'l': 010001
'm': 1111
'n': 101
'o': 1110
'p': 10011
'r': 10010
's': 1100
't': 01111
'u': 01110
'x': 01001
encoded: 0111111010011110000000111100000100010100001010100110001111100110100010101000001011101001000011010111000100010111110001010000101101011011110011000011101010000
decoded: this is an example for huffman encoding

View file

@ -1,85 +0,0 @@
huffman_encoding_table = (counts) ->
# counts is a hash where keys are characters and
# values are frequencies;
# return a hash where keys are codes and values
# are characters
build_huffman_tree = ->
# returns a Huffman tree. Each node has
# cnt: total frequency of all chars in subtree
# c: character to be encoded (leafs only)
# children: children nodes (branches only)
q = min_queue()
for c, cnt of counts
q.enqueue cnt,
cnt: cnt
c: c
while q.size() >= 2
a = q.dequeue()
b = q.dequeue()
cnt = a.cnt + b.cnt
node =
cnt: cnt
children: [a, b]
q.enqueue cnt, node
root = q.dequeue()
root = build_huffman_tree()
codes = {}
encode = (node, code) ->
if node.c?
codes[code] = node.c
else
encode node.children[0], code + "0"
encode node.children[1], code + "1"
encode(root, "")
codes
min_queue = ->
# This is very non-optimized; you could use a binary heap for better
# performance. Items with smaller priority get dequeued first.
arr = []
enqueue: (priority, data) ->
i = 0
while i < arr.length
if priority < arr[i].priority
break
i += 1
arr.splice i, 0,
priority: priority
data: data
dequeue: ->
arr.shift().data
size: -> arr.length
_internal: ->
arr
freq_count = (s) ->
cnts = {}
for c in s
cnts[c] ?= 0
cnts[c] += 1
cnts
rpad = (s, n) ->
while s.length < n
s += ' '
s
examples = [
"this is an example for huffman encoding"
"abcd"
"abbccccddddddddeeeeeeeee"
]
for s in examples
console.log "---- #{s}"
counts = freq_count(s)
huffman_table = huffman_encoding_table(counts)
codes = (code for code of huffman_table).sort()
for code in codes
c = huffman_table[code]
console.log "#{rpad(code, 5)}: #{c} (#{counts[c]})"
console.log()

View file

@ -1,34 +0,0 @@
> coffee huffman.coffee
---- this is an example for huffman encoding
000 : n (4)
0010 : s (2)
0011 : m (2)
0100 : o (2)
01010: t (1)
01011: x (1)
01100: p (1)
01101: l (1)
01110: r (1)
01111: u (1)
10000: c (1)
10001: d (1)
1001 : i (3)
101 : (6)
1100 : a (3)
1101 : e (3)
1110 : f (3)
11110: g (1)
11111: h (2)
---- abcd
00 : a (1)
01 : b (1)
10 : c (1)
11 : d (1)
---- abbccccddddddddeeeeeeeee
0 : e (9)
1000 : a (1)
1001 : b (2)
101 : c (4)
11 : d (8)

View file

@ -2,23 +2,36 @@ import Data.List
import Control.Arrow
import Data.Ord
data HTree a = Leaf a | Branch (HTree a) (HTree a)
deriving (Show, Eq, Ord)
data HTree a
= Leaf a
| Branch (HTree a)
(HTree a)
deriving (Show, Eq, Ord)
test :: String -> IO ()
test s = mapM_ (\(a,b)-> putStrLn ('\'' : a : "\' : " ++ b))
. serialize . huffmanTree . freq $ s
test =
mapM_ (\(a, b) -> putStrLn ('\'' : a : "\' : " ++ b)) .
serialize . huffmanTree . freq
serialize :: HTree a -> [(a, String)]
serialize (Branch l r) = map (second('0':)) (serialize l) ++ map (second('1':)) (serialize r)
serialize (Leaf x) = [(x, "")]
serialize (Branch l r) =
(second ('0' :) <$> serialize l) ++ (second ('1' :) <$> serialize r)
serialize (Leaf x) = [(x, "")]
huffmanTree :: (Ord w, Num w) => [(w, a)] -> HTree a
huffmanTree = snd . head . until (null.tail) hstep
. sortBy (comparing fst) . map (second Leaf)
huffmanTree
:: (Ord w, Num w)
=> [(w, a)] -> HTree a
huffmanTree =
snd .
head . until (null . tail) hstep . sortBy (comparing fst) . (second Leaf <$>)
hstep :: (Ord a, Num a) => [(a, HTree b)] -> [(a, HTree b)]
hstep ((w1,t1):(w2,t2):wts) = insertBy (comparing fst) (w1 + w2, Branch t1 t2) wts
hstep
:: (Ord a, Num a)
=> [(a, HTree b)] -> [(a, HTree b)]
hstep ((w1, t1):(w2, t2):wts) =
insertBy (comparing fst) (w1 + w2, Branch t1 t2) wts
freq :: Ord a => [a] -> [(Int, a)]
freq = map (length &&& head) . group . sort
freq
:: Ord a
=> [a] -> [(Int, a)]
freq = ((length &&& head) <$>) . group . sort

View file

@ -1,3 +1,5 @@
import java.util.*
abstract class HuffmanTree(var freq: Int) : Comparable<HuffmanTree> {
override fun compareTo(other: HuffmanTree) = freq - other.freq
}

View file

@ -1,52 +0,0 @@
sub make_tree {
my %letters;
$letters{$_}++ for (split "", shift);
my (@nodes, $n) = map({ a=>$_, freq=>$letters{$_} }, keys %letters);
while ((@nodes = sort { $a->{freq} <=> $b->{freq}
or $a->{a} cmp $b->{a} } @nodes) > 1)
{
$n = { "0"=>shift(@nodes), "1"=>shift(@nodes) };
$n->{freq} = $n->{0}{freq} + $n->{1}{freq};
push @nodes, $n;
}
walk($n, "", $n->{tree} = {});
$n;
}
sub walk {
my ($n, $s, $h) = @_;
exists $n->{a} and do {
print "'$n->{a}': $s\n";
$h->{$n->{a}} = $s if $h;
return;
};
walk($n->{0}, $s.0, $h);
walk($n->{1}, $s.1, $h);
}
sub encode {
my ($s, $t) = @_;
$t = $t->{tree};
join("", map($t->{$_}, split("", $s)));
}
sub decode {
my @b = split("", shift);
my ($n, $out) = $_[0];
while (@b) {
$n = $n->{shift @b};
if ($n->{a}) {
$out .= $n->{a};
$n = $_[0];
}
}
$out;
}
my $text = "this is an example for huffman encoding";
my $tree = make_tree($text);
my $e = encode($text, $tree);
print "$e\n";
print decode($e, $tree), "\n";

View file

@ -1,21 +0,0 @@
'g': 00000
'l': 00001
'p': 00010
'r': 00011
't': 00100
'u': 00101
'h': 0011
'm': 0100
'o': 0101
'n': 011
's': 1000
'x': 10010
'c': 100110
'd': 100111
'a': 1010
'e': 1011
'f': 1100
'i': 1101
' ': 111
0010000111101100011111...111110101100000
this is an example for huffman encoding

View file

@ -0,0 +1,86 @@
use std::collections::BTreeMap;
use std::collections::binary_heap::BinaryHeap;
#[derive(Debug, Eq, PartialEq)]
enum NodeKind {
Internal(Box<Node>, Box<Node>),
Leaf(char),
}
#[derive(Debug, Eq, PartialEq)]
struct Node {
frequency: usize,
kind: NodeKind,
}
impl Ord for Node {
fn cmp(&self, rhs: &Self) -> std::cmp::Ordering {
rhs.frequency.cmp(&self.frequency)
}
}
impl PartialOrd for Node {
fn partial_cmp(&self, rhs: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(&rhs))
}
}
type HuffmanCodeMap = BTreeMap<char, Vec<u8>>;
fn main() {
let text = "this is an example for huffman encoding";
let mut frequencies = BTreeMap::new();
for ch in text.chars() {
*frequencies.entry(ch).or_insert(0) += 1;
}
let mut prioritized_frequencies = BinaryHeap::new();
for counted_char in frequencies {
prioritized_frequencies.push(Node {
frequency: counted_char.1,
kind: NodeKind::Leaf(counted_char.0),
});
}
while prioritized_frequencies.len() > 1 {
let left_child = prioritized_frequencies.pop().unwrap();
let right_child = prioritized_frequencies.pop().unwrap();
prioritized_frequencies.push(Node {
frequency: right_child.frequency + left_child.frequency,
kind: NodeKind::Internal(Box::new(left_child), Box::new(right_child)),
});
}
let mut codes = HuffmanCodeMap::new();
generate_codes(
prioritized_frequencies.peek().unwrap(),
vec![0u8; 0],
&mut codes,
);
for item in codes {
print!("{}: ", item.0);
for bit in item.1 {
print!("{}", bit);
}
println!();
}
}
fn generate_codes(node: &Node, prefix: Vec<u8>, out_codes: &mut HuffmanCodeMap) {
match node.kind {
NodeKind::Internal(ref left_child, ref right_child) => {
let mut left_prefix = prefix.clone();
left_prefix.push(0);
generate_codes(&left_child, left_prefix, out_codes);
let mut right_prefix = prefix;
right_prefix.push(1);
generate_codes(&right_child, right_prefix, out_codes);
}
NodeKind::Leaf(ch) => {
out_codes.insert(ch, prefix);
}
}
}

View file

@ -0,0 +1,23 @@
fcn buildHuffman(text){ //-->(encode dictionary, decode dictionary)
ft:=Dictionary();
foreach c in (text){ ft[c]=ft.find(c,0)+1 } // leafs w/count
// build the tree, which is a list of lists of ...
tree:=ft.pump(List,fcn([(c,cnt)]){ //-->L( (cnt, ((sym,code))), ...)
L(cnt, L(L(c,"")))
}).copy(); // make it writable
while(tree.len()>1){ // fake up a [lame] priorty queue
tree=tree.sort(fcn(a,b){ a[0]>b[0] }); //prioritize high to low
a,b:=tree.pop(-2,2); //remove 2 least frequent symbols
mc:=fcn(n,c){ n[1] = c + n[1]; }; //(sym,code),"0"|"1"
a[1].apply2(mc,"0"); b[1].apply2(mc,"1"); // mc(a[1],"0")
tree.append( L(a[0]+b[0],a[1].extend(b[1])) ); //(a,b)-->new node
}//-->L(L(39, L( L(" ","000"),L("e","0010"),L("a","0011") ...
tree=tree[0][1].pump(List,fcn(i){ // flatten rather than traverse
if(T.isType(i))return(Void.Recurse,i,self.fcn); i });
encodeTable:=tree.toDictionary(); // symbol:Huffman code
decodeTable:=encodeTable.pump(Dictionary(),"reverse"); // code:symbol
return(encodeTable,decodeTable);
}

View file

@ -0,0 +1,8 @@
fcn encode(text,table){ text.pump(String,table.get) }
fcn decode(bits,table){ // this is a horrible decoder, for testing only
w:=bits.walker(); sink:=Sink(String);
try{ s:=""; while(1){
s+=w.next(); if(c:=table.find(s)) { sink.write(c); s=""; }
}}catch(TheEnd){}
sink.close();
}

View file

@ -0,0 +1,10 @@
text:="this is an example for huffman encoding";
encodeTable,decodeTable := buildHuffman(text);
encodeTable.pump(Console.println,fcn(kv){"%s : %s".fmt(kv.xplode())});
e:=encode(text,encodeTable);
"Encode %d characters (%d bits) to %d bits (%d bytes):"
.fmt(text.len(),text.len()*8,e.len(),(e.len()+7)/8).println();
println(e);
0'|Bits decoded to: "%s"|.fmt(decode(e,decodeTable)).println();