31 lines
924 B
Text
31 lines
924 B
Text
main =>
|
||
NTop = 10,
|
||
File = "les_miserables.txt",
|
||
Chars = read_file_chars(File),
|
||
|
||
% Remove the Project Gutenberg header/footer
|
||
find(Chars,"*** START OF THE PROJECT GUTENBERG EBOOK LES MISÉRABLES ***",_,HeaderEnd),
|
||
find(Chars,"*** END OF THE PROJECT GUTENBERG EBOOK LES MISÉRABLES ***",FooterStart,_),
|
||
|
||
Book = [to_lowercase(C) : C in slice(Chars,HeaderEnd+1,FooterStart-1)],
|
||
|
||
% Split into words (different set of split characters)
|
||
member(SplitType,[all,space_punct,space]),
|
||
println(split_type=SplitType),
|
||
split_chars(SplitType,SplitChars),
|
||
Words = split(Book,SplitChars),
|
||
|
||
println(freq(Words).to_list.sort_down(2).take(NTop)),
|
||
nl,
|
||
fail.
|
||
|
||
freq(L) = Freq =>
|
||
Freq = new_map(),
|
||
foreach(E in L)
|
||
Freq.put(E,Freq.get(E,0)+1)
|
||
end.
|
||
|
||
% different set of split chars
|
||
split_chars(all,"\n\r \t,;!.?()[]”\"-“—-__‘’*").
|
||
split_chars(space_punct,"\n\r \t,;!.?").
|
||
split_chars(space,"\n\r \t").
|