Data update
This commit is contained in:
Ingy döt Net
parent
4bb20c9b71
commit
cbaf4c4b64
12390 changed files with 318560 additions and 27248 deletions
|
|
@ -0,0 +1,102 @@
|
|||
with Aws.Client, Aws.Messages, Aws.Response, Aws.Resources, Aws.Url;
|
||||
with Dom.Readers, Dom.Core, Dom.Core.Documents, Dom.Core.Nodes, Dom.Core.Attrs;
|
||||
with Input_Sources.Strings, Unicode, Unicode.Ces.Utf8;
|
||||
with Ada.Strings.Unbounded, Ada.Strings.Fixed, Ada.Text_IO, Ada.Command_Line;
|
||||
with Ada.Containers.Vectors;
|
||||
|
||||
use Aws.Client, Aws.Messages, Aws.Response, Aws.Resources, Aws.Url;
|
||||
use Dom.Readers, Dom.Core, Dom.Core.Documents, Dom.Core.Nodes, Dom.Core.Attrs;
|
||||
use Aws, Ada.Strings.Unbounded, Ada.Strings.Fixed, Input_Sources.Strings;
|
||||
use Ada.Text_IO, Ada.Command_Line;
|
||||
|
||||
procedure Count_Examples is
|
||||
|
||||
package Members_Vectors is new Ada.Containers.Vectors (
|
||||
Index_Type => Positive,
|
||||
Element_Type => Unbounded_String);
|
||||
use Members_Vectors;
|
||||
|
||||
Exemples : Vector;
|
||||
Nbr_Lg, Total : Natural := 0;
|
||||
|
||||
procedure Get_Vector (Category : in String; Mbr_Vector : in out Vector) is
|
||||
Reader : Tree_Reader;
|
||||
Doc : Document;
|
||||
List : Node_List;
|
||||
N : Node;
|
||||
A : Attr;
|
||||
Page : Aws.Response.Data;
|
||||
Uri_Xml : constant String :=
|
||||
"https://rosettacode.org/w/api.php?action=query&list=categorymembers"
|
||||
&
|
||||
"&format=xml&cmlimit=500&cmtitle=Category:";
|
||||
begin
|
||||
Page := Client.Get (Uri_Xml & Category, Follow_Redirection => True);
|
||||
if AWS.Response.Status_Code (Page) not in Messages.Success then
|
||||
raise Client.Connection_Error;
|
||||
end if;
|
||||
declare
|
||||
Xml : constant String := Message_Body (Page);
|
||||
Source : String_Input;
|
||||
begin
|
||||
Open
|
||||
(Xml'Unrestricted_Access,
|
||||
Unicode.Ces.Utf8.Utf8_Encoding,
|
||||
Source);
|
||||
Parse (Reader, Source);
|
||||
Close (Source);
|
||||
end;
|
||||
Doc := Get_Tree (Reader);
|
||||
List := Get_Elements_By_Tag_Name (Doc, "cm");
|
||||
for Index in 1 .. Length (List) loop
|
||||
N := Item (List, Index - 1);
|
||||
A := Get_Named_Item (Attributes (N), "title");
|
||||
Append (Mbr_Vector, To_Unbounded_String (Value (A)));
|
||||
end loop;
|
||||
Free (List);
|
||||
Free (Reader);
|
||||
end Get_Vector;
|
||||
|
||||
function Scan_Page (Title : String) return Natural is
|
||||
Page : Aws.Response.Data;
|
||||
File : Aws.Resources.File_Type;
|
||||
Buffer : String (1 .. 1024);
|
||||
Languages, Position, Last : Natural := 0;
|
||||
begin
|
||||
Page :=
|
||||
Client.Get
|
||||
("https://rosettacode.org/w/index.php?title=" &
|
||||
Aws.Url.Encode (Title) &
|
||||
"&action=raw",
|
||||
Follow_Redirection => True);
|
||||
AWS.Response.Message_Body (Page, File);
|
||||
while not End_Of_File (File) loop
|
||||
Resources.Get_Line (File, Buffer, Last);
|
||||
Position :=
|
||||
Index
|
||||
(Source => Buffer (Buffer'First .. Last),
|
||||
Pattern => "=={{header|");
|
||||
if Position > 0 then
|
||||
Languages := Languages + 1;
|
||||
end if;
|
||||
end loop;
|
||||
Close (File);
|
||||
return Languages;
|
||||
end Scan_Page;
|
||||
|
||||
begin
|
||||
Get_Vector ("Programming_Tasks", Exemples);
|
||||
|
||||
for I in First_Index (Exemples) .. Last_Index (Exemples) loop
|
||||
declare
|
||||
Title : constant String :=
|
||||
To_String (Members_Vectors.Element (Exemples, I));
|
||||
begin
|
||||
Nbr_Lg := Scan_Page (Title);
|
||||
Total := Total + Nbr_Lg;
|
||||
Put_Line (Title & " :" & Integer'Image (Nbr_Lg) & " exemples.");
|
||||
end;
|
||||
end loop;
|
||||
|
||||
Put_Line ("Total :" & Integer'Image (Total) & " exemples.");
|
||||
end Count_Examples;
|
||||
|
|
@ -14,63 +14,63 @@ handler RosettaCodeHandler type RUIhandler{initialUI =[ui], title = "Rosetta Cod
|
|||
l4 TextLabel{ layoutData = new GridLayoutData{ row = 2, column = 3 }, text = "0" };
|
||||
|
||||
dg1 DataGrid{ layoutData = new GridLayoutData{ row = 3, column = 1, horizontalSpan = 3 },
|
||||
pageSize = 10, showScrollbar = true,
|
||||
columns = [ new DataGridColumn{name = "title", displayName = "Task", width=220},
|
||||
new DataGridColumn{name = "count", displayName = "Count", width=100} ] };
|
||||
pageSize = 10, showScrollbar = true,
|
||||
columns = [ new DataGridColumn{name = "title", displayName = "Task", width=220},
|
||||
new DataGridColumn{name = "count", displayName = "Count", width=100} ] };
|
||||
|
||||
cmcontinue string?;
|
||||
title string?;
|
||||
allTasks Task[];
|
||||
|
||||
|
||||
restBindingTasks IHttp? = new HttpRest{
|
||||
restType = eglx.rest.ServiceType.TrueRest,
|
||||
request.uri = "http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=1&format=json"};
|
||||
request.uri = "http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=1&format=json"};
|
||||
|
||||
restBindingPageDetail IHttp? = new HttpRest{
|
||||
restType = eglx.rest.ServiceType.TrueRest,
|
||||
request.uri = "http://rosettacode.org/mw/index.php"};
|
||||
|
||||
request.uri = "http://rosettacode.org/mw/index.php"};
|
||||
|
||||
function b1_onClick(event Event in)
|
||||
call ProxyFunctions.listTasks("") using restBindingTasks
|
||||
returning to listTasksCallBack onException exceptionHandler;
|
||||
call ProxyFunctions.listTasks("") using restBindingTasks
|
||||
returning to listTasksCallBack onException exceptionHandler;
|
||||
end
|
||||
|
||||
function listTasksCallBack(retResult RosettaCodeJSON in)
|
||||
title = retResult.query.categorymembers[1].title;
|
||||
cmcontinue = retResult.queryContinue.categorymembers.cmcontinue;
|
||||
title = retResult.query.categorymembers[1].title;
|
||||
cmcontinue = retResult.queryContinue.categorymembers.cmcontinue;
|
||||
|
||||
call ProxyFunctions.fetchPageDetail(title) using restBindingPageDetail
|
||||
returning to pageDetailCallBack onException exceptionHandler;
|
||||
call ProxyFunctions.fetchPageDetail(title) using restBindingPageDetail
|
||||
returning to pageDetailCallBack onException exceptionHandler;
|
||||
end
|
||||
|
||||
function pageDetailCallBack(pageResults string in)
|
||||
count int = countSubstring("=={{header", pageResults);
|
||||
allTasks.appendElement(new Task { title = title, count = count });
|
||||
l2.text = l2.text as int + 1;
|
||||
l4.text = l4.text as int + count;
|
||||
|
||||
if(cmcontinue != null)
|
||||
call ProxyFunctions.listTasks(cmcontinue) using restBindingTasks
|
||||
returning to listTasksCallBack onException exceptionHandler;
|
||||
else
|
||||
dg1.data = allTasks as any[];
|
||||
end
|
||||
count int = countSubstring("=={{header", pageResults);
|
||||
allTasks.appendElement(new Task { title = title, count = count });
|
||||
l2.text = l2.text as int + 1;
|
||||
l4.text = l4.text as int + count;
|
||||
|
||||
if(cmcontinue != null)
|
||||
call ProxyFunctions.listTasks(cmcontinue) using restBindingTasks
|
||||
returning to listTasksCallBack onException exceptionHandler;
|
||||
else
|
||||
dg1.data = allTasks as any[];
|
||||
end
|
||||
end
|
||||
|
||||
function countSubstring(substr string in, str string in) returns(int)
|
||||
if(str.length() > 0 and substr.length() > 0)
|
||||
return (str.length() - str.replaceStr(subStr, "").length()) / subStr.length();
|
||||
else
|
||||
return 0;
|
||||
end
|
||||
if(str.length() > 0 and substr.length() > 0)
|
||||
return (str.length() - str.replaceStr(subStr, "").length()) / subStr.length();
|
||||
else
|
||||
return 0;
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
function exceptionHandler(exp AnyException in)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
record Task
|
||||
title string;
|
||||
count int;
|
||||
title string;
|
||||
count int;
|
||||
end
|
||||
|
|
|
|||
|
|
@ -3,16 +3,16 @@ package com.eglexamples.client;
|
|||
library ProxyFunctions
|
||||
|
||||
function listTasks(continueLocation String in) returns (RosettaCodeJSON) {
|
||||
@Rest{method = _GET, uriTemplate = "&cmcontinue={continueLocation}",
|
||||
requestFormat = None, responseFormat = JSON}
|
||||
@Rest{method = _GET, uriTemplate = "&cmcontinue={continueLocation}",
|
||||
requestFormat = None, responseFormat = JSON}
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
function fetchPageDetail(title String in) returns (String) {
|
||||
@Rest{method = _GET, uriTemplate = "?title={title}&action=raw",
|
||||
requestFormat = None, responseFormat = None}
|
||||
@Rest{method = _GET, uriTemplate = "?title={title}&action=raw",
|
||||
requestFormat = None, responseFormat = None}
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,17 @@
|
|||
local(root = json_deserialize(curl('http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=10&format=json')->result))
|
||||
local(tasks = array, title = string, urltitle = string, thiscount = 0, totalex = 0)
|
||||
with i in #root->find('query')->find('categorymembers') do => {^
|
||||
#thiscount = 0
|
||||
#title = #i->find('title')
|
||||
#urltitle = #i->find('title')
|
||||
#urltitle->replace(' ','_')
|
||||
|
||||
#title+': '
|
||||
local(src = curl('http://rosettacode.org/mw/index.php?title='+#urltitle->asBytes->encodeurl+'&action=raw')->result->asString)
|
||||
#thiscount = (#src->split('=={{header|'))->size - 1
|
||||
#thiscount < 0 ? #thiscount = 0
|
||||
#thiscount + ' examples.'
|
||||
#totalex += #thiscount
|
||||
'\r'
|
||||
#thiscount = 0
|
||||
#title = #i->find('title')
|
||||
#urltitle = #i->find('title')
|
||||
#urltitle->replace(' ','_')
|
||||
|
||||
#title+': '
|
||||
local(src = curl('http://rosettacode.org/mw/index.php?title='+#urltitle->asBytes->encodeurl+'&action=raw')->result->asString)
|
||||
#thiscount = (#src->split('=={{header|'))->size - 1
|
||||
#thiscount < 0 ? #thiscount = 0
|
||||
#thiscount + ' examples.'
|
||||
#totalex += #thiscount
|
||||
'\r'
|
||||
^}
|
||||
'Total: '+#totalex+' examples.'
|
||||
|
|
|
|||
|
|
@ -14,5 +14,5 @@
|
|||
e = find(s(ix(k):end)==34,1)-2;
|
||||
t = s(ix(k)+[0:e]); % task
|
||||
c = count_examples(['http://rosettacode.org',t]);
|
||||
printf('Task "%s" has %i examples.\n',t(7:end), c);
|
||||
printf('Task "%s" has %i examples.\n',t(7:end), c);
|
||||
end;
|
||||
|
|
|
|||
|
|
@ -14,24 +14,24 @@ ConvertUTF8 := proc( str )
|
|||
end try;
|
||||
end proc:
|
||||
print_examples := proc(lst)
|
||||
local task, count, url, headers, item;
|
||||
for task in lst do
|
||||
count := 0:
|
||||
url := cat("http://www.rosettacode.org/mw/index.php?title=", StringTools:-Encode(StringTools:-SubstituteAll(task["title"], " ", "_"), 'percent'), "&action=raw"):
|
||||
headers := [StringTools:-SearchAll("=={{header|",URL:-Get(url))]:
|
||||
for item in headers do
|
||||
count++:
|
||||
end do:
|
||||
printf("%s has %d examples\n",ConvertUTF8(task["title"]), count);
|
||||
end do:
|
||||
local task, count, url, headers, item;
|
||||
for task in lst do
|
||||
count := 0:
|
||||
url := cat("http://www.rosettacode.org/mw/index.php?title=", StringTools:-Encode(StringTools:-SubstituteAll(task["title"], " ", "_"), 'percent'), "&action=raw"):
|
||||
headers := [StringTools:-SearchAll("=={{header|",URL:-Get(url))]:
|
||||
for item in headers do
|
||||
count++:
|
||||
end do:
|
||||
printf("%s has %d examples\n",ConvertUTF8(task["title"]), count);
|
||||
end do:
|
||||
end proc:
|
||||
|
||||
|
||||
x := JSON:-ParseFile("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=20&format=json"):
|
||||
print_examples(x["query"]["categorymembers"]);
|
||||
while(assigned(x["continue"]["cmcontinue"])) do
|
||||
continue := x["continue"]["cmcontinue"]:
|
||||
more_tasks:= cat("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=20&format=json", "&continue=", x["continue"]["continue"], "&cmcontinue=", x["continue"]["cmcontinue"]):
|
||||
x := JSON:-ParseFile(more_tasks):
|
||||
print_examples(x["query"]["categorymembers"]);
|
||||
continue := x["continue"]["cmcontinue"]:
|
||||
more_tasks:= cat("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=20&format=json", "&continue=", x["continue"]["continue"], "&cmcontinue=", x["continue"]["cmcontinue"]):
|
||||
x := JSON:-ParseFile(more_tasks):
|
||||
print_examples(x["query"]["categorymembers"]);
|
||||
end do:
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@ titles = as.character( sapply(nodes, xmlGetAttr, "title") )
|
|||
headers <- list()
|
||||
counts <- list()
|
||||
for (i in 1:length(titles)){
|
||||
headers[[i]] <- getURL( paste("http://rosettacode.org/mw/index.php?title=", gsub(" ", "_", titles[i]), "&action=raw", sep="") )
|
||||
counts[[i]] <- strsplit(headers[[i]],split=" ")[[1]]
|
||||
counts[[i]] <- grep("\\{\\{header", counts[[i]])
|
||||
cat(titles[i], ":", length(counts[[i]]), "examples\n")
|
||||
headers[[i]] <- getURL( paste("http://rosettacode.org/mw/index.php?title=", gsub(" ", "_", titles[i]), "&action=raw", sep="") )
|
||||
counts[[i]] <- strsplit(headers[[i]],split=" ")[[1]]
|
||||
counts[[i]] <- grep("\\{\\{header", counts[[i]])
|
||||
cat(titles[i], ":", length(counts[[i]]), "examples\n")
|
||||
}
|
||||
cat("Total: ", length(unlist(counts)), "examples\n")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
Rebol [
|
||||
title: "Rosetta code: Rosetta_Code/Count_examples"
|
||||
file: %Rosetta_Code-Count_examples.r3
|
||||
url: https://rosettacode.org/wiki/Rosetta_Code/Count_examples
|
||||
]
|
||||
|
||||
;; Import caching and HTML decoding utilities
|
||||
import thru-cache
|
||||
import html-entities
|
||||
|
||||
;; Set HTTPS request timeout to 30 seconds
|
||||
system/schemes/https/spec/timeout: 30
|
||||
|
||||
get-all-task-titles: function [
|
||||
"Scrapes all Rosetta Code task titles"
|
||||
][
|
||||
;; Build the base API URL for querying the Programming Tasks category
|
||||
base-url: rejoin [
|
||||
https://rosettacode.org/w/api.php
|
||||
"?action=query&format=xml&list=categorymembers&cmlimit=500"
|
||||
"&cmtitle=Category:Programming_Tasks"
|
||||
]
|
||||
que: to block! base-url ;; Initialize queue with the starting URL
|
||||
titles: copy [] ;; Will hold task titles as [pageid title]
|
||||
|
||||
;; Process each URL in the queue (handles pagination across subcategory pages)
|
||||
while [not empty? que][
|
||||
;; Dequeue and read the next URL
|
||||
url: take que
|
||||
print ["Reading:" as-blue find/tail url "&cmtitle="]
|
||||
xml: read-thru/string url
|
||||
|
||||
cmcontinue: none ;; Will hold the "next page" subcategory token, if present
|
||||
parse xml [
|
||||
;; Try to extract the pagination continuation token
|
||||
opt [
|
||||
thru {<continue cmcontinue="}
|
||||
copy cmcontinue: to #"^""
|
||||
]
|
||||
thru <categorymembers>
|
||||
;; Extract pageid and title from each <cm> element
|
||||
any [
|
||||
thru {<cm pageid="} copy pageid: to #"^""
|
||||
thru { title="} copy title: to #"^""
|
||||
(
|
||||
;; Store pageid as integer alongside title
|
||||
repend titles [to integer! pageid title]
|
||||
)
|
||||
]
|
||||
]
|
||||
;; If a cmcontinue token was found, enqueue the continuation URL
|
||||
if cmcontinue [
|
||||
append que rejoin [base-url "&cmcontinue=" cmcontinue]
|
||||
]
|
||||
]
|
||||
titles
|
||||
]
|
||||
|
||||
get-task-examples: function[
|
||||
"Return language names with solution for a given task"
|
||||
task
|
||||
][
|
||||
;; Decode HTML entities and re-encode for use in a URL
|
||||
task: enhex decode 'html-entities task
|
||||
url: join https://rosettacode.org/w/index.php?action=raw&title= task
|
||||
data: read-thru/string url
|
||||
;; Collect all language header names from the raw wiki markup
|
||||
parse data [
|
||||
collect any [thru "=={{header|" keep to "}}=="]
|
||||
]
|
||||
]
|
||||
|
||||
;; Fetch all task titles from Rosetta Code
|
||||
titles: get-all-task-titles
|
||||
|
||||
;; Count language examples per task
|
||||
counts: make block! length? titles
|
||||
foreach [pageid title] titles [
|
||||
try/with [
|
||||
langs: get-task-examples title
|
||||
num: length? langs
|
||||
print [title as-green num]
|
||||
;; Store [count title] pairs for later sorting
|
||||
repend counts [num title]
|
||||
] :print ;; On error, print the error and continue
|
||||
]
|
||||
|
||||
;; Sort by example count, descending
|
||||
sort/skip/reverse counts 2
|
||||
|
||||
;; Display the 10 tasks with the most language examples
|
||||
print as-yellow "^/Top 10 tasks with the most examples:"
|
||||
loop 10 [
|
||||
print [counts/2 "has" counts/1 "examples."]
|
||||
counts: skip counts 2
|
||||
]
|
||||
|
||||
;; Seek to the last 20 entries (10 pairs) to find the least-covered tasks
|
||||
print as-yellow "^/Top 10 tasks with the minimum examples:"
|
||||
counts: skip tail counts -20
|
||||
loop 10 [
|
||||
print [counts/2 "has" counts/1 "examples."]
|
||||
counts: skip counts 2
|
||||
]
|
||||
|
||||
;; Timestamp when the data fetch completed
|
||||
print ["^/Data received:" as-green now]
|
||||
|
||||
;; Optionaly clear all localy stored pages
|
||||
; clear-thru/only https://rosettacode.org/*
|
||||
|
|
@ -1,28 +1,28 @@
|
|||
html "<table border=1><tr bgcolor=wheat align=center><td>Num</td><td>Task</td><td>Examples</td></tr>"
|
||||
|
||||
a$ = httpGet$("http://rosettacode.org/wiki/Category:Programming_Tasks")
|
||||
a$ = word$(a$,1,"</table></div>")
|
||||
i = instr(a$,"<a href=""/wiki/")
|
||||
i = instr(a$,"<a href=""/wiki/",i+1)
|
||||
a$ = httpGet$("http://rosettacode.org/wiki/Category:Programming_Tasks")
|
||||
a$ = word$(a$,1,"</table></div>")
|
||||
i = instr(a$,"<a href=""/wiki/")
|
||||
i = instr(a$,"<a href=""/wiki/",i+1)
|
||||
while i > 0
|
||||
count = count + 1
|
||||
i = instr(a$,"<a href=""/wiki/",i+1)
|
||||
j = instr(a$,">",i+5)
|
||||
a1$ = mid$(a$,i+15,j-i)
|
||||
count = count + 1
|
||||
i = instr(a$,"<a href=""/wiki/",i+1)
|
||||
j = instr(a$,">",i+5)
|
||||
a1$ = mid$(a$,i+15,j-i)
|
||||
taskId$ = word$(a1$,1,"""")
|
||||
task$ = word$(a1$,3,"""")
|
||||
url$ = "http://rosettacode.org/wiki/";taskId$
|
||||
a2$ = httpGet$(url$)
|
||||
ii = instr(a2$,"<span class=""tocnumber"">")
|
||||
jj = 0
|
||||
url$ = "http://rosettacode.org/wiki/";taskId$
|
||||
a2$ = httpGet$(url$)
|
||||
ii = instr(a2$,"<span class=""tocnumber"">")
|
||||
jj = 0
|
||||
while ii > 0
|
||||
jj = ii
|
||||
ii = instr(a2$,"<span class=""tocnumber"">",ii+10)
|
||||
jj = ii
|
||||
ii = instr(a2$,"<span class=""tocnumber"">",ii+10)
|
||||
wend
|
||||
if jj = 0 then
|
||||
examp = 0
|
||||
else
|
||||
kk = instr(a2$,"<",jj+24)
|
||||
kk = instr(a2$,"<",jj+24)
|
||||
examp = int(val(mid$(a2$,jj+24,kk-jj-24)))
|
||||
end if
|
||||
html "<tr><td align=right>";count;"</td><td>";task$;"</td><td align=right>";examp;"</td></tr>"
|
||||
|
|
|
|||
|
|
@ -5,10 +5,10 @@ fcn getTasks(language){
|
|||
do{
|
||||
page:=CURL().get(("http://rosettacode.org/mw/api.php?"
|
||||
"action=query&cmlimit=500"
|
||||
"&format=json"
|
||||
"&list=categorymembers"
|
||||
"&cmtitle=Category:%s"
|
||||
"&cmcontinue=%s").fmt(language,continueValue));
|
||||
"&format=json"
|
||||
"&list=categorymembers"
|
||||
"&cmtitle=Category:%s"
|
||||
"&cmcontinue=%s").fmt(language,continueValue));
|
||||
page=page[0].del(0,page[1]); // get rid of HTML header
|
||||
json:=YAJL().write(page).close();
|
||||
json["query"]["categorymembers"].pump(tasks,T("get","title"));
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue