Data commit
This commit is contained in:
parent
7387c8f97b
commit
cb5bb5e222
199093 changed files with 3378972 additions and 0 deletions
3
Task/Rosetta-Code-Count-examples/00-META.yaml
Normal file
3
Task/Rosetta-Code-Count-examples/00-META.yaml
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
---
|
||||
from: http://rosettacode.org/wiki/Rosetta_Code/Count_examples
|
||||
note: Rosetta Code related
|
||||
16
Task/Rosetta-Code-Count-examples/00-TASK.txt
Normal file
16
Task/Rosetta-Code-Count-examples/00-TASK.txt
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
Find the total number of programming examples for each [[:Category:Programming Tasks|task]] and the total for all tasks.
|
||||
|
||||
Essentially, count the number of occurrences of <tt>=={{header|</tt> on each task page.
|
||||
|
||||
Output:
|
||||
|
||||
<pre>100 doors: 20 examples.
|
||||
99 Bottles of Beer: 29 examples.
|
||||
Abstract type: 10 examples.
|
||||
|
||||
Total: X examples.</pre>
|
||||
|
||||
For a full output, updated periodically, see [[Rosetta Code/Count examples/Full list]].
|
||||
|
||||
You'll need to use the Media Wiki API, which you can find out about locally, [http://rosettacode.org/mw/api.php here], or in Media Wiki's API documentation at, [http://www.mediawiki.org/wiki/API_Query API:Query]
|
||||
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
with Aws.Client, Aws.Messages, Aws.Response, Aws.Resources, Aws.Url;
|
||||
with Dom.Readers, Dom.Core, Dom.Core.Documents, Dom.Core.Nodes, Dom.Core.Attrs;
|
||||
with Input_Sources.Strings, Unicode, Unicode.Ces.Utf8;
|
||||
with Ada.Strings.Unbounded, Ada.Strings.Fixed, Ada.Text_IO, Ada.Command_Line;
|
||||
with Ada.Containers.Vectors;
|
||||
|
||||
use Aws.Client, Aws.Messages, Aws.Response, Aws.Resources, Aws.Url;
|
||||
use Dom.Readers, Dom.Core, Dom.Core.Documents, Dom.Core.Nodes, Dom.Core.Attrs;
|
||||
use Aws, Ada.Strings.Unbounded, Ada.Strings.Fixed, Input_Sources.Strings;
|
||||
use Ada.Text_IO, Ada.Command_Line;
|
||||
|
||||
procedure Count_Examples is
|
||||
|
||||
package Members_Vectors is new Ada.Containers.Vectors (
|
||||
Index_Type => Positive,
|
||||
Element_Type => Unbounded_String);
|
||||
use Members_Vectors;
|
||||
|
||||
Exemples : Vector;
|
||||
Nbr_Lg, Total : Natural := 0;
|
||||
|
||||
procedure Get_Vector (Category : in String; Mbr_Vector : in out Vector) is
|
||||
Reader : Tree_Reader;
|
||||
Doc : Document;
|
||||
List : Node_List;
|
||||
N : Node;
|
||||
A : Attr;
|
||||
Page : Aws.Response.Data;
|
||||
Uri_Xml : constant String :=
|
||||
"http://rosettacode.org/mw/api.php?action=query&list=categorymembers"
|
||||
&
|
||||
"&format=xml&cmlimit=500&cmtitle=Category:";
|
||||
begin
|
||||
Page := Client.Get (Uri_Xml & Category);
|
||||
if Response.Status_Code (Page) not in Messages.Success then
|
||||
raise Client.Connection_Error;
|
||||
end if;
|
||||
declare
|
||||
Xml : constant String := Message_Body (Page);
|
||||
Source : String_Input;
|
||||
begin
|
||||
Open
|
||||
(Xml'Unrestricted_Access,
|
||||
Unicode.Ces.Utf8.Utf8_Encoding,
|
||||
Source);
|
||||
Parse (Reader, Source);
|
||||
Close (Source);
|
||||
end;
|
||||
Doc := Get_Tree (Reader);
|
||||
List := Get_Elements_By_Tag_Name (Doc, "cm");
|
||||
for Index in 1 .. Length (List) loop
|
||||
N := Item (List, Index - 1);
|
||||
A := Get_Named_Item (Attributes (N), "title");
|
||||
Append (Mbr_Vector, To_Unbounded_String (Value (A)));
|
||||
end loop;
|
||||
Free (List);
|
||||
Free (Reader);
|
||||
end Get_Vector;
|
||||
|
||||
function Scan_Page (Title : String) return Natural is
|
||||
Page : Aws.Response.Data;
|
||||
File : Aws.Resources.File_Type;
|
||||
Buffer : String (1 .. 1024);
|
||||
Languages, Position, Last : Natural := 0;
|
||||
begin
|
||||
Page :=
|
||||
Client.Get
|
||||
("http://rosettacode.org/mw/index.php?title=" &
|
||||
Aws.Url.Encode (Title) &
|
||||
"&action=raw");
|
||||
Response.Message_Body (Page, File);
|
||||
while not End_Of_File (File) loop
|
||||
Resources.Get_Line (File, Buffer, Last);
|
||||
Position :=
|
||||
Index
|
||||
(Source => Buffer (Buffer'First .. Last),
|
||||
Pattern => "=={{header|");
|
||||
if Position > 0 then
|
||||
Languages := Languages + 1;
|
||||
end if;
|
||||
end loop;
|
||||
Close (File);
|
||||
return Languages;
|
||||
end Scan_Page;
|
||||
|
||||
begin
|
||||
Get_Vector ("Programming_Tasks", Exemples);
|
||||
|
||||
for I in First_Index (Exemples) .. Last_Index (Exemples) loop
|
||||
declare
|
||||
Title : constant String :=
|
||||
To_String (Members_Vectors.Element (Exemples, I));
|
||||
begin
|
||||
Nbr_Lg := Scan_Page (Title);
|
||||
Total := Total + Nbr_Lg;
|
||||
Put_Line (Title & " :" & Integer'Image (Nbr_Lg) & " exemples.");
|
||||
end;
|
||||
end loop;
|
||||
|
||||
Put_Line ("Total :" & Integer'Image (Total) & " exemples.");
|
||||
end Count_Examples;
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
UrlDownloadToFile
|
||||
, http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml
|
||||
, tasks.xml
|
||||
FileRead, tasks, tasks.xml
|
||||
pos = 0
|
||||
quote = " ; "
|
||||
regtitle := "<cm.*?title=" . quote . "(.*?)" . quote
|
||||
While, pos := RegExMatch(tasks, regtitle, title, pos + 1)
|
||||
{
|
||||
UrlDownloadToFile
|
||||
, % "http://www.rosettacode.org/w/index.php?title=" . title1 . "&action=raw"
|
||||
, task.xml
|
||||
FileRead, task, task.xml
|
||||
RegExReplace(task, "\{\{header\|", "", count)
|
||||
current := title1 . ": " . count . " examples.`n"
|
||||
output .= current
|
||||
TrayTip, current, % current
|
||||
}
|
||||
MsgBox % output
|
||||
Return
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
VDU 23,22,640;512;8,16,16,128+8 : REM Enable UTF-8 support
|
||||
|
||||
SYS "LoadLibrary", "URLMON.DLL" TO urlmon%
|
||||
SYS "GetProcAddress", urlmon%, "URLDownloadToFileA" TO UDTF
|
||||
special$ = "+()'"
|
||||
|
||||
url$ = "http://www.rosettacode.org/w/api.php?action=query" + \
|
||||
\ "&list=categorymembers&cmtitle=Category:Programming_Tasks" + \
|
||||
\ "&cmlimit=500&format=xml"
|
||||
file$ = @tmp$ + "tasks.xml"
|
||||
SYS UDTF, 0, url$, file$, 0, 0 TO fail%
|
||||
IF fail% ERROR 100, "File download failed (tasks)"
|
||||
|
||||
Total% = 0
|
||||
file% = OPENIN(file$)
|
||||
WHILE NOT EOF#file%
|
||||
a$ = GET$#file%
|
||||
i% = 0
|
||||
REPEAT
|
||||
i% = INSTR(a$, "title=", i%+1)
|
||||
IF i% THEN
|
||||
j% = INSTR(a$, ">", i%)
|
||||
title$ = MID$(a$, i%+7, j%-i%-10)
|
||||
REM Replace HTML codes:
|
||||
REPEAT
|
||||
k% = INSTR(title$, "&")
|
||||
IF k% THEN
|
||||
l% = INSTR(title$, ";", k%)
|
||||
title$ = LEFT$(title$,k%-1) + \
|
||||
\ FNhtml(MID$(title$,k%,l%-k%+1)) + MID$(title$,l%+1)
|
||||
ENDIF
|
||||
UNTIL k% = 0
|
||||
t$ = title$
|
||||
REM Substitute characters not allowed in a URL:
|
||||
FOR s% = 1 TO LEN(special$)
|
||||
REPEAT
|
||||
s$ = MID$(special$, s%, 1)
|
||||
k% = INSTR(t$, s$)
|
||||
IF k% t$ = LEFT$(t$,k%-1) + "%" + STR$~ASCs$ + MID$(t$,k%+1)
|
||||
UNTIL k% = 0
|
||||
NEXT
|
||||
url$ = "http://www.rosettacode.org/w/index.php?title=" + t$ + \
|
||||
\ "&action=raw"
|
||||
file$ = @tmp$ + "title.htm"
|
||||
SYS UDTF, 0, url$, file$, 0, 0 TO fail%
|
||||
IF fail% ERROR 100, "File download failed " + t$
|
||||
examples% = 0
|
||||
task% = OPENIN(file$)
|
||||
WHILE NOT EOF#task%
|
||||
IF INSTR(GET$#task%, "=={{header|") examples% += 1
|
||||
ENDWHILE
|
||||
CLOSE #task%
|
||||
Total% += examples%
|
||||
PRINT title$ ": " ; examples% " examples."
|
||||
ENDIF
|
||||
UNTIL i% = 0
|
||||
i% = INSTR(a$, "cmcontinue=")
|
||||
IF i% THEN
|
||||
CLOSE #file%
|
||||
j% = INSTR(a$, """", i%+1)
|
||||
k% = INSTR(a$, """", j%+1)
|
||||
url$ = "http://www.rosettacode.org/w/api.php?action=query" + \
|
||||
\ "&list=categorymembers&cmtitle=Category:Programming_Tasks" + \
|
||||
\ "&cmlimit=500&format=xml&cmcontinue=" + MID$(a$,j%+1,k%-j%)
|
||||
REPEAT
|
||||
i% = INSTR(url$, "|")
|
||||
IF i% url$ = LEFT$(url$,i%-1) + "%7C" + MID$(url$,i%+1)
|
||||
UNTIL i% = 0
|
||||
file$ = @tmp$ + "tasks.xml"
|
||||
SYS UDTF, 0, url$, file$, 0, 0 TO fail%
|
||||
IF fail% ERROR 100, "File download failed (continue)"
|
||||
file% = OPENIN(file$)
|
||||
ENDIF
|
||||
ENDWHILE
|
||||
CLOSE #file%
|
||||
PRINT ' "Total: " ; Total% " examples."
|
||||
END
|
||||
|
||||
DEF FNhtml(h$)
|
||||
IF LEFT$(h$,2) = "&#" THEN = CHR$(VALMID$(h$,3))
|
||||
CASE h$ OF
|
||||
WHEN """: = """"
|
||||
ENDCASE
|
||||
= h$
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
( ( get-page
|
||||
=
|
||||
. sys$(str$("wget -q -O wget.out \"" !arg \"))
|
||||
& get$("wget.out",HT ML)
|
||||
)
|
||||
& get-page$"http://rosettacode.org/wiki/Category:Programming_Tasks"
|
||||
: ? (table.?) ?tasklist (.table.) ?
|
||||
& 0:?list
|
||||
& whl
|
||||
' ( !tasklist
|
||||
: ?
|
||||
( a
|
||||
. (href.@(?:"/wiki/" ?href)) (title.?title)
|
||||
& get-page$(str$("http://rosettacode.org/wiki/" !href))
|
||||
: ?task
|
||||
& 0:?cnt
|
||||
& whl
|
||||
' ( !task
|
||||
: ?
|
||||
( (span.(class.mw-headline) (id.?))
|
||||
?span
|
||||
(.span.)
|
||||
?task
|
||||
& !span
|
||||
: ?
|
||||
( a
|
||||
. (href.@(?:"/wiki/Category:" ?))
|
||||
(title.@(?:"Category:" ?))
|
||||
)
|
||||
@
|
||||
(.a.)
|
||||
?
|
||||
)
|
||||
& 1+!cnt:?cnt
|
||||
)
|
||||
& (!cnt.!title)+!list:?list
|
||||
)
|
||||
?tasklist
|
||||
)
|
||||
& lst$(list,taskfreq,NEW)
|
||||
)
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Net;
|
||||
|
||||
class Task {
|
||||
private string _task;
|
||||
private int _examples;
|
||||
|
||||
public Task(string task, int examples) {
|
||||
_task = task;
|
||||
_examples = examples;
|
||||
}
|
||||
|
||||
public string Name {
|
||||
get { return _task; }
|
||||
}
|
||||
|
||||
public int Examples {
|
||||
get { return _examples; }
|
||||
}
|
||||
|
||||
public override string ToString() {
|
||||
return String.Format("{0}: {1} examples.", this._task, this._examples);
|
||||
}
|
||||
}
|
||||
|
||||
class Program {
|
||||
static List<string> GetTitlesFromCategory(string category, WebClient wc) {
|
||||
string content = wc.DownloadString(
|
||||
String.Format("http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:{0}&cmlimit=500&format=json", category)
|
||||
);
|
||||
|
||||
return new Regex("\"title\":\"(.+?)\"").Matches(content).Cast<Match>().Select(x => x.Groups[1].Value.Replace("\\/", "/")).ToList();
|
||||
}
|
||||
|
||||
static string GetSourceCodeFromPage(string page, WebClient wc) {
|
||||
return wc.DownloadString(
|
||||
String.Format("http://www.rosettacode.org/w/index.php?title={0}&action=raw", page)
|
||||
);
|
||||
}
|
||||
|
||||
static void Main(string[] args) {
|
||||
WebClient wc = new WebClient();
|
||||
List<Task> tasks = new List<Task>();
|
||||
List<string> tasknames = GetTitlesFromCategory("Programming_Tasks", wc);
|
||||
|
||||
foreach (string task in tasknames) {
|
||||
try {
|
||||
string content = GetSourceCodeFromPage(WebUtility.UrlEncode(task), wc);
|
||||
int count = new Regex("=={{header", RegexOptions.IgnoreCase).Matches(content).Count;
|
||||
Task t = new Task(task, count);
|
||||
|
||||
Console.WriteLine(t);
|
||||
tasks.Add(t);
|
||||
}
|
||||
catch (Exception ex) {
|
||||
Console.WriteLine("**** Unable to get task \"" + task + "\": " + ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("\nTotal: {0} examples.", tasks.Select(x => x.Examples).Sum());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
(ns count-examples
|
||||
(:import [java.net URLEncoder])
|
||||
(:use [clojure.contrib.http.agent :only (http-agent string)]
|
||||
[clojure.contrib.json :only (read-json)]
|
||||
[clojure.contrib.string :only (join)]))
|
||||
|
||||
(defn url-encode [v] (URLEncoder/encode (str v) "utf-8"))
|
||||
|
||||
(defn rosettacode-get [path params]
|
||||
(let [param-string (join "&" (for [[n v] params] (str (name n) "=" (url-encode v))))]
|
||||
(string (http-agent (format "http://www.rosettacode.org/w/%s?%s" path param-string)))))
|
||||
|
||||
(defn rosettacode-query [params]
|
||||
(read-json (rosettacode-get "api.php" (merge {:action "query" :format "json"} params))))
|
||||
|
||||
(defn list-cm
|
||||
([params] (list-cm params nil))
|
||||
([params continue]
|
||||
(let [cm-params (merge {:list "categorymembers"} params (or continue {}))
|
||||
result (rosettacode-query cm-params)]
|
||||
(concat (-> result (:query) (:categorymembers))
|
||||
(if-let [cmcontinue (-> result (:query-continue) (:categorymembers))]
|
||||
(list-cm params cmcontinue))))))
|
||||
|
||||
(defn programming-tasks []
|
||||
(let [result (list-cm {:cmtitle "Category:Programming_Tasks" :cmlimit 50})]
|
||||
(map #(:title %) result)))
|
||||
|
||||
(defn task-count [task]
|
||||
[task (count
|
||||
(re-seq #"==\{\{header"
|
||||
(rosettacode-get "index.php" {:action "raw" :title task})))])
|
||||
|
||||
(defn print-result []
|
||||
(let [task-counts (map task-count (programming-tasks))]
|
||||
(doseq [[task count] task-counts]
|
||||
(println (str task ":") count)
|
||||
(flush))
|
||||
(println "Total: " (reduce #(+ %1 (second %2)) 0 task-counts))))
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
count-examples> (print-result)
|
||||
100 doors: 73
|
||||
24 game: 18
|
||||
24 game/Solve: 14
|
||||
99 Bottles of Beer: 89
|
||||
Abstract type: 27
|
||||
Accumulator factory: 23
|
||||
Ackermann function: 73
|
||||
Active Directory/Connect: 4
|
||||
Active Directory/Search for a user: 3
|
||||
Active object: 14
|
||||
Add a variable to a class instance at runtime: 21
|
||||
Address of a variable: 20
|
||||
...
|
||||
Total: 11216
|
||||
nil
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
import tango.io.Stdout;
|
||||
import tango.net.http.HttpClient;
|
||||
import tango.net.http.HttpHeaders;
|
||||
import tango.text.xml.Document;
|
||||
import tango.text.Util;
|
||||
|
||||
alias HttpHeader.ContentLength CL;
|
||||
|
||||
auto url = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
|
||||
void main()
|
||||
{
|
||||
auto client = new HttpClient (HttpClient.Get, url);
|
||||
client.open();
|
||||
char[] mainData, tmp;
|
||||
int total, i;
|
||||
|
||||
void cat(void[] content) { tmp ~= cast(char[]) content; }
|
||||
|
||||
if (client.isResponseOK) {
|
||||
client.read(&cat, client.getResponseHeaders.getInt(CL));
|
||||
mainData = tmp;
|
||||
tmp = null;
|
||||
|
||||
auto doc = new Document!(char);
|
||||
doc.parse(mainData);
|
||||
foreach (n; doc.query.descendant("cm").attribute("title")) {
|
||||
auto subClient = new HttpClient(HttpClient.Get,
|
||||
"http://www.rosettacode.org/w/index.php?title=" ~
|
||||
replace(n.value.dup, ' ', '_') ~ "&action=raw");
|
||||
subClient.open();
|
||||
if (! subClient.isResponseOK) {
|
||||
Stderr (client.getResponse);
|
||||
break;
|
||||
}
|
||||
subClient.read(&cat, subClient.getResponseHeaders.getInt(CL));
|
||||
foreach (segment; patterns(cast(char[])tmp, "=={{header|")) i++;
|
||||
--i;
|
||||
if (i) --i;
|
||||
Stdout.formatln ("{0,-40} - {}", n.value, i);
|
||||
total += i;
|
||||
tmp = null;
|
||||
i = 0;
|
||||
}
|
||||
Stdout("total examples: ", total).newline;
|
||||
} else {
|
||||
Stderr (client.getResponse);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
package com.eglexamples.client;
|
||||
|
||||
import org.eclipse.edt.rui.widgets.*;
|
||||
|
||||
handler RosettaCodeHandler type RUIhandler{initialUI =[ui], title = "Rosetta Code Tasks and Counts"}
|
||||
|
||||
ui GridLayout{columns = 3, rows = 4, cellPadding = 4, children = [ b1, dg1, l1, l2, l3, l4 ]};
|
||||
|
||||
b1 Button{ layoutData = new GridLayoutData{ row = 1, column = 1 }, text = "Go!", onClick ::= b1_onClick };
|
||||
l1 TextLabel{ layoutData = new GridLayoutData{ row = 1, column = 2 }, text = "Total Tasks:" };
|
||||
l2 TextLabel{ layoutData = new GridLayoutData{ row = 1, column = 3 }, text = "0" };
|
||||
|
||||
l3 TextLabel{ layoutData = new GridLayoutData{ row = 2, column = 2 }, text = "Total Implementations:" };
|
||||
l4 TextLabel{ layoutData = new GridLayoutData{ row = 2, column = 3 }, text = "0" };
|
||||
|
||||
dg1 DataGrid{ layoutData = new GridLayoutData{ row = 3, column = 1, horizontalSpan = 3 },
|
||||
pageSize = 10, showScrollbar = true,
|
||||
columns = [ new DataGridColumn{name = "title", displayName = "Task", width=220},
|
||||
new DataGridColumn{name = "count", displayName = "Count", width=100} ] };
|
||||
|
||||
cmcontinue string?;
|
||||
title string?;
|
||||
allTasks Task[];
|
||||
|
||||
restBindingTasks IHttp? = new HttpRest{
|
||||
restType = eglx.rest.ServiceType.TrueRest,
|
||||
request.uri = "http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=1&format=json"};
|
||||
|
||||
restBindingPageDetail IHttp? = new HttpRest{
|
||||
restType = eglx.rest.ServiceType.TrueRest,
|
||||
request.uri = "http://rosettacode.org/mw/index.php"};
|
||||
|
||||
function b1_onClick(event Event in)
|
||||
call ProxyFunctions.listTasks("") using restBindingTasks
|
||||
returning to listTasksCallBack onException exceptionHandler;
|
||||
end
|
||||
|
||||
function listTasksCallBack(retResult RosettaCodeJSON in)
|
||||
title = retResult.query.categorymembers[1].title;
|
||||
cmcontinue = retResult.queryContinue.categorymembers.cmcontinue;
|
||||
|
||||
call ProxyFunctions.fetchPageDetail(title) using restBindingPageDetail
|
||||
returning to pageDetailCallBack onException exceptionHandler;
|
||||
end
|
||||
|
||||
function pageDetailCallBack(pageResults string in)
|
||||
count int = countSubstring("=={{header", pageResults);
|
||||
allTasks.appendElement(new Task { title = title, count = count });
|
||||
l2.text = l2.text as int + 1;
|
||||
l4.text = l4.text as int + count;
|
||||
|
||||
if(cmcontinue != null)
|
||||
call ProxyFunctions.listTasks(cmcontinue) using restBindingTasks
|
||||
returning to listTasksCallBack onException exceptionHandler;
|
||||
else
|
||||
dg1.data = allTasks as any[];
|
||||
end
|
||||
end
|
||||
|
||||
function countSubstring(substr string in, str string in) returns(int)
|
||||
if(str.length() > 0 and substr.length() > 0)
|
||||
return (str.length() - str.replaceStr(subStr, "").length()) / subStr.length();
|
||||
else
|
||||
return 0;
|
||||
end
|
||||
end
|
||||
|
||||
function exceptionHandler(exp AnyException in)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
record Task
|
||||
title string;
|
||||
count int;
|
||||
end
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
package com.eglexamples.client;
|
||||
|
||||
library ProxyFunctions
|
||||
|
||||
function listTasks(continueLocation String in) returns (RosettaCodeJSON) {
|
||||
@Rest{method = _GET, uriTemplate = "&cmcontinue={continueLocation}",
|
||||
requestFormat = None, responseFormat = JSON}
|
||||
}
|
||||
end
|
||||
|
||||
function fetchPageDetail(title String in) returns (String) {
|
||||
@Rest{method = _GET, uriTemplate = "?title={title}&action=raw",
|
||||
requestFormat = None, responseFormat = None}
|
||||
}
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
record RosettaCodeJSON
|
||||
query Query;
|
||||
queryContinue QueryContinue{JSONName = "query-continue"};
|
||||
end
|
||||
|
||||
record Query
|
||||
categorymembers Categorymembers[];
|
||||
end
|
||||
|
||||
record Categorymembers
|
||||
cmcontinue string?;
|
||||
pageid int?;
|
||||
ns int?;
|
||||
title string?;
|
||||
end
|
||||
|
||||
record QueryContinue
|
||||
categorymembers Categorymembers;
|
||||
end
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
-module(rosseta_examples).
|
||||
-include_lib("xmerl/include/xmerl.hrl").
|
||||
|
||||
-export([main/0]).
|
||||
|
||||
main() ->
|
||||
application:start(inets),
|
||||
Titles = read_titles(empty),
|
||||
Result = lists:foldl(fun(Title,Acc) -> Acc + calculate_one(Title) end, 0, Titles),
|
||||
io:format("Total: ~p examples.\n",[Result]),
|
||||
application:stop(inets).
|
||||
|
||||
read_titles(CurrentContinue) ->
|
||||
URL0 = "http://rosettacode.org/mw/api.php?" ++
|
||||
"action=query&list=categorymembers&cmtitle=Category:Programming_Tasks" ++
|
||||
"&cmlimit=500&format=xml",
|
||||
URL =
|
||||
case CurrentContinue of
|
||||
empty -> URL0;
|
||||
_ -> URL0 ++ "&cmcontinue=" ++ CurrentContinue
|
||||
end,
|
||||
{ok,Answer} = httpc:request(URL),
|
||||
{Document,_} = xmerl_scan:string(lists:last(tuple_to_list(Answer))),
|
||||
Continue =
|
||||
[Value || #xmlAttribute{value = Value} <- xmerl_xpath:string("//@cmcontinue", Document)],
|
||||
Titles =
|
||||
[Value || #xmlAttribute{value = Value} <- xmerl_xpath:string("//@title", Document)],
|
||||
case Continue of
|
||||
[]->
|
||||
Titles;
|
||||
[ContValue | _] ->
|
||||
Titles ++ read_titles(ContValue)
|
||||
end.
|
||||
|
||||
calculate_one(Title0) ->
|
||||
Title = replace_chars(Title0),
|
||||
URL = "http://www.rosettacode.org/w/index.php?title=" ++
|
||||
Title ++ "&action=raw",
|
||||
case httpc:request(URL) of
|
||||
{ok,Result} ->
|
||||
{match,Found} =
|
||||
re:run(lists:last(tuple_to_list(Result)), "\n=={{header(|)", [global]),
|
||||
io:format("~ts: ~p examples.\n",[Title0,length(Found)]),
|
||||
length(Found);
|
||||
{error,socket_closed_remotely} ->
|
||||
io:format("Socket closed remotely. Retry.\n"),
|
||||
calculate_one(Title0)
|
||||
end.
|
||||
|
||||
replace_chars(String) ->
|
||||
replace_chars(String,[]).
|
||||
|
||||
replace_chars([$ | T],Acc) ->
|
||||
replace_chars(T, [$_| Acc]);
|
||||
replace_chars([$+| T],Acc) ->
|
||||
replace_chars(T, lists:reverse("%2B") ++ Acc);
|
||||
replace_chars([8211| T],Acc) ->
|
||||
replace_chars(T, lists:reverse("%E2%80%93") ++ Acc);
|
||||
replace_chars([Other| T],Acc) ->
|
||||
replace_chars(T, [Other| Acc]);
|
||||
replace_chars([],Acc) ->
|
||||
lists:reverse(Acc).
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
#r "System.Xml.Linq.dll"
|
||||
|
||||
let uri1 = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
|
||||
let uri2 task = sprintf "http://www.rosettacode.org/w/index.php?title=%s&action=raw" task
|
||||
|
||||
[|for xml in (System.Xml.Linq.XDocument.Load uri1).Root.Descendants() do
|
||||
for attrib in xml.Attributes() do
|
||||
if attrib.Name.LocalName = "title" then
|
||||
yield async {
|
||||
let uri = uri2 (attrib.Value.Replace(" ", "_") |> System.Web.HttpUtility.UrlEncode)
|
||||
use client = new System.Net.WebClient()
|
||||
let! html = client.AsyncDownloadString(System.Uri uri)
|
||||
let sols' = html.Split([|"{{header|"|], System.StringSplitOptions.None).Length - 1
|
||||
lock stdout (fun () -> printfn "%s: %d examples" attrib.Value sols')
|
||||
return sols' }|]
|
||||
|> Async.Parallel
|
||||
|> Async.RunSynchronously
|
||||
|> fun xs -> printfn "Total: %d examples" (Seq.sum xs)
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
USING: arrays assocs concurrency.combinators
|
||||
concurrency.semaphores formatting hashtables http.client io
|
||||
json.reader kernel math math.parser sequences splitting
|
||||
urls.encoding ;
|
||||
IN: rosetta-code.count-examples
|
||||
|
||||
CONSTANT: list-url "http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&cmprop=title&format=json"
|
||||
|
||||
: titles ( query -- titles )
|
||||
"query" of "categorymembers" of [ "title" of ] map ;
|
||||
: continued-url ( query -- url/f )
|
||||
"query-continue" of "categorymembers" of
|
||||
[ assoc>query list-url swap "&" glue ] [ f ] if* ;
|
||||
|
||||
: (all-programming-titles) ( titles url -- titles' url' )
|
||||
http-get nip json> [ titles append ] [ continued-url ] bi
|
||||
[ (all-programming-titles) ] [ f ] if* ;
|
||||
: all-programming-titles ( -- titles ) { } list-url (all-programming-titles) drop ;
|
||||
|
||||
CONSTANT: content-base-url "http://rosettacode.org/mw/index.php?title=&action=raw"
|
||||
: content-url ( title -- url )
|
||||
" " "_" replace
|
||||
"title" associate assoc>query
|
||||
content-base-url swap "&" glue ;
|
||||
|
||||
: occurences ( seq subseq -- n ) split-subseq length 1 - ;
|
||||
: count-examples ( title -- n )
|
||||
content-url http-get nip "=={{header|" occurences ;
|
||||
|
||||
: print-task ( title n -- ) "%s: %d examples.\n" printf ;
|
||||
: print-total ( assoc -- ) values sum "Total: %d examples.\n" printf ;
|
||||
: fetch-counts ( titles -- assoc )
|
||||
10 <semaphore> [
|
||||
[ dup count-examples 2array ] with-semaphore
|
||||
] curry parallel-map ;
|
||||
|
||||
: print-counts ( titles -- )
|
||||
[ [ print-task ] assoc-each nl ] [ print-total ] bi ;
|
||||
: rosetta-examples ( -- )
|
||||
all-programming-titles fetch-counts print-counts ;
|
||||
|
||||
MAIN: rosetta-examples
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func req(u string, foundCm func(string)) string {
|
||||
resp, err := http.Get(u)
|
||||
if err != nil {
|
||||
fmt.Println(err) // connection or request fail
|
||||
return ""
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
for p := xml.NewDecoder(resp.Body); ; {
|
||||
t, err := p.RawToken()
|
||||
switch s, ok := t.(xml.StartElement); {
|
||||
case err == io.EOF:
|
||||
return ""
|
||||
case err != nil:
|
||||
fmt.Println(err)
|
||||
return ""
|
||||
case !ok:
|
||||
continue
|
||||
case s.Name.Local == "cm":
|
||||
for _, a := range s.Attr {
|
||||
if a.Name.Local == "title" {
|
||||
foundCm(a.Value)
|
||||
}
|
||||
}
|
||||
case s.Name.Local == "categorymembers" && len(s.Attr) > 0 &&
|
||||
s.Attr[0].Name.Local == "cmcontinue":
|
||||
return url.QueryEscape(s.Attr[0].Value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func main() {
|
||||
taskQuery := "http://rosettacode.org/mw/api.php?action=query" +
|
||||
"&format=xml&list=categorymembers&cmlimit=500" +
|
||||
"&cmtitle=Category:Programming_Tasks"
|
||||
continueAt := req(taskQuery, count)
|
||||
for continueAt > "" {
|
||||
continueAt = req(taskQuery+"&cmcontinue="+continueAt, count)
|
||||
}
|
||||
fmt.Printf("Total: %d examples.\n", total)
|
||||
}
|
||||
|
||||
var marker = []byte("=={{header|")
|
||||
var total int
|
||||
|
||||
func count(cm string) {
|
||||
taskFmt := "http://rosettacode.org/mw/index.php?title=%s&action=raw"
|
||||
taskEsc := url.QueryEscape(strings.Replace(cm, " ", "_", -1))
|
||||
resp, err := http.Get(fmt.Sprintf(taskFmt, taskEsc))
|
||||
var page []byte
|
||||
if err == nil {
|
||||
page, err = ioutil.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
examples := bytes.Count(page, marker)
|
||||
fmt.Printf("%s: %d\n", cm, examples)
|
||||
total += examples
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
import Network.Browser
|
||||
import Network.HTTP
|
||||
import Network.URI
|
||||
import Data.List
|
||||
import Data.Maybe
|
||||
import Text.XML.Light
|
||||
import Control.Arrow
|
||||
|
||||
justifyR w = foldl ((.return).(++).tail) (replicate w ' ')
|
||||
showFormatted t n = t ++ ": " ++ justifyR 4 (show n)
|
||||
|
||||
getRespons url = do
|
||||
rsp <- Network.Browser.browse $ do
|
||||
setAllowRedirects True
|
||||
setOutHandler $ const (return ()) -- quiet
|
||||
request $ getRequest url
|
||||
return $ rspBody $ snd rsp
|
||||
|
||||
getNumbOfExampels p = do
|
||||
let pg = intercalate "_" $ words p
|
||||
rsp <- getRespons $ "http://www.rosettacode.org/w/index.php?title=" ++ pg ++ "&action=raw"
|
||||
let taskPage = rsp
|
||||
countEx = length $ filter (=="=={{header|") $ takeWhile(not.null) $ unfoldr (Just. (take 11 &&& drop 1)) taskPage
|
||||
return countEx
|
||||
|
||||
progTaskExamples = do
|
||||
rsp <- getRespons "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
|
||||
|
||||
let xmls = onlyElems $ parseXML $ rsp
|
||||
tasks = concatMap (map (fromJust.findAttr (unqual "title")). filterElementsName (== unqual "cm")) xmls
|
||||
|
||||
taskxx <- mapM getNumbOfExampels tasks
|
||||
let ns = taskxx
|
||||
tot = sum ns
|
||||
|
||||
mapM_ putStrLn $ zipWith showFormatted tasks ns
|
||||
putStrLn $ ("Total: " ++) $ show tot
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
*Main> progTaskExamples
|
||||
100 doors: 56
|
||||
24 game: 11
|
||||
24 game Player: 9
|
||||
99 Bottles of Beer: 73
|
||||
Abstract type: 23
|
||||
Ackermann Function: 61
|
||||
Active object: 9
|
||||
...
|
||||
Total: 9156
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
$define RCINDEX "http://rosettacode.org/mw/api.php?format=xml&action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500"
|
||||
$define RCTASK "http://rosettacode.org/mw/index.php?action=raw&title="
|
||||
$define RCUA "User-Agent: Unicon Rosetta 0.1"
|
||||
$define RCXUA "X-Unicon: http://unicon.org/"
|
||||
$define TASKTOT "* Total Tasks *"
|
||||
$define TOTTOT "* Total Headers*"
|
||||
|
||||
link strings
|
||||
link hexcvt
|
||||
|
||||
procedure main(A) # simple single threaded read all at once implementation
|
||||
Tasks := table(0)
|
||||
every task := taskNames() do {
|
||||
Tasks[TASKTOT] +:= 1 # count tasks
|
||||
every lang := languages(task) do { # count languages
|
||||
Tasks[task] +:= 1
|
||||
Tasks[TOTTOT] +:= 1
|
||||
}
|
||||
}
|
||||
every insert(O := set(),key(Tasks)) # extract & sort keys
|
||||
O := put(sort(O--set(TOTTOT,TASKTOT)),TASKTOT,TOTTOT) # move totals to end
|
||||
every write(k := !O, " : ", Tasks[k]," examples.") # report
|
||||
end
|
||||
|
||||
# Generate task names
|
||||
procedure taskNames()
|
||||
continue := ""
|
||||
while \(txt := ReadURL(RCINDEX||continue)) do {
|
||||
txt ? {
|
||||
while tab(find("<cm ") & find(s :="title=\"")+*s) do
|
||||
suspend tab(find("\""))\1
|
||||
if tab(find("cmcontinue=")) then {
|
||||
continue := "&"||tab(upto(' \t'))
|
||||
}
|
||||
else break
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
# Generate language headers in a task
|
||||
procedure languages(task)
|
||||
static WS
|
||||
initial WS := ' \t'
|
||||
page := ReadURL(RCTASK||CleanURI(task))
|
||||
page ? while (tab(find("\n==")),tab(many(WS))|"",tab(find("{{"))) do {
|
||||
header := tab(find("=="))
|
||||
header ? {
|
||||
while tab(find("{{header|")) do {
|
||||
suspend 2(="{{header|",tab(find("}}")))\1
|
||||
}
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
procedure CleanURI(u) #: clean up a URI
|
||||
static tr,dxml # xml & http translation
|
||||
initial {
|
||||
tr := table()
|
||||
every c := !string(~(&digits++&letters++'-_.!~*()/\'`')) do
|
||||
tr[c] := "%"||hexstring(ord(c),2)
|
||||
every /tr[c := !string(&cset)] := c
|
||||
tr[" "] := "_" # wiki convention
|
||||
every push(dxml := [],"&#"||right(ord(c := !"&<>'\""),3,"0")||";",c)
|
||||
}
|
||||
|
||||
dxml[1] := u # insert URI as 1st arg
|
||||
u := replacem!dxml # de-xml it
|
||||
every (c := "") ||:= tr[!u] # reencode everything
|
||||
c := replace(c,"%3E","'") # Hack to put single quotes back in
|
||||
c := replace(c,"%26quot%3B","\"") # Hack to put double quotes back in
|
||||
return c
|
||||
end
|
||||
|
||||
procedure ReadURL(url) #: read URL into string
|
||||
page := open(url,"m",RCUA,RCXUA) | stop("Unable to open ",url)
|
||||
text := ""
|
||||
if page["Status-Code"] < 300 then while text ||:= reads(page,-1)
|
||||
else write(&errout,image(url),": ",
|
||||
page["Status-Code"]," ",page["Reason-Phrase"])
|
||||
close(page)
|
||||
return text
|
||||
end
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
require 'web/gethttp'
|
||||
|
||||
getAllTaskSolnCounts=: monad define
|
||||
tasks=. getCategoryMembers 'Programming_Tasks'
|
||||
counts=. getTaskSolnCounts &> tasks
|
||||
tasks;counts
|
||||
)
|
||||
|
||||
getTaskSolnCounts=: monad define
|
||||
makeuri=. 'http://www.rosettacode.org/w/index.php?title=' , ,&'&action=raw'
|
||||
wikidata=. gethttp makeuri urlencode y
|
||||
([: +/ '{{header|'&E.) wikidata
|
||||
)
|
||||
|
||||
formatSolnCounts=: monad define
|
||||
'tasks counts'=. y
|
||||
tasks=. tasks , &.>':'
|
||||
res=. ;:^:_1 tasks ,. (8!:0 counts) ,. <'examples.'
|
||||
res , 'Total examples: ' , ": +/counts
|
||||
)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
formatSolnCounts getAllTaskSolnCounts ''
|
||||
100 doors: 61 examples.
|
||||
24 game: 15 examples.
|
||||
24 game Player: 11 examples.
|
||||
99 Bottles of Beer: 76 examples.
|
||||
...
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
import java.util.ArrayList;
|
||||
import ScreenScrape;
|
||||
|
||||
public class CountProgramExamples {
|
||||
private static final String baseURL = "http://rosettacode.org/wiki/";
|
||||
private static final String rootURL = "http://www.rosettacode.org/w/"
|
||||
+ "api.php?action=query&list=categorymembers"
|
||||
+ "&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
|
||||
private static final String taskBegin = "title=\"";
|
||||
private static final String taskEnd = "\"";
|
||||
private static final String exmplBegin = "<span class=\"tocnumber\">";
|
||||
private static final String exmplEnd = "</span>";
|
||||
private static final String editBegin = "<span class=\"editsection\">";
|
||||
|
||||
/**
|
||||
* @param args
|
||||
*/
|
||||
public static void main(String[] args) {
|
||||
int exTotal = 0;
|
||||
try {
|
||||
// Get root query results
|
||||
ArrayList<String> tasks = new ArrayList<String>();
|
||||
ScreenScrape ss = new ScreenScrape();
|
||||
String rootPage = ss.read(rootURL);
|
||||
while (rootPage.contains(taskBegin)) {
|
||||
rootPage = rootPage.substring(rootPage.indexOf(taskBegin)
|
||||
+ taskBegin.length());
|
||||
String title = rootPage.substring(0, rootPage.indexOf(taskEnd));
|
||||
if (!title.contains("Category:")) {
|
||||
tasks.add(title);
|
||||
}
|
||||
rootPage = rootPage.substring(rootPage.indexOf(taskEnd));
|
||||
}
|
||||
// Loop through each task and print count
|
||||
for (String task : tasks) {
|
||||
String title = task.replaceAll("'", "'");
|
||||
String taskPage = ss.read(baseURL + title.replaceAll(" ", "_"));
|
||||
int exSubTot;
|
||||
if (taskPage.contains(exmplBegin)) {
|
||||
int startPos = taskPage.lastIndexOf(exmplBegin)
|
||||
+ exmplBegin.length();
|
||||
String countStr = taskPage.substring(startPos,
|
||||
taskPage.indexOf(exmplEnd, startPos));
|
||||
exSubTot = Integer
|
||||
.parseInt(countStr.contains(".") ? countStr
|
||||
.substring(0, countStr.indexOf("."))
|
||||
: countStr);
|
||||
} else {
|
||||
exSubTot = 0;
|
||||
while (taskPage.contains(editBegin)) {
|
||||
taskPage = taskPage.substring(taskPage
|
||||
.indexOf(editBegin) + editBegin.length());
|
||||
exSubTot++;
|
||||
}
|
||||
}
|
||||
exTotal += exSubTot;
|
||||
System.out.println(title + ": " + exSubTot + " examples.");
|
||||
}
|
||||
// Print total
|
||||
System.out.println("\nTotal: " + exTotal + " examples.");
|
||||
} catch (Exception e) {
|
||||
System.out.println(title);
|
||||
System.out.println(startPos + ":"
|
||||
+ taskPage.indexOf(exmplEnd, startPos));
|
||||
System.out.println(taskPage);
|
||||
e.printStackTrace(System.out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Produce lines of the form: URI TITLE
|
||||
function titles {
|
||||
local uri="http://www.rosettacode.org/mw/api.php?action=query&list=categorymembers"
|
||||
uri+="&cmtitle=Category:Programming_Tasks&cmlimit=5000&format=json"
|
||||
curl -Ss "$uri" |
|
||||
jq -r '.query.categorymembers[] | .title | "\(@uri) \(.)"'
|
||||
}
|
||||
|
||||
# Syntax: count URI
|
||||
function count {
|
||||
local uri="$1"
|
||||
curl -Ss "http://rosettacode.org/mw/index.php?title=${uri}&action=raw" |
|
||||
jq -R -n 'reduce (inputs|select(test("=={{header\\|"))) as $x(0; .+1)'
|
||||
}
|
||||
|
||||
local n=0 i
|
||||
while read uri title
|
||||
do
|
||||
i=$(count "$uri")
|
||||
echo "$title: $i examples."
|
||||
n=$((n + i))
|
||||
done < <(titles)
|
||||
echo Total: $n examples.
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
using HTTP, JSON, Dates
|
||||
|
||||
rosorg = "http://rosettacode.org"
|
||||
qURI = "/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=json"
|
||||
qdURI = "/w/api.php?action=query&list=categorymembers&cmtitle=Category:Draft_Programming_Tasks&cmlimit=500&format=json"
|
||||
sqURI = rosorg * "/wiki/"
|
||||
topages(js, v) = for d in js["query"]["categorymembers"] push!(v, sqURI * replace(d["title"], " " => "_")) end
|
||||
|
||||
function getpages(uri)
|
||||
wikipages = Vector{String}()
|
||||
response = HTTP.request("GET", rosorg * uri)
|
||||
if response.status == 200
|
||||
fromjson = JSON.parse(String(response.body))
|
||||
topages(fromjson, wikipages)
|
||||
while haskey(fromjson, "continue")
|
||||
cmcont, cont = fromjson["continue"]["cmcontinue"], fromjson["continue"]["continue"]
|
||||
response = HTTP.request("GET", rosorg * uri * "&cmcontinue=$cmcont&continue=$cont")
|
||||
fromjson = JSON.parse(String(response.body))
|
||||
topages(fromjson, wikipages)
|
||||
end
|
||||
end
|
||||
wikipages
|
||||
end
|
||||
|
||||
function processtaskpages(wpages, verbose=false)
|
||||
totalexamples = 0
|
||||
for pag in wpages
|
||||
response = HTTP.request("GET", pag)
|
||||
if response.status == 200
|
||||
n = length(collect(eachmatch(r"span class=\"mw-headline\"", String(response.body))))
|
||||
if verbose
|
||||
println("Wiki page $pag => $n examples.")
|
||||
end
|
||||
totalexamples += n
|
||||
end
|
||||
end
|
||||
println("Total of $totalexamples on $(length(wpages)) task pages.\n")
|
||||
end
|
||||
|
||||
|
||||
println("Programming examples at $(DateTime(now())):")
|
||||
qURI |> getpages |> processtaskpages
|
||||
|
||||
println("Draft programming tasks:")
|
||||
qdURI |> getpages |> processtaskpages
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
local(root = json_deserialize(curl('http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=10&format=json')->result))
|
||||
local(tasks = array, title = string, urltitle = string, thiscount = 0, totalex = 0)
|
||||
with i in #root->find('query')->find('categorymembers') do => {^
|
||||
#thiscount = 0
|
||||
#title = #i->find('title')
|
||||
#urltitle = #i->find('title')
|
||||
#urltitle->replace(' ','_')
|
||||
|
||||
#title+': '
|
||||
local(src = curl('http://rosettacode.org/mw/index.php?title='+#urltitle->asBytes->encodeurl+'&action=raw')->result->asString)
|
||||
#thiscount = (#src->split('=={{header|'))->size - 1
|
||||
#thiscount < 0 ? #thiscount = 0
|
||||
#thiscount + ' examples.'
|
||||
#totalex += #thiscount
|
||||
'\r'
|
||||
^}
|
||||
'Total: '+#totalex+' examples.'
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
on mouseUp
|
||||
put empty into fld "taskurls"
|
||||
put URL "http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=10&format=xml" into apixml
|
||||
put revXMLCreateTree(apixml,true,true,false) into pDocID
|
||||
put "/api/query/categorymembers/cm" into pXPathExpression
|
||||
repeat for each line xmlnode in revXMLEvaluateXPath(pDocID, pXPathExpression)
|
||||
put revXMLAttribute(pDocID,xmlnode,"title") into pgTitle
|
||||
put revXMLAttribute(pDocID,xmlnode,"pageid") into pageId
|
||||
put "http://www.rosettacode.org/w/index.php?title=" & urlEncode(pgTitle) & "&action=raw" into taskURL
|
||||
put URL taskURL into taskPage
|
||||
filter lines of taskPage with "=={{header|*"
|
||||
put the number of lines of taskPage into taskTotal
|
||||
put pgTitle & comma & taskTotal & cr after fld "tasks"
|
||||
add taskTotal to allTaskTotal
|
||||
end repeat
|
||||
put "Total" & comma & allTaskTotal after fld "tasks"
|
||||
end mouseUp
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
function c = count_examples(url)
|
||||
c = 0;
|
||||
[s, success] = urlread (url);
|
||||
if ~success, return; end;
|
||||
c = length(strfind(s,'<h2><span class='));
|
||||
end;
|
||||
|
||||
% script
|
||||
s = urlread ('http://rosettacode.org/wiki/Category:Programming_Tasks');
|
||||
pat = '<li><a href="/wiki/';
|
||||
ix = strfind(s,pat)+length(pat)-6;
|
||||
for k = 1:length(ix);
|
||||
% look through all tasks
|
||||
e = find(s(ix(k):end)==34,1)-2;
|
||||
t = s(ix(k)+[0:e]); % task
|
||||
c = count_examples(['http://rosettacode.org',t]);
|
||||
printf('Task "%s" has %i examples.\n',t(7:end), c);
|
||||
end;
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
ConvertUTF8 := proc( str )
|
||||
local i, tempstring, uniindex;
|
||||
try
|
||||
tempstring := str;
|
||||
uniindex := [StringTools:-SearchAll("\u",str)];
|
||||
if uniindex <> [] then
|
||||
for i in uniindex do
|
||||
tempstring := StringTools:-Substitute(tempstring, str[i..i+5], UTF8:-unicode(str[i+2..i+5]));
|
||||
end do:
|
||||
end if;
|
||||
return tempstring;
|
||||
catch:
|
||||
return str;
|
||||
end try;
|
||||
end proc:
|
||||
print_examples := proc(lst)
|
||||
local task, count, url, headers, item;
|
||||
for task in lst do
|
||||
count := 0:
|
||||
url := cat("http://www.rosettacode.org/mw/index.php?title=", StringTools:-Encode(StringTools:-SubstituteAll(task["title"], " ", "_"), 'percent'), "&action=raw"):
|
||||
headers := [StringTools:-SearchAll("=={{header|",URL:-Get(url))]:
|
||||
for item in headers do
|
||||
count++:
|
||||
end do:
|
||||
printf("%s has %d examples\n",ConvertUTF8(task["title"]), count);
|
||||
end do:
|
||||
end proc:
|
||||
|
||||
|
||||
x := JSON:-ParseFile("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=20&format=json"):
|
||||
print_examples(x["query"]["categorymembers"]);
|
||||
while(assigned(x["continue"]["cmcontinue"])) do
|
||||
continue := x["continue"]["cmcontinue"]:
|
||||
more_tasks:= cat("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=20&format=json", "&continue=", x["continue"]["continue"], "&cmcontinue=", x["continue"]["cmcontinue"]):
|
||||
x := JSON:-ParseFile(more_tasks):
|
||||
print_examples(x["query"]["categorymembers"]);
|
||||
end do:
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
TaskList = Flatten[
|
||||
Import["http://rosettacode.org/wiki/Category:Programming_Tasks", "Data"][[1, 1]]];
|
||||
Print["Task \"", StringReplace[#, "_" -> " "], "\" has ",
|
||||
Length@Select[Import["http://rosettacode.org/wiki/" <> #, "Data"][[1,2]],
|
||||
StringFreeQ[#, __ ~~ "Programming Task" | __ ~~ "Omit"]& ], " example(s)"]&
|
||||
~Map~ StringReplace[TaskList, " " -> "_"]
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
import httpclient, strutils, xmltree, xmlparser, cgi
|
||||
|
||||
proc count(s, sub: string): int =
|
||||
var i = 0
|
||||
while true:
|
||||
i = s.find(sub, i)
|
||||
if i < 0: break
|
||||
inc i
|
||||
inc result
|
||||
|
||||
const
|
||||
mainSite = "http://www.rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
|
||||
subSite = "http://www.rosettacode.org/mw/index.php?title=$#&action=raw"
|
||||
|
||||
var client = newHttpClient()
|
||||
var sum = 0
|
||||
for node in client.getContent(mainSite).parseXml().findAll("cm"):
|
||||
let t = node.attr("title").replace(" ", "_")
|
||||
let c = client.getContent(subSite % encodeUrl(t)).toLower().count("{{header|")
|
||||
echo t.replace("_", " "), ": ", c, " examples."
|
||||
inc sum, c
|
||||
|
||||
echo "\nTotal: ", sum, " examples."
|
||||
|
|
@ -0,0 +1,62 @@
|
|||
open Http_client.Convenience
|
||||
|
||||
|
||||
let repl_quote s =
|
||||
let reg = Str.regexp_string "'" in
|
||||
(Str.global_replace reg "%27" s)
|
||||
|
||||
let repl_space s =
|
||||
let s = String.copy s in
|
||||
for i = 0 to pred(String.length s) do
|
||||
if s.[i] = ' ' then s.[i] <- '_'
|
||||
done;
|
||||
(s)
|
||||
(* or in OCaml 4.00+:
|
||||
let repl_space = String.map (fun c -> if c = ' ' then '_' else c)
|
||||
*)
|
||||
|
||||
let count_ex s =
|
||||
let pat = Str.regexp_string "=={{header|" in
|
||||
let rec aux n p =
|
||||
try
|
||||
let p = Str.search_forward pat s p in
|
||||
aux (n+1) (p+1)
|
||||
with Not_found -> (n)
|
||||
in
|
||||
aux 0 0
|
||||
|
||||
let get_child child xml =
|
||||
let child =
|
||||
List.find
|
||||
(function Xml.Element (tag,_,_) when tag = child -> true | _ -> false) xml
|
||||
in
|
||||
Xml.children child
|
||||
|
||||
let () =
|
||||
let url = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&\
|
||||
cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml" in
|
||||
|
||||
let xml = Xml.parse_string (http_get url) in
|
||||
|
||||
let total = ref 0 in
|
||||
at_exit (fun () -> Printf.printf "\n Total: %d\n" !total);
|
||||
|
||||
let f = function
|
||||
| Xml.Element ("cm", attrs, _) ->
|
||||
(try
|
||||
let _title = List.assoc "title" attrs in
|
||||
let title = repl_quote (repl_space _title) in
|
||||
let url = "http://www.rosettacode.org/w/index.php?title="^ title ^"&action=raw" in
|
||||
let n = count_ex (http_get url) in
|
||||
Printf.printf "%s: %d\n%!" _title n;
|
||||
total := n + !total;
|
||||
with Http_client.Http_error (404, _) -> ())
|
||||
| _ -> ()
|
||||
in
|
||||
|
||||
match xml with
|
||||
| Xml.Element ("api", _, ch) ->
|
||||
let query = get_child "query" ch in
|
||||
let catmb = get_child "categorymembers" query in
|
||||
List.iter f catmb
|
||||
| _ -> ()
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
use HTTP;
|
||||
use XML;
|
||||
|
||||
class RosettaCount {
|
||||
function : Main(args : String[]) ~ Nil {
|
||||
taks_xml := HttpGet("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml");
|
||||
parser := XmlParser->New(taks_xml);
|
||||
if(parser->Parse()) {
|
||||
task_names := parser->FindElements("/api/query/categorymembers/cm");
|
||||
if(task_names <> Nil) {
|
||||
each(i : task_names) {
|
||||
task_name := task_names->Get(i)->As(XmlElement)->GetAttribute("title")->GetValue();
|
||||
task_url := "http://rosettacode.org/mw/index.php?title=";
|
||||
task_url->Append(task_name);
|
||||
task_url->Append("&action=raw");
|
||||
|
||||
task := HttpGet(task_url);
|
||||
counts := task->FindAll("=={{header|");
|
||||
if(counts->Size() > 0) {
|
||||
IO.Console->Print(UrlUtility->Decode(task_name))->Print(": ")->PrintLine(counts->Size());
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function : HttpGet(url : String) ~ String {
|
||||
xml := "";
|
||||
|
||||
client := HttpClient->New();
|
||||
lines := client->Get(url);
|
||||
each(i : lines) {
|
||||
xml->Append(lines->Get(i)->As(String));
|
||||
};
|
||||
|
||||
return xml;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
declare
|
||||
[HTTPClient] = {Module.link ['x-ozlib://mesaros/net/HTTPClient.ozf']}
|
||||
[XMLParser] = {Module.link ['x-oz://system/xml/Parser.ozf']}
|
||||
[StringX] = {Module.link ['x-oz://system/String.ozf']}
|
||||
[Regex] = {Module.link ['x-oz://contrib/regex']}
|
||||
|
||||
AllTasksUrl = "http://rosettacode.org/mw/api.php?action=query&list="#
|
||||
"categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
|
||||
|
||||
proc {Main}
|
||||
AllTasks = {Parse {GetPage AllTasksUrl}}
|
||||
TaskTitles = {GetTitles AllTasks}
|
||||
Total = {NewCell 0}
|
||||
in
|
||||
for Task in TaskTitles do
|
||||
TaskPage = {GetPage {TaskUrl Task}}
|
||||
RE = {Regex.compile "{{header\\|" [extended newline icase]}
|
||||
NumMatches = {Length {Regex.allMatches RE TaskPage}}
|
||||
in
|
||||
{System.showInfo Task#": "#NumMatches#" examples."}
|
||||
Total := @Total + NumMatches
|
||||
end
|
||||
{System.showInfo "Total: "#@Total#" examples."}
|
||||
end
|
||||
|
||||
fun {TaskUrl Task}
|
||||
"http://rosettacode.org/mw/index.php?"#
|
||||
"title="#{PercentEncode {StringX.replace Task " " "_"}}#
|
||||
"&action=raw"
|
||||
end
|
||||
|
||||
%% GetPage
|
||||
local
|
||||
Client = {New HTTPClient.urlGET init(inPrms(toFile:false toStrm:true) _)}
|
||||
in
|
||||
fun {GetPage RawUrl}
|
||||
Url = {VirtualString.toString RawUrl}
|
||||
OutParams
|
||||
HttpResponseParams
|
||||
in
|
||||
{Client getService(Url ?OutParams ?HttpResponseParams)}
|
||||
OutParams.sOut
|
||||
end
|
||||
end
|
||||
|
||||
%% Parse
|
||||
local
|
||||
Parser = {New XMLParser.parser init}
|
||||
in
|
||||
fun {Parse Xs} {Parser parseVS(Xs $)} end
|
||||
end
|
||||
|
||||
fun {GetTitles Doc}
|
||||
CMs = Doc.2.1.children.1.children.1.children
|
||||
fun {Attributes element(attributes:As ...)} As end
|
||||
fun {IsTitle attribute(name:N ...)} N == title end
|
||||
in
|
||||
{Map {Filter {Flatten {Map CMs Attributes}} IsTitle}
|
||||
fun {$ A} {Atom.toString A.value} end}
|
||||
end
|
||||
|
||||
fun {PercentEncode Xs}
|
||||
case Xs of nil then nil
|
||||
[] X|Xr then
|
||||
if {Char.isDigit X} orelse {Member X [&- &_ &. &~]}
|
||||
orelse X >= &a andthen X =< &z
|
||||
orelse X >= &z andthen X =< &Z then
|
||||
X|{PercentEncode Xr}
|
||||
else
|
||||
{Append &%|{ToHex2 X} {PercentEncode Xr}}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
fun {ToHex2 X}
|
||||
[{ToHex1 X div 16} {ToHex1 X mod 16}]
|
||||
end
|
||||
|
||||
fun {ToHex1 X}
|
||||
if X >= 0 andthen X =< 9 then &0 + X
|
||||
elseif X >= 10 andthen X =< 15 then &A + X - 10
|
||||
end
|
||||
end
|
||||
in
|
||||
{Main}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
use HTTP::Tiny;
|
||||
|
||||
my $site = "http://rosettacode.org";
|
||||
my $list_url = "/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
|
||||
|
||||
my $response = HTTP::Tiny->new->get("$site$list_url");
|
||||
for ($response->{content} =~ /cm.*?title="(.*?)"/g) {
|
||||
(my $slug = $_) =~ tr/ /_/;
|
||||
my $response = HTTP::Tiny->new->get("$site/wiki/$slug");
|
||||
my $count = () = $response->{content} =~ /toclevel-1/g;
|
||||
print "$_: $count examples\n";
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
use Mojo::UserAgent;
|
||||
|
||||
my $site = "http://rosettacode.org";
|
||||
my $list_url = "/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
|
||||
|
||||
my $ua = Mojo::UserAgent->new;
|
||||
$ua->get("$site$list_url")->res->dom->find('cm')->each(sub {
|
||||
(my $slug = $_->{title}) =~ tr/ /_/;
|
||||
my $count = $ua->get("$site/wiki/$slug")->res->dom->find("#toc .toclevel-1")->size;
|
||||
say "$_->{title}: $count examples";
|
||||
});
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
(notonline)-->
|
||||
<span style="color: #000080;font-style:italic;">--
|
||||
-- demo\rosetta\rosettacode_cache.e
|
||||
-- ================================
|
||||
--
|
||||
-- Common routines for handling rc_cache etc.
|
||||
--</span>
|
||||
<span style="color: #008080;">without</span> <span style="color: #008080;">js</span> <span style="color: #000080;font-style:italic;">-- (libcurl, file i/o, peek, progress..)</span>
|
||||
<span style="color: #008080;">include</span> <span style="color: #000000;">builtins</span><span style="color: #0000FF;">\</span><span style="color: #004080;">timedate</span><span style="color: #0000FF;">.</span><span style="color: #000000;">e</span>
|
||||
<span style="color: #008080;">constant</span> <span style="color: #000000;">day</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">timedelta</span><span style="color: #0000FF;">(</span><span style="color: #000000;">days</span><span style="color: #0000FF;">:=</span><span style="color: #000000;">1</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">refresh_cache</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">21</span><span style="color: #0000FF;">*</span><span style="color: #000000;">day</span> <span style="color: #000080;font-style:italic;">-- 0 for always [NB refresh_cache += timedelta(days:=1) below]</span>
|
||||
|
||||
<span style="color: #008080;">function</span> <span style="color: #000000;">days</span><span style="color: #0000FF;">(</span><span style="color: #004080;">atom</span> <span style="color: #000000;">delta</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">d</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">ceil</span><span style="color: #0000FF;">(</span><span style="color: #000000;">delta</span><span style="color: #0000FF;">/</span><span style="color: #000000;">day</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">return</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%d day%s"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">d</span><span style="color: #0000FF;">,</span><span style="color: #008080;">iff</span><span style="color: #0000FF;">(</span><span style="color: #000000;">d</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span><span style="color: #0000FF;">?</span><span style="color: #008000;">""</span><span style="color: #0000FF;">:</span><span style="color: #008000;">"s"</span><span style="color: #0000FF;">)})</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
|
||||
|
||||
<span style="color: #008080;">constant</span> <span style="color: #0000FF;">{</span><span style="color: #000000;">hex</span><span style="color: #0000FF;">,</span><span style="color: #000000;">ascii</span><span style="color: #0000FF;">}</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">columnize</span><span style="color: #0000FF;">({{</span><span style="color: #008000;">"%22"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">`"`</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%27"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"'"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%2A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"*"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%2B"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"+"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%3A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">":"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%5E"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">`^`</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%E2%80%93"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"-"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%E2%80%99"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"'"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%C3%A8"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"e"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%C3%A9"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"e"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%C3%B6"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"o"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%C5%91"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"o"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"""</span><span style="color: #0000FF;">,</span><span style="color: #008000;">`"`</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"'"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"'"</span><span style="color: #0000FF;">},</span>
|
||||
<span style="color: #0000FF;">{</span><span style="color: #008000;">"_"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">" "</span><span style="color: #0000FF;">}})</span>
|
||||
|
||||
<span style="color: #008080;">global</span> <span style="color: #008080;">function</span> <span style="color: #000000;">html_clean</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">return</span> <span style="color: #7060A8;">substitute_all</span><span style="color: #0000FF;">(</span><span style="color: #000000;">s</span><span style="color: #0000FF;">,</span><span style="color: #000000;">hex</span><span style="color: #0000FF;">,</span><span style="color: #000000;">ascii</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
|
||||
|
||||
<span style="color: #008080;">include</span> <span style="color: #000000;">builtins</span><span style="color: #0000FF;">\</span><span style="color: #000000;">libcurl</span><span style="color: #0000FF;">.</span><span style="color: #000000;">e</span>
|
||||
<span style="color: #004080;">atom</span> <span style="color: #000000;">curl</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">NULL</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">pErrorBuffer</span>
|
||||
|
||||
<span style="color: #008080;">function</span> <span style="color: #000000;">write_callback</span><span style="color: #0000FF;">(</span><span style="color: #004080;">atom</span> <span style="color: #000000;">pData</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">integer</span> <span style="color: #000000;">size</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">nmemb</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">fn</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">bytes_written</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">size</span> <span style="color: #0000FF;">*</span> <span style="color: #000000;">nmemb</span>
|
||||
<span style="color: #7060A8;">puts</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fn</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">peek</span><span style="color: #0000FF;">({</span><span style="color: #000000;">pData</span><span style="color: #0000FF;">,</span><span style="color: #000000;">bytes_written</span><span style="color: #0000FF;">}))</span>
|
||||
<span style="color: #008080;">return</span> <span style="color: #000000;">bytes_written</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
|
||||
<span style="color: #008080;">constant</span> <span style="color: #000000;">write_cb</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">call_back</span><span style="color: #0000FF;">({</span><span style="color: #008000;">'+'</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">write_callback</span><span style="color: #0000FF;">})</span>
|
||||
|
||||
<span style="color: #008080;">global</span> <span style="color: #004080;">string</span> <span style="color: #000000;">wastitle</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">""</span> <span style="color: #000080;font-style:italic;">-- don't clobber "NEED EDITING"/Downloading messages</span>
|
||||
<span style="color: #008080;">global</span> <span style="color: #004080;">integer</span> <span style="color: #000000;">show_title</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">progress</span>
|
||||
|
||||
<span style="color: #008080;">global</span> <span style="color: #008080;">function</span> <span style="color: #000000;">open_download</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">filename</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">url</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">integer</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">n</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #004080;">object</span> <span style="color: #000000;">text</span>
|
||||
<span style="color: #004080;">bool</span> <span style="color: #000000;">refetch</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">false</span>
|
||||
<span style="color: #004080;">string</span> <span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"not found"</span>
|
||||
<span style="color: #000000;">filename</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">join_path</span><span style="color: #0000FF;">({</span><span style="color: #008000;">"rc_cache"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">})</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #7060A8;">file_exists</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000080;font-style:italic;">-- use existing file if <= refresh_cache days old</span>
|
||||
<span style="color: #004080;">sequence</span> <span style="color: #000000;">last_mod</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">get_file_date</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #004080;">atom</span> <span style="color: #000000;">delta</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">timedate_diff</span><span style="color: #0000FF;">(</span><span style="color: #000000;">last_mod</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">date</span><span style="color: #0000FF;">())</span>
|
||||
<span style="color: #000000;">refetch</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">true</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">delta</span><span style="color: #0000FF;">></span><span style="color: #000000;">refresh_cache</span>
|
||||
<span style="color: #008080;">and</span> <span style="color: #008080;">not</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">".hist"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">days</span><span style="color: #0000FF;">(</span><span style="color: #000000;">delta</span><span style="color: #0000FF;">)</span> <span style="color: #0000FF;">&</span> <span style="color: #008000;">" > "</span> <span style="color: #0000FF;">&</span> <span style="color: #000000;">days</span><span style="color: #0000FF;">(</span><span style="color: #000000;">refresh_cache</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">elsif</span> <span style="color: #7060A8;">get_file_size</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"filesize of 0"</span>
|
||||
<span style="color: #008080;">else</span>
|
||||
<span style="color: #000000;">text</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">trim</span><span style="color: #0000FF;">(</span><span style="color: #7060A8;">get_text</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">))</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #004080;">sequence</span><span style="color: #0000FF;">(</span><span style="color: #000000;">text</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"no text"</span>
|
||||
<span style="color: #008080;">elsif</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">text</span><span style="color: #0000FF;">)<</span><span style="color: #000000;">10</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"<10 bytes"</span>
|
||||
<span style="color: #008080;">else</span>
|
||||
<span style="color: #000000;">refetch</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">false</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">else</span>
|
||||
<span style="color: #000000;">refetch</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">true</span>
|
||||
<span style="color: #004080;">string</span> <span style="color: #000000;">directory</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">get_file_path</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #7060A8;">get_file_type</span><span style="color: #0000FF;">(</span><span style="color: #000000;">directory</span><span style="color: #0000FF;">)!=</span><span style="color: #004600;">FILETYPE_DIRECTORY</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #000000;">create_directory</span><span style="color: #0000FF;">(</span><span style="color: #000000;">directory</span><span style="color: #0000FF;">,</span><span style="color: #000000;">make_parent</span><span style="color: #0000FF;">:=</span><span style="color: #004600;">true</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #7060A8;">crash</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"cannot create %s directory"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">directory</span><span style="color: #0000FF;">})</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">refetch</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">wastitle</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"x"</span> <span style="color: #000080;font-style:italic;">-- don't clobber</span>
|
||||
<span style="color: #004080;">string</span> <span style="color: #000000;">nofn</span> <span style="color: #0000FF;">=</span> <span style="color: #008080;">iff</span><span style="color: #0000FF;">(</span><span style="color: #000000;">n</span><span style="color: #0000FF;">?</span><span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"(%d/%d, %.1f%%) "</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">i</span><span style="color: #0000FF;">,</span><span style="color: #000000;">n</span><span style="color: #0000FF;">,</span><span style="color: #000000;">i</span><span style="color: #0000FF;">/</span><span style="color: #000000;">n</span><span style="color: #0000FF;">*</span><span style="color: #000000;">100</span><span style="color: #0000FF;">}):</span><span style="color: #008000;">""</span><span style="color: #0000FF;">),</span>
|
||||
<span style="color: #000000;">title</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Downloading %s%s (%s)..."</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">nofn</span><span style="color: #0000FF;">,</span><span style="color: #000000;">html_clean</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">),</span><span style="color: #000000;">why</span><span style="color: #0000FF;">})</span>
|
||||
<span style="color: #000000;">show_title</span><span style="color: #0000FF;">(</span><span style="color: #000000;">title</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">curl</span><span style="color: #0000FF;">=</span><span style="color: #004600;">NULL</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #7060A8;">curl_global_init</span><span style="color: #0000FF;">()</span>
|
||||
<span style="color: #000000;">curl</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">curl_easy_init</span><span style="color: #0000FF;">()</span>
|
||||
<span style="color: #000000;">pErrorBuffer</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">allocate</span><span style="color: #0000FF;">(</span><span style="color: #000000;">CURL_ERROR_SIZE</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #7060A8;">curl_easy_setopt</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">CURLOPT_ERRORBUFFER</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">pErrorBuffer</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #7060A8;">curl_easy_setopt</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">,</span> <span style="color: #004600;">CURLOPT_WRITEFUNCTION</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">write_cb</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #000000;">url</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">substitute</span><span style="color: #0000FF;">(</span><span style="color: #000000;">url</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%3A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">":"</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #000000;">url</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">substitute</span><span style="color: #0000FF;">(</span><span style="color: #000000;">url</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%2A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"*"</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #7060A8;">curl_easy_setopt</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">,</span> <span style="color: #004600;">CURLOPT_URL</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">url</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">fn</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">open</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"wb"</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #7060A8;">assert</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fn</span><span style="color: #0000FF;">!=-</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"cannot open "</span><span style="color: #0000FF;">&</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #7060A8;">curl_easy_setopt</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">CURLOPT_WRITEDATA</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">fn</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">while</span> <span style="color: #004600;">true</span> <span style="color: #008080;">do</span>
|
||||
<span style="color: #004080;">CURLcode</span> <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">curl_easy_perform</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">res</span><span style="color: #0000FF;">=</span><span style="color: #004600;">CURLE_OK</span> <span style="color: #008080;">then</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #004080;">string</span> <span style="color: #000000;">error</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%d"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">res</span><span style="color: #0000FF;">=</span><span style="color: #000000;">CURLE_COULDNT_RESOLVE_HOST</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">error</span> <span style="color: #0000FF;">&=</span> <span style="color: #008000;">" [CURLE_COULDNT_RESOLVE_HOST]"</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Error %s downloading file, retry?(Y/N):"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">error</span><span style="color: #0000FF;">})</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #7060A8;">lower</span><span style="color: #0000FF;">(</span><span style="color: #7060A8;">wait_key</span><span style="color: #0000FF;">())!=</span><span style="color: #008000;">'y'</span> <span style="color: #008080;">then</span> <span style="color: #7060A8;">abort</span><span style="color: #0000FF;">(</span><span style="color: #000000;">0</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #7060A8;">printf</span><span style="color: #0000FF;">(</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"Y\n"</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">while</span>
|
||||
<span style="color: #7060A8;">close</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fn</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #000000;">refresh_cache</span> <span style="color: #0000FF;">+=</span> <span style="color: #7060A8;">timedelta</span><span style="color: #0000FF;">(</span><span style="color: #000000;">days</span><span style="color: #0000FF;">:=</span><span style="color: #000000;">1</span><span style="color: #0000FF;">)</span> <span style="color: #000080;font-style:italic;">-- did I mention it is slow?</span>
|
||||
<span style="color: #000000;">text</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">get_text</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">return</span> <span style="color: #000000;">text</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
|
||||
|
||||
<span style="color: #008080;">global</span> <span style="color: #008080;">function</span> <span style="color: #000000;">open_category</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">filename</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">integer</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">n</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">return</span> <span style="color: #000000;">open_download</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">&</span><span style="color: #008000;">".htm"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"http://rosettacode.org/wiki/Category:"</span><span style="color: #0000FF;">&</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">,</span><span style="color: #000000;">i</span><span style="color: #0000FF;">,</span><span style="color: #000000;">n</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
|
||||
|
||||
<span style="color: #008080;">global</span> <span style="color: #008080;">function</span> <span style="color: #000000;">dewiki</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">sequence</span> <span style="color: #000000;">exclude</span><span style="color: #0000FF;">={})</span>
|
||||
<span style="color: #000080;font-style:italic;">-- extract tasks from eg `<li><a href="/wiki/100_doors"`</span>
|
||||
<span style="color: #004080;">sequence</span> <span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">1</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">finish</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<div class="printfooter">`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">s</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #000000;">s</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">[</span><span style="color: #000000;">1</span><span style="color: #0000FF;">..</span><span style="color: #000000;">finish</span><span style="color: #0000FF;">-</span><span style="color: #000000;">1</span><span style="color: #0000FF;">]</span>
|
||||
<span style="color: #008080;">while</span> <span style="color: #004600;">true</span> <span style="color: #008080;">do</span>
|
||||
<span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<li><a href="/wiki/`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">s</span><span style="color: #0000FF;">,</span><span style="color: #000000;">start</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">start</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #000000;">start</span> <span style="color: #0000FF;">+=</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<li><a href="/wiki/`</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #000000;">finish</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">find</span><span style="color: #0000FF;">(</span><span style="color: #008000;">'"'</span><span style="color: #0000FF;">,</span><span style="color: #000000;">s</span><span style="color: #0000FF;">,</span><span style="color: #000000;">start</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #004080;">string</span> <span style="color: #000000;">task</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">[</span><span style="color: #000000;">start</span><span style="color: #0000FF;">..</span><span style="color: #000000;">finish</span><span style="color: #0000FF;">-</span><span style="color: #000000;">1</span><span style="color: #0000FF;">]</span>
|
||||
<span style="color: #000000;">task</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">substitute_all</span><span style="color: #0000FF;">(</span><span style="color: #000000;">task</span><span style="color: #0000FF;">,{</span><span style="color: #008000;">"*"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">":"</span><span style="color: #0000FF;">},{</span><span style="color: #008000;">"%2A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%3A"</span><span style="color: #0000FF;">})</span>
|
||||
<span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">append</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">,</span><span style="color: #000000;">task</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">finish</span><span style="color: #0000FF;">+</span><span style="color: #000000;">1</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">while</span>
|
||||
<span style="color: #008080;">return</span> <span style="color: #000000;">tasks</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
|
||||
|
||||
<span style="color: #008080;">global</span> <span style="color: #008080;">procedure</span> <span style="color: #000000;">curl_cleanup</span><span style="color: #0000FF;">()</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">curl</span><span style="color: #0000FF;">!=</span><span style="color: #004600;">NULL</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #7060A8;">curl_easy_cleanup</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #7060A8;">free</span><span style="color: #0000FF;">(</span><span style="color: #000000;">pErrorBuffer</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #000000;">curl</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">NULL</span>
|
||||
<span style="color: #000000;">pErrorBuffer</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">NULL</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">procedure</span>
|
||||
<!--
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
(notonline)-->
|
||||
<span style="color: #000080;font-style:italic;">--
|
||||
-- demo\rosetta\Count_examples.exw
|
||||
-- ===============================
|
||||
--
|
||||
-- (This uses a few '&' instead of/as well as 'a', fairly obviously for everyone's sanity..)
|
||||
-- Counts no of "<nowiki>{{</nowiki>he&der|" (nb not "==<nowiki>{{</nowiki>he&der|") via web api (but gets tasks via scraping).
|
||||
-- Since downloading all the pages can be very slow, this uses a cache.
|
||||
-- Limiting (notdone) by "Phix" fairly obviously speeds it up tenfold :-)
|
||||
--</span>
|
||||
<span style="color: #008080;">without</span> <span style="color: #008080;">js</span> <span style="color: #000080;font-style:italic;">-- (fairly obviously this will never ever run in a browser!)</span>
|
||||
<span style="color: #008080;">constant</span> <span style="color: #000000;">include_drafts</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">true</span><span style="color: #0000FF;">,</span>
|
||||
<span style="color: #000000;">sort_by_count</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">false</span><span style="color: #0000FF;">,</span>
|
||||
<span style="color: #000080;font-style:italic;">-- notlang = "Phix" -- or "" (ie a zero length string) for all</span>
|
||||
<span style="color: #000000;">notlang</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">""</span>
|
||||
|
||||
<span style="color: #008080;">include</span> <span style="color: #000000;">rosettacode_cache</span><span style="color: #0000FF;">.</span><span style="color: #000000;">e</span>
|
||||
|
||||
<span style="color: #008080;">function</span> <span style="color: #000000;">count_tasks</span><span style="color: #0000FF;">()</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #7060A8;">get_file_type</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"rc_cache"</span><span style="color: #0000FF;">)!=</span><span style="color: #004600;">FILETYPE_DIRECTORY</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #000000;">create_directory</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"rc_cache"</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #7060A8;">crash</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"cannot create rc_cache directory"</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #000080;font-style:italic;">-- note this lot use web scraping (as cribbed from a similar task) ...</span>
|
||||
<span style="color: #004080;">sequence</span> <span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">dewiki</span><span style="color: #0000FF;">(</span><span style="color: #000000;">open_category</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Programming_Tasks"</span><span style="color: #0000FF;">))</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">include_drafts</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">tasks</span> <span style="color: #0000FF;">&=</span> <span style="color: #000000;">dewiki</span><span style="color: #0000FF;">(</span><span style="color: #000000;">open_category</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Draft_Programming_Tasks"</span><span style="color: #0000FF;">))</span>
|
||||
<span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sort</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">notlang</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000080;font-style:italic;">-- filter already done in specified language</span>
|
||||
<span style="color: #004080;">string</span> <span style="color: #000000;">langurl</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"http://rosettacode.org/wiki/Category:"</span><span style="color: #0000FF;">&</span><span style="color: #000000;">notlang</span>
|
||||
<span style="color: #004080;">sequence</span> <span style="color: #000000;">done</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">dewiki</span><span style="color: #0000FF;">(</span><span style="color: #000000;">open_download</span><span style="color: #0000FF;">(</span><span style="color: #000000;">notlang</span><span style="color: #0000FF;">&</span><span style="color: #008000;">".htm"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">langurl</span><span style="color: #0000FF;">))</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">k</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span>
|
||||
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">do</span>
|
||||
<span style="color: #004080;">string</span> <span style="color: #000000;">ti</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">d</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">find</span><span style="color: #0000FF;">(</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">,</span><span style="color: #000000;">done</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #000000;">d</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">k</span> <span style="color: #0000FF;">+=</span> <span style="color: #000000;">1</span>
|
||||
<span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">k</span><span style="color: #0000FF;">]</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">ti</span>
|
||||
<span style="color: #008080;">else</span>
|
||||
<span style="color: #000000;">done</span><span style="color: #0000FF;">[</span><span style="color: #000000;">d</span><span style="color: #0000FF;">..</span><span style="color: #000000;">d</span><span style="color: #0000FF;">]</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
|
||||
<span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">1</span><span style="color: #0000FF;">..</span><span style="color: #000000;">k</span><span style="color: #0000FF;">]</span>
|
||||
<span style="color: #000000;">done</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%d tasks found\n"</span><span style="color: #0000FF;">,{</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)})</span>
|
||||
<span style="color: #000080;font-style:italic;">-- ... whereas the individual tasks use the web api instead (3x smaller/faster)</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">total_count</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span>
|
||||
<span style="color: #004080;">sequence</span> <span style="color: #000000;">task_counts</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">repeat</span><span style="color: #0000FF;">(</span><span style="color: #000000;">0</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">))</span>
|
||||
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">do</span>
|
||||
<span style="color: #004080;">string</span> <span style="color: #000000;">ti</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">],</span>
|
||||
<span style="color: #000000;">url</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"http://rosettacode.org/mw/index.php?title=%s&action=raw"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">}),</span>
|
||||
<span style="color: #000000;">contents</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">open_download</span><span style="color: #0000FF;">(</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">&</span><span style="color: #008000;">".raw"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">url</span><span style="color: #0000FF;">),</span>
|
||||
<span style="color: #000000;">prev</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">""</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">curr</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">count</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">1</span>
|
||||
<span style="color: #008080;">while</span> <span style="color: #004600;">true</span> <span style="color: #008080;">do</span>
|
||||
<span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<nowiki>{{</nowiki>hea`</span><span style="color: #0000FF;">&</span><span style="color: #008000;">`der|`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">contents</span><span style="color: #0000FF;">,</span><span style="color: #000000;">start</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">start</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #000080;font-style:italic;">--
|
||||
-- skip duplicates/we also have to cope with eg
|
||||
-- ==<nowiki>{{</nowiki>he&der|Python<nowiki>}}</nowiki>== \
|
||||
-- ===<nowiki>{{</nowiki>he&der|Python<nowiki>}}</nowiki> Original=== } count
|
||||
-- ===<nowiki>{{</nowiki>he&der|Python<nowiki>}}</nowiki> Succinct=== } once
|
||||
-- ===<nowiki>{{</nowiki>he&der|Python<nowiki>}}</nowiki> Recursive === /
|
||||
-- ==<nowiki>{{</nowiki>he&der|Mathematica<nowiki>}}</nowiki> / <nowiki>{{</nowiki>he&der|Wolfram Language<nowiki>}}</nowiki>== \
|
||||
-- ==<nowiki>{{</nowiki>he&der|Icon<nowiki>}}</nowiki> and <nowiki>{{</nowiki>he&der|Unicon<nowiki>}}</nowiki>== } count
|
||||
-- == <nowiki>{{</nowiki>he&der|Icon<nowiki>}}</nowiki> and <nowiki>{{</nowiki>he&der|Unicon<nowiki>}}</nowiki> == / both
|
||||
-- == <nowiki>{{</nowiki>he&der|Java<nowiki>}}</nowiki>==
|
||||
-- etc. Note however that this /does/ count eg
|
||||
-- ===<nowiki>{{</nowiki>he&der|Applesoft BASIC<nowiki>}}</nowiki>=== \
|
||||
-- ===<nowiki>{{</nowiki>he&der|BASIC256<nowiki>}}</nowiki>=== } count
|
||||
-- ===<nowiki>{{</nowiki>he&der|Commodore BASIC<nowiki>}}</nowiki>=== } 'em
|
||||
-- ===<nowiki>{{</nowiki>he&der|IS-BASIC<nowiki>}}</nowiki>=== } all
|
||||
-- ===<nowiki>{{</nowiki>he&der|Sinclair ZX81 BASIC<nowiki>}}</nowiki>=== /
|
||||
--</span>
|
||||
<span style="color: #000000;">curr</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">contents</span><span style="color: #0000FF;">[</span><span style="color: #000000;">start</span><span style="color: #0000FF;">..</span><span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<nowiki>}}</nowiki>`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">contents</span><span style="color: #0000FF;">,</span><span style="color: #000000;">start</span><span style="color: #0000FF;">+</span><span style="color: #000000;">1</span><span style="color: #0000FF;">)]</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">curr</span><span style="color: #0000FF;">!=</span><span style="color: #000000;">prev</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">count</span> <span style="color: #0000FF;">+=</span> <span style="color: #000000;">1</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #000000;">prev</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">curr</span>
|
||||
<span style="color: #000000;">start</span> <span style="color: #0000FF;">+=</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<nowiki>{{</nowiki>hea`</span><span style="color: #0000FF;">&</span><span style="color: #008000;">`der|`</span><span style="color: #0000FF;">)</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">while</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">sort_by_count</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #000000;">task_counts</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">count</span>
|
||||
<span style="color: #008080;">elsif</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">notlang</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">or</span> <span style="color: #000000;">i</span><span style="color: #0000FF;"><=</span><span style="color: #000000;">2</span> <span style="color: #008080;">or</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">>=</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)-</span><span style="color: #000000;">1</span> <span style="color: #008080;">or</span> <span style="color: #7060A8;">mod</span><span style="color: #0000FF;">(</span><span style="color: #000000;">i</span><span style="color: #0000FF;">,</span><span style="color: #000000;">200</span><span style="color: #0000FF;">)=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%s: %d\n"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">html_clean</span><span style="color: #0000FF;">(</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">),</span><span style="color: #000000;">count</span><span style="color: #0000FF;">})</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #000000;">total_count</span> <span style="color: #0000FF;">+=</span> <span style="color: #000000;">count</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #7060A8;">get_key</span><span style="color: #0000FF;">()=</span><span style="color: #000000;">#1B</span> <span style="color: #008080;">then</span> <span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"escape keyed\n"</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
|
||||
<span style="color: #000000;">curl_cleanup</span><span style="color: #0000FF;">()</span>
|
||||
<span style="color: #008080;">if</span> <span style="color: #000000;">sort_by_count</span> <span style="color: #008080;">then</span>
|
||||
<span style="color: #004080;">sequence</span> <span style="color: #000000;">tags</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">custom_sort</span><span style="color: #0000FF;">(</span><span style="color: #000000;">task_counts</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">tagset</span><span style="color: #0000FF;">(</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)))</span>
|
||||
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tags</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">to</span> <span style="color: #000000;">1</span> <span style="color: #008080;">by</span> <span style="color: #0000FF;">-</span><span style="color: #000000;">1</span> <span style="color: #008080;">do</span>
|
||||
<span style="color: #004080;">integer</span> <span style="color: #000000;">ti</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">tags</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]</span>
|
||||
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%s: %d\n"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">html_clean</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">]),</span><span style="color: #000000;">task_counts</span><span style="color: #0000FF;">[</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">]})</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
|
||||
<span style="color: #008080;">return</span> <span style="color: #000000;">total_count</span>
|
||||
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
|
||||
|
||||
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Total: %d\n"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">count_tasks</span><span style="color: #0000FF;">()})</span>
|
||||
<!--
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
(load "@lib/http.l")
|
||||
|
||||
(client "rosettacode.org" 80
|
||||
"mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
|
||||
(while (from " title=\"")
|
||||
(let Task (till "\"")
|
||||
(client "rosettacode.org" 80 (pack "wiki/" (replace Task " " "_"))
|
||||
(let Cnt 0
|
||||
(while (from "<span class=\"tocnumber\">")
|
||||
(unless (sub? "." (till "<" T))
|
||||
(inc 'Cnt) ) )
|
||||
(out NIL (prinl (ht:Pack Task) ": " Cnt)) ) ) ) ) )
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
Procedure handleError(value, msg.s)
|
||||
If value = 0
|
||||
MessageRequester("Error", msg)
|
||||
End
|
||||
EndIf
|
||||
EndProcedure
|
||||
|
||||
handleError(InitNetwork(), "Unable to initialize network functions.")
|
||||
If OpenConsole()
|
||||
Define url$, x1$, y1$, title$, unescapedTitle$, encodedURL$
|
||||
Define x2, i, j, totalExamples, totalTasks
|
||||
url$ = "http://www.rosettacode.org/mw/api.php?action=query" +
|
||||
"&list=categorymembers&cmtitle=Category:Programming_Tasks" +
|
||||
"&cmlimit=500&format=xml"
|
||||
|
||||
Repeat
|
||||
handleError(ReceiveHTTPFile(url$, "tasks.xml"), "Unable to access tasks URL.")
|
||||
|
||||
handleError(ReadFile(0, "tasks.xml"), "Unable to read 'task.xml' file.")
|
||||
x1$ = ReadString(0)
|
||||
CloseFile(0)
|
||||
|
||||
Repeat
|
||||
x2 = FindString(x1$, "title=", x2 + 1)
|
||||
If x2
|
||||
title$ = Mid(x1$, x2 + 7, 99)
|
||||
title$ = Left(title$, FindString(title$, ">", 1) - 4)
|
||||
unescapedTitle$ = UnescapeString(ReplaceString(title$, "'", "'"), #PB_String_EscapeXML)
|
||||
encodedURL$ = URLEncoder("http://www.rosettacode.org/mw/index.php?title=" + unescapedTitle$ + "&action=raw")
|
||||
If ReceiveHTTPFile(encodedURL$, "task.xml")
|
||||
ReadFile(0, "task.xml")
|
||||
While Not Eof(0)
|
||||
y1$ = ReadString(0)
|
||||
If FindString(y1$, "=={{header|", 1, #PB_String_NoCase)
|
||||
totalExamples + 1
|
||||
EndIf
|
||||
Wend
|
||||
CloseFile(0)
|
||||
|
||||
PrintN(unescapedTitle$ +": " + Str(totalExamples) + " examples")
|
||||
|
||||
totalTasks + totalExamples
|
||||
totalExamples = 0
|
||||
EndIf
|
||||
EndIf
|
||||
Until x2 = 0
|
||||
|
||||
;check for additional pages of tasks
|
||||
x2 = FindString(x1$, "cmcontinue=")
|
||||
If x2
|
||||
i = FindString(x1$, #DQUOTE$, x2 + 1)
|
||||
j = FindString(x1$, #DQUOTE$, i + 1)
|
||||
url$ = URLEncoder("http://www.rosettacode.org/mw/api.php?action=query" +
|
||||
"&list=categorymembers&cmtitle=Category:Programming_Tasks" +
|
||||
"&cmlimit=500&format=xml&cmcontinue=" + Mid(x1$, i + 1, j - i))
|
||||
Else
|
||||
Break ;all done
|
||||
EndIf
|
||||
ForEver
|
||||
|
||||
PrintN("Total: " + Str(totalTasks) + " examples")
|
||||
Input()
|
||||
CloseConsole()
|
||||
EndIf
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
from urllib.request import urlopen, Request
|
||||
import xml.dom.minidom
|
||||
|
||||
r = Request(
|
||||
'https://www.rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml',
|
||||
headers={'User-Agent': 'Mozilla/5.0'})
|
||||
x = urlopen(r)
|
||||
|
||||
tasks = []
|
||||
for i in xml.dom.minidom.parseString(x.read()).getElementsByTagName('cm'):
|
||||
t = i.getAttribute('title').replace(' ', '_')
|
||||
r = Request(f'https://www.rosettacode.org/mw/index.php?title={t}&action=raw',
|
||||
headers={'User-Agent': 'Mozilla/5.0'})
|
||||
y = urlopen(r)
|
||||
tasks.append( y.read().lower().count(b'{{header|') )
|
||||
print(t.replace('_', ' ') + f': {tasks[-1]} examples.')
|
||||
|
||||
print(f'\nTotal: {sum(tasks)} examples.')
|
||||
|
|
@ -0,0 +1,394 @@
|
|||
"""Count Rosetta Code tasks implementations using the Semantic MediaWiki API.
|
||||
Works with Python >= 3.7."""
|
||||
import json
|
||||
import logging
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import field
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from typing import Any
|
||||
from typing import Dict
|
||||
from typing import List
|
||||
from typing import Optional
|
||||
from typing import Set
|
||||
from typing import Tuple
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from requests.adapters import Retry
|
||||
|
||||
logging.basicConfig(level=logging.WARN)
|
||||
|
||||
# See https://www.semantic-mediawiki.org/wiki/Help:API:ask
|
||||
_SM_ASK: Dict[str, str] = {
|
||||
"action": "ask",
|
||||
"format": "json",
|
||||
"formatversion": "2",
|
||||
"api_version": "3",
|
||||
}
|
||||
|
||||
_SM_ASK_REQUEST_BLOCK_SIZE = 500
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Page:
|
||||
# fulltext is the page's title, not the page content.
|
||||
fulltext: str
|
||||
fullurl: str
|
||||
namespace: int
|
||||
exists: str
|
||||
displaytitle: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, eq=False)
|
||||
class Lang(Page):
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if isinstance(other, Lang):
|
||||
return self.fullurl == other.fullurl
|
||||
elif isinstance(other, str):
|
||||
return self.fullurl == other
|
||||
return False
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash(self.fullurl)
|
||||
|
||||
def unimplemented(self, tasks: Set["Task"]) -> Set["Task"]:
|
||||
return {task for task in tasks if self.fullurl not in task.exclude}
|
||||
|
||||
def omitted_from(self, tasks: Set["Task"]) -> Set["Task"]:
|
||||
return {task for task in tasks if self.fullurl in task.omitted_from}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Task(Page):
|
||||
title: str
|
||||
implemented_in: Set[Lang] = field(repr=False, compare=False)
|
||||
omitted_from: Set[Lang] = field(repr=False, compare=False)
|
||||
exclude: Set[Lang] = field(repr=False, compare=False)
|
||||
|
||||
def not_implemented_in(self, langs: Set[Lang]) -> Set[Lang]:
|
||||
return langs.difference(self.exclude)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskResponseBlock:
|
||||
tasks: List[Task]
|
||||
continue_offset: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LanguageResponseBlock:
|
||||
langs: List[Lang]
|
||||
continue_offset: Optional[int] = None
|
||||
|
||||
|
||||
def sm_ask_category(
|
||||
session: requests.Session,
|
||||
url: str,
|
||||
category: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
known_langs: Set[Lang],
|
||||
) -> TaskResponseBlock:
|
||||
query_params = {
|
||||
**_SM_ASK,
|
||||
"query": (
|
||||
f"[[Category:{category}]]"
|
||||
"|?Implemented in language"
|
||||
"|?Omitted from language"
|
||||
f"|limit={limit}"
|
||||
f"|offset={offset}"
|
||||
),
|
||||
}
|
||||
|
||||
# Display some progress
|
||||
log(f"ask [[Category:{category}]] offset={offset}")
|
||||
|
||||
response = session.get(url, params=query_params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
handle_warnings_and_errors(data)
|
||||
return _transform_implemented_in_response_data(data, known_langs)
|
||||
|
||||
|
||||
def sm_ask_tasks(
|
||||
session: requests.Session,
|
||||
url: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
known_langs: Set[Lang],
|
||||
):
|
||||
return sm_ask_category(
|
||||
session, url, "Programming Tasks", limit, offset, known_langs
|
||||
)
|
||||
|
||||
|
||||
def sm_ask_drafts(
|
||||
session: requests.Session,
|
||||
url: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
known_langs: Set[Lang],
|
||||
):
|
||||
return sm_ask_category(
|
||||
session, url, "Draft Programming Tasks", limit, offset, known_langs
|
||||
)
|
||||
|
||||
|
||||
def sm_ask_languages(
|
||||
session: requests.Session,
|
||||
url: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
) -> LanguageResponseBlock:
|
||||
query_params = {
|
||||
**_SM_ASK,
|
||||
"query": (
|
||||
"[[Is language::+]]"
|
||||
"|?Implemented in language"
|
||||
"|?Omitted from language"
|
||||
f"|limit={limit}"
|
||||
f"|offset={offset}"
|
||||
),
|
||||
}
|
||||
|
||||
# Display some progress
|
||||
log(f"ask [[Is language::+]] offset={offset}")
|
||||
|
||||
response = session.get(url, params=query_params)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
handle_warnings_and_errors(data)
|
||||
return _transform_language_response_data(data)
|
||||
|
||||
|
||||
def sm_ask_all_tasks(
|
||||
session: requests.Session, url: str, known_langs: Set[Lang]
|
||||
) -> List[Task]:
|
||||
block = sm_ask_tasks(
|
||||
session,
|
||||
url,
|
||||
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
|
||||
offset=0,
|
||||
known_langs=known_langs,
|
||||
)
|
||||
tasks = block.tasks
|
||||
|
||||
while block.continue_offset:
|
||||
block = sm_ask_tasks(
|
||||
session,
|
||||
url,
|
||||
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
|
||||
offset=block.continue_offset,
|
||||
known_langs=known_langs,
|
||||
)
|
||||
tasks.extend(block.tasks)
|
||||
|
||||
return tasks
|
||||
|
||||
|
||||
def sm_ask_all_drafts(
|
||||
session: requests.Session, url: str, known_langs: Set[Lang]
|
||||
) -> List[Task]:
|
||||
block = sm_ask_drafts(
|
||||
session,
|
||||
url,
|
||||
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
|
||||
offset=0,
|
||||
known_langs=known_langs,
|
||||
)
|
||||
tasks = block.tasks
|
||||
|
||||
while block.continue_offset:
|
||||
block = sm_ask_drafts(
|
||||
session,
|
||||
url,
|
||||
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
|
||||
offset=block.continue_offset,
|
||||
known_langs=known_langs,
|
||||
)
|
||||
tasks.extend(block.tasks)
|
||||
|
||||
return tasks
|
||||
|
||||
|
||||
def sm_ask_all_languages(session: requests.Session, url: str) -> List[Lang]:
|
||||
block = sm_ask_languages(
|
||||
session,
|
||||
url,
|
||||
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
|
||||
offset=0,
|
||||
)
|
||||
langs = block.langs
|
||||
|
||||
while block.continue_offset:
|
||||
block = sm_ask_languages(
|
||||
session,
|
||||
url,
|
||||
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
|
||||
offset=block.continue_offset,
|
||||
)
|
||||
langs.extend(block.langs)
|
||||
|
||||
return langs
|
||||
|
||||
|
||||
def _transform_implemented_in_response_data(
|
||||
data: Any, known_langs: Set[Lang]
|
||||
) -> TaskResponseBlock:
|
||||
tasks: List[Task] = []
|
||||
for result in data["query"]["results"]:
|
||||
for task_title, task_page in result.items():
|
||||
# We're excluding implementations that don't have a corresponding
|
||||
# category page with an "Is Language" property.
|
||||
implemented_in = {
|
||||
Lang(**lang)
|
||||
for lang in task_page["printouts"]["Implemented in language"]
|
||||
}.intersection(known_langs)
|
||||
|
||||
omitted_from = (
|
||||
{
|
||||
Lang(**lang)
|
||||
for lang in task_page["printouts"]["Omitted from language"]
|
||||
}
|
||||
.intersection(known_langs)
|
||||
.difference(implemented_in)
|
||||
)
|
||||
|
||||
tasks.append(
|
||||
Task(
|
||||
title=task_title,
|
||||
implemented_in=implemented_in,
|
||||
omitted_from=omitted_from,
|
||||
fulltext=task_page["fulltext"],
|
||||
fullurl=task_page["fullurl"],
|
||||
namespace=task_page["namespace"],
|
||||
exists=task_page["exists"],
|
||||
displaytitle=task_page["displaytitle"],
|
||||
exclude=implemented_in.union(omitted_from),
|
||||
)
|
||||
)
|
||||
|
||||
return TaskResponseBlock(
|
||||
tasks=tasks, continue_offset=data.get("query-continue-offset", None)
|
||||
)
|
||||
|
||||
|
||||
def _transform_language_response_data(data: Any) -> LanguageResponseBlock:
|
||||
langs: List[Lang] = []
|
||||
for result in data["query"]["results"]:
|
||||
for _, task_page in result.items():
|
||||
langs.append(
|
||||
Lang(
|
||||
fulltext=task_page["fulltext"],
|
||||
fullurl=task_page["fullurl"],
|
||||
namespace=task_page["namespace"],
|
||||
exists=task_page["exists"],
|
||||
displaytitle=task_page["displaytitle"],
|
||||
)
|
||||
)
|
||||
|
||||
return LanguageResponseBlock(
|
||||
langs=langs, continue_offset=data.get("query-continue-offset", None)
|
||||
)
|
||||
|
||||
|
||||
def get_session() -> requests.Session:
|
||||
"""Setup a requests.Session with retries."""
|
||||
retry_strategy = Retry(
|
||||
total=5,
|
||||
status_forcelist=[429, 500, 502, 503, 504],
|
||||
allowed_methods=["HEAD", "GET", "OPTIONS"],
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry_strategy)
|
||||
session = requests.Session()
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
print(f"{datetime.now().isoformat(' ', 'seconds')}: {msg}")
|
||||
|
||||
|
||||
def handle_warnings_and_errors(data: Any) -> None:
|
||||
if data.get("errors"):
|
||||
for error in data["errors"]:
|
||||
logging.error(json.dumps(error))
|
||||
# legacy format
|
||||
if data.get("error"):
|
||||
logging.error(json.dumps(data["error"]))
|
||||
if data.get("warnings"):
|
||||
for warning in data["warnings"]:
|
||||
logging.warning(json.dumps(warning))
|
||||
|
||||
|
||||
def count_examples(url: str, n: int = 30) -> None:
|
||||
"""Print a table to stdout containing implementation counts for the first
|
||||
`n` tasks, sorted by number implementations (most to least)."""
|
||||
session = get_session()
|
||||
langs = set(sm_ask_all_languages(session, url))
|
||||
tasks = sm_ask_all_tasks(session, url, langs)
|
||||
drafts = sm_ask_all_drafts(session, url, langs)
|
||||
all_tasks = [*tasks, *drafts]
|
||||
|
||||
# Map of task to (implemented in, not implemented in, omitted from)
|
||||
counts: Dict[Task, Tuple[int, int, int]] = {}
|
||||
|
||||
# Running total of examples for all tasks. Where a language has multiple examples
|
||||
# for a single tasks, we only count one example.
|
||||
total: int = 0
|
||||
|
||||
for task in all_tasks:
|
||||
total += len(task.implemented_in)
|
||||
counts[task] = (
|
||||
len(task.implemented_in),
|
||||
len(task.not_implemented_in(langs)),
|
||||
len(task.omitted_from),
|
||||
)
|
||||
|
||||
# Pretty print
|
||||
top = sorted(counts.items(), key=lambda it: it[1][0], reverse=True)[:n]
|
||||
pad = max([len(task.fulltext) for task, _ in top])
|
||||
|
||||
print("\nKnown languages:", len(langs))
|
||||
print("Total tasks:", len(all_tasks))
|
||||
print("Total examples:", total)
|
||||
print(f"{'Task':>{pad}} | Implemented In | Not Implemented In | Omitted From")
|
||||
print("-" * (pad + 1), "+", "-" * 16, "+", "-" * 20, "+", "-" * 13, sep="")
|
||||
|
||||
for task, _counts in top:
|
||||
implemented_in, not_implemented_in, omitted_from = _counts
|
||||
print(
|
||||
f"{task.fulltext:>{pad}} |"
|
||||
f"{implemented_in:>15} |"
|
||||
f"{not_implemented_in:>19} |"
|
||||
f"{omitted_from:>13}"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
URL = "https://rosettacode.org/w/api.php"
|
||||
parser = argparse.ArgumentParser(description="Count tasks on Rosetta Code.")
|
||||
|
||||
parser.add_argument(
|
||||
"--rows",
|
||||
"-n",
|
||||
type=int,
|
||||
default=30,
|
||||
dest="n",
|
||||
help="number of rows to display in the output table (default: 30)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--url",
|
||||
default=URL,
|
||||
help=f"target MediaWiki URL (default: {URL})",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
count_examples(args.url, args.n)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
library(XML)
|
||||
library(RCurl)
|
||||
doc <- xmlInternalTreeParse("http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml")
|
||||
nodes <- getNodeSet(doc,"//cm")
|
||||
titles = as.character( sapply(nodes, xmlGetAttr, "title") )
|
||||
headers <- list()
|
||||
counts <- list()
|
||||
for (i in 1:length(titles)){
|
||||
headers[[i]] <- getURL( paste("http://rosettacode.org/mw/index.php?title=", gsub(" ", "_", titles[i]), "&action=raw", sep="") )
|
||||
counts[[i]] <- strsplit(headers[[i]],split=" ")[[1]]
|
||||
counts[[i]] <- grep("\\{\\{header", counts[[i]])
|
||||
cat(titles[i], ":", length(counts[[i]]), "examples\n")
|
||||
}
|
||||
cat("Total: ", length(unlist(counts)), "examples\n")
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#lang racket
|
||||
|
||||
(require net/url net/uri-codec json (only-in racket/dict [dict-ref ref]))
|
||||
|
||||
(define (RC-get verb params)
|
||||
((compose1 get-pure-port string->url format)
|
||||
"http://rosettacode.org/mw/~a.php?~a" verb (alist->form-urlencoded params)))
|
||||
|
||||
(define (get-category catname)
|
||||
(let loop ([c #f])
|
||||
(define t
|
||||
((compose1 read-json RC-get) 'api
|
||||
`([action . "query"] [format . "json"]
|
||||
[list . "categorymembers"] [cmtitle . ,(format "Category:~a" catname)]
|
||||
[cmcontinue . ,(and c (ref c 'cmcontinue))] [cmlimit . "500"])))
|
||||
(define (c-m key) (ref (ref t key '()) 'categorymembers #f))
|
||||
(append (for/list ([page (c-m 'query)]) (ref page 'title))
|
||||
(cond [(c-m 'query-continue) => loop] [else '()]))))
|
||||
|
||||
(printf "Total: ~a\n"
|
||||
(for/sum ([task (get-category 'Programming_Tasks)])
|
||||
(define s ((compose1 length regexp-match-positions*)
|
||||
#rx"=={{" (RC-get 'index `([action . "raw"] [title . ,task]))))
|
||||
(printf "~a: ~a\n" task s)
|
||||
s))
|
||||
|
|
@ -0,0 +1,175 @@
|
|||
use HTTP::UserAgent;
|
||||
use URI::Escape;
|
||||
use JSON::Fast;
|
||||
use Lingua::EN::Numbers :short;
|
||||
|
||||
unit sub MAIN ( Bool :nf(:$no-fetch) = False, :t(:$tier) = 1 );
|
||||
|
||||
# Friendlier descriptions for task categories
|
||||
my %cat = (
|
||||
'Programming_Tasks' => 'Task',
|
||||
'Draft_Programming_Tasks' => 'Draft'
|
||||
);
|
||||
|
||||
my $client = HTTP::UserAgent.new;
|
||||
$client.timeout = 10;
|
||||
|
||||
my $url = 'https://rosettacode.org/w';
|
||||
|
||||
my $hashfile = './RC_Task_count.json';
|
||||
my $tablefile = "./RC_Task_count-{$tier}.txt";
|
||||
|
||||
my %tasks;
|
||||
|
||||
my @places = <① ② ③ ④ ⑤ ⑥ ⑦ ⑧ ⑨ ⑩ ⑪ ⑫ ⑬ ⑭ ⑮ ⑯ ⑰ ⑱ ⑲ ⑳ ㉑ ㉒ ㉓ ㉔ ㉕
|
||||
㉖ ㉗ ㉘ ㉙ ㉚ ㉛ ㉜ ㉝ ㉞ ㉟ ㊱ ㊲ ㊳ ㊴ ㊵ ㊶ ㊷ ㊸ ㊹ ㊺ ㊺ ㊻ ㊼ ㊽ ㊾ ㊿>;
|
||||
|
||||
# clear screen
|
||||
run($*DISTRO.is-win ?? 'cls' !! 'clear') unless $no-fetch;
|
||||
|
||||
my %counts =
|
||||
mediawiki-query(
|
||||
$url, 'pages',
|
||||
:generator<categorymembers>,
|
||||
:gcmtitle<Category:Programming Languages>,
|
||||
:gcmlimit<350>,
|
||||
:rawcontinue(),
|
||||
:prop<categoryinfo>
|
||||
)
|
||||
.map({ .<title>.subst(/^'Category:'/, '') => .<categoryinfo><pages> || 0 });
|
||||
|
||||
my $per-tier = 10;
|
||||
|
||||
my $which = (^$per-tier) »+» $per-tier * ($tier - 1);
|
||||
|
||||
my @top-n = %counts.sort( {-.value, .key} )[|$which].map: *.key.trans(' ' => '_');
|
||||
|
||||
# dump a copy to STDOUT, mostly for debugging purposes
|
||||
say "<pre>{tc $tier.&ord-n} {$per-tier.&card} programming languages by number of task examples completed:";
|
||||
say ' ', join ' ', .map( {("{(@places[|$which])[$_]} {@top-n[$_]}").fmt("%-15s")} ) for (^@top-n).batch(5);
|
||||
say "</pre>\n";
|
||||
|
||||
unless $no-fetch {
|
||||
|
||||
note 'Retrieving task information...';
|
||||
|
||||
mkdir('./pages') unless './pages'.IO.e;
|
||||
|
||||
@top-n = %counts.sort( {-.value, .key} )[^@places].map: *.key.trans(' ' => '_');;
|
||||
|
||||
for %cat.keys.sort -> $cat {
|
||||
mediawiki-query(
|
||||
$url, 'pages',
|
||||
:generator<categorymembers>,
|
||||
:gcmtitle("Category:$cat"),
|
||||
:gcmlimit<350>,
|
||||
:rawcontinue(),
|
||||
:prop<title>
|
||||
).map({
|
||||
my $page;
|
||||
my $response;
|
||||
|
||||
loop {
|
||||
$response = $client.get("{ $url }/index.php?title={ uri-escape .<title> }&action=raw");
|
||||
if $response.is-success {
|
||||
$page = $response.content;
|
||||
last;
|
||||
} else {
|
||||
redo;
|
||||
}
|
||||
}
|
||||
|
||||
"./pages/{ uri-escape .<title>.subst(/' '/, '_', :g) }".IO.spurt($page);
|
||||
my $lc = $page.lc.trans(' ' => '_');
|
||||
my $count = +$lc.comb(/ ^^'==' <-[\n=]>* '{{header|' <-[}]>+? '}}==' \h* $$ /);
|
||||
%tasks{.<title>} = {'cat' => %cat{$cat}, :$count};
|
||||
%tasks{.<title>}<top-n> = (^@top-n).map( {
|
||||
($lc.contains("==\{\{header|{@top-n[$_].lc}}}") or
|
||||
# Deal with 3 part headers - {{header|F_Sharp|F#}}, {{header|C_Sharp|C#}}, etc.
|
||||
$lc.contains("==\{\{header|{@top-n[$_].lc}|") or
|
||||
# Icon and Unicon are their own special flowers
|
||||
$lc.contains("}}_and_\{\{header|{@top-n[$_].lc}}}==") or
|
||||
# Language1 / Language2 for shared entries (e.g. C / C++)
|
||||
$lc.contains(rx/'}}''_'*'/''_'*'{{header|'$(@top-n[$_].lc)'}}=='/)) ??
|
||||
(@places[$_]) !!
|
||||
# Check if the task was omitted
|
||||
$lc.contains("\{\{omit_from|{@top-n[$_].lc}") ?? 'O' !!
|
||||
# The task is neither done or omitted
|
||||
' '
|
||||
} ).join;
|
||||
print clear, 1 + $++, ' ', %cat{$cat}, ' ', .<title>;
|
||||
})
|
||||
}
|
||||
|
||||
print clear;
|
||||
|
||||
note "\nTask information saved to local file: {$hashfile.IO.absolute}";
|
||||
$hashfile.IO.spurt(%tasks.&to-json);
|
||||
|
||||
}
|
||||
|
||||
# Load information from local file
|
||||
%tasks = $hashfile.IO.e ?? $hashfile.IO.slurp.&from-json !! ( );
|
||||
|
||||
@top-n = %counts.sort( {-.value, .key} )[|$which].map: *.key.trans(' ' => '_');
|
||||
|
||||
# Convert saved task info to a table
|
||||
note "\nBuilding table...";
|
||||
my $count = +%tasks;
|
||||
my $taskcnt = +%tasks.grep: *.value.<cat> eq %cat<Programming_Tasks>;
|
||||
my $draftcnt = $count - $taskcnt;
|
||||
my $total = sum %tasks{*}»<count>;
|
||||
|
||||
# Dump table to a file
|
||||
my $out = open($tablefile, :w) or die "$!\n";
|
||||
|
||||
$out.say: "<pre>{tc $tier.&ord-n} {$per-tier.&card} programming languages by number of task examples completed:";
|
||||
$out.say: ' ', join ' ', .map( {("{(@places[|$which])[$_]} {@top-n[$_]}").fmt("%-15s")} ) for (^@top-n).batch(5);
|
||||
$out.say: "</pre>\n\n<div style=\"height:40em;overflow:scroll;\">";
|
||||
|
||||
# Add table boilerplate and caption
|
||||
$out.say:
|
||||
'{|class="wikitable sortable"', "\n",
|
||||
"|+ As of { DateTime.new(time) } :: Tasks: { $taskcnt } ::<span style=\"background-color:#ffd\"> Draft Tasks:",
|
||||
"{ $draftcnt } </span>:: Total Tasks: { $count } :: Total Examples: { $total }\n",
|
||||
"!Count!!Task!!{(@places[|$which]).join('!!')}"
|
||||
;
|
||||
|
||||
# Sort tasks by count then add row
|
||||
for %tasks.sort: { [-.value<count>, .key] } -> $task {
|
||||
$out.say:
|
||||
( $task.value<cat> eq 'Draft'
|
||||
?? "|- style=\"background-color: #ffc\"\n"
|
||||
!! "|-\n"
|
||||
),
|
||||
"| { $task.value<count> }\n",
|
||||
( $task.key ~~ /\d/
|
||||
?? "|data-sort-value=\"{ $task.key.&naturally }\"| [[{uri-escape $task.key}|{$task.key}]]\n"
|
||||
!! "| [[{uri-escape $task.key}|{$task.key}]]\n"
|
||||
),
|
||||
"|{ $task.value<top-n>.comb[|$which].join('||') }"
|
||||
}
|
||||
|
||||
$out.say( "|}\n</div>" );
|
||||
$out.close;
|
||||
|
||||
note "Table file saved as: {$tablefile.IO.absolute}";
|
||||
|
||||
sub mediawiki-query ($site, $type, *%query) {
|
||||
my $url = "$site/api.php?" ~ uri-query-string(
|
||||
:action<query>, :format<json>, :formatversion<2>, |%query);
|
||||
my $continue = '';
|
||||
|
||||
gather loop {
|
||||
my $response = $client.get("$url&$continue");
|
||||
my $data = from-json($response.content);
|
||||
take $_ for $data.<query>.{$type}.values;
|
||||
$continue = uri-query-string |($data.<query-continue>{*}».hash.hash or last);
|
||||
}
|
||||
}
|
||||
|
||||
sub uri-query-string (*%fields) { %fields.map({ "{.key}={uri-escape .value}" }).join("&") }
|
||||
|
||||
sub naturally ($a) { $a.lc.subst(/(\d+)/, ->$/ {0~(65+$0.chars).chr~$0},:g) }
|
||||
|
||||
sub clear { "\r" ~ ' ' x 116 ~ "\r" }
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# Project: Rosetta Code/Count examples
|
||||
|
||||
load "stdlib.ring"
|
||||
ros= download("http://rosettacode.org/wiki/Category:Programming_Tasks")
|
||||
pos = 1
|
||||
num = 0
|
||||
totalros = 0
|
||||
rosname = ""
|
||||
rostitle = ""
|
||||
for n = 1 to len(ros)
|
||||
nr = searchstring(ros,'<li><a href="/wiki/',pos)
|
||||
if nr = 0
|
||||
exit
|
||||
else
|
||||
pos = nr + 1
|
||||
ok
|
||||
nr = searchname(nr)
|
||||
nr = searchtitle(nr)
|
||||
next
|
||||
see nl
|
||||
see "Total: " + totalros + " examples." + nl
|
||||
|
||||
func searchstring(str,substr,n)
|
||||
newstr=right(str,len(str)-n+1)
|
||||
nr = substr(newstr, substr)
|
||||
if nr = 0
|
||||
return 0
|
||||
else
|
||||
return n + nr -1
|
||||
ok
|
||||
|
||||
func searchname(sn)
|
||||
nr2 = searchstring(ros,'">',sn)
|
||||
nr3 = searchstring(ros,"</a></li>",sn)
|
||||
rosname = substr(ros,nr2+2,nr3-nr2-2)
|
||||
return sn
|
||||
|
||||
func searchtitle(sn)
|
||||
st = searchstring(ros,"title=",sn)
|
||||
rostitle = substr(ros,sn+19,st-sn-21)
|
||||
rostitle = "rosettacode.org/wiki/" + rostitle
|
||||
rostitle = download(rostitle)
|
||||
sum = count(rostitle,"Edit section:")
|
||||
num = num + 1
|
||||
see "" + num + ". " + rosname + ": " + sum + " examples." + nl
|
||||
totalros = totalros + sum
|
||||
return sn
|
||||
|
||||
func count(cstring,dstring)
|
||||
sum = 0
|
||||
while substr(cstring,dstring) > 0
|
||||
sum = sum + 1
|
||||
cstring = substr(cstring,substr(cstring,dstring)+len(string(sum)))
|
||||
end
|
||||
return sum
|
||||
|
|
@ -0,0 +1,51 @@
|
|||
require 'open-uri'
|
||||
require 'rexml/document'
|
||||
|
||||
module RosettaCode
|
||||
|
||||
URL_ROOT = "http://rosettacode.org/mw"
|
||||
|
||||
def self.get_url(page, query)
|
||||
begin
|
||||
# Ruby 1.9.2
|
||||
pstr = URI.encode_www_form_component(page)
|
||||
qstr = URI.encode_www_form(query)
|
||||
rescue NoMethodError
|
||||
require 'cgi'
|
||||
pstr = CGI.escape(page)
|
||||
qstr = query.map {|k,v|
|
||||
"%s=%s" % [CGI.escape(k.to_s), CGI.escape(v.to_s)]}.join("&")
|
||||
end
|
||||
url = "#{URL_ROOT}/#{pstr}?#{qstr}"
|
||||
p url if $DEBUG
|
||||
url
|
||||
end
|
||||
|
||||
def self.get_api_url(query)
|
||||
get_url "api.php", query
|
||||
end
|
||||
|
||||
def self.category_members(category)
|
||||
query = {
|
||||
"action" => "query",
|
||||
"list" => "categorymembers",
|
||||
"cmtitle" => "Category:#{category}",
|
||||
"format" => "xml",
|
||||
"cmlimit" => 500,
|
||||
}
|
||||
while true
|
||||
url = get_api_url query
|
||||
doc = REXML::Document.new open(url)
|
||||
|
||||
REXML::XPath.each(doc, "//cm") do |task|
|
||||
yield task.attribute("title").value
|
||||
end
|
||||
|
||||
continue = REXML::XPath.first(doc, "//query-continue")
|
||||
break if continue.nil?
|
||||
cm = REXML::XPath.first(continue, "categorymembers")
|
||||
query["cmcontinue"] = cm.attribute("cmcontinue").value
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
require 'rosettacode'
|
||||
|
||||
total_examples = 0
|
||||
|
||||
RosettaCode.category_members("Programming_Tasks") do |task|
|
||||
url = RosettaCode.get_url("index.php", {"action" => "raw", "title" => task})
|
||||
examples = open(url).read.scan("=={{header").length
|
||||
puts "#{task}: #{examples}"
|
||||
total_examples += examples
|
||||
end
|
||||
|
||||
puts
|
||||
puts "Total: #{total_examples}"
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
html "<table border=1><tr bgcolor=wheat align=center><td>Num</td><td>Task</td><td>Examples</td></tr>"
|
||||
|
||||
a$ = httpGet$("http://rosettacode.org/wiki/Category:Programming_Tasks")
|
||||
a$ = word$(a$,1,"</table></div>")
|
||||
i = instr(a$,"<a href=""/wiki/")
|
||||
i = instr(a$,"<a href=""/wiki/",i+1)
|
||||
while i > 0
|
||||
count = count + 1
|
||||
i = instr(a$,"<a href=""/wiki/",i+1)
|
||||
j = instr(a$,">",i+5)
|
||||
a1$ = mid$(a$,i+15,j-i)
|
||||
taskId$ = word$(a1$,1,"""")
|
||||
task$ = word$(a1$,3,"""")
|
||||
url$ = "http://rosettacode.org/wiki/";taskId$
|
||||
a2$ = httpGet$(url$)
|
||||
ii = instr(a2$,"<span class=""tocnumber"">")
|
||||
jj = 0
|
||||
while ii > 0
|
||||
jj = ii
|
||||
ii = instr(a2$,"<span class=""tocnumber"">",ii+10)
|
||||
wend
|
||||
if jj = 0 then
|
||||
examp = 0
|
||||
else
|
||||
kk = instr(a2$,"<",jj+24)
|
||||
examp = int(val(mid$(a2$,jj+24,kk-jj-24)))
|
||||
end if
|
||||
html "<tr><td align=right>";count;"</td><td>";task$;"</td><td align=right>";examp;"</td></tr>"
|
||||
totExamp = totExamp + examp
|
||||
wend
|
||||
html "<tr bgcolor=wheat><td>**</td><td>** Total **</td><td align=right>";totExamp;"</td></tr></table>"
|
||||
end
|
||||
|
|
@ -0,0 +1,112 @@
|
|||
extern crate reqwest;
|
||||
extern crate url;
|
||||
extern crate rustc_serialize;
|
||||
|
||||
use std::io::Read;
|
||||
use self::url::Url;
|
||||
use rustc_serialize::json::{self, Json};
|
||||
|
||||
pub struct Task {
|
||||
page_id: u64,
|
||||
pub title: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ParseError {
|
||||
/// Something went wrong with the HTTP request to the API.
|
||||
Http(reqwest::Error),
|
||||
|
||||
/// There was a problem parsing the API response into JSON.
|
||||
Json(json::ParserError),
|
||||
|
||||
/// Unexpected JSON format from response
|
||||
UnexpectedFormat,
|
||||
}
|
||||
impl From<json::ParserError> for ParseError {
|
||||
fn from(error: json::ParserError) -> Self {
|
||||
ParseError::Json(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<reqwest::Error> for ParseError {
|
||||
fn from(error: reqwest::Error) -> Self {
|
||||
ParseError::Http(error)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fn construct_query_category(category: &str) -> Url {
|
||||
let mut base_url = Url::parse("http://rosettacode.org/mw/api.php").unwrap();
|
||||
let cat = format!("Category:{}", category);
|
||||
let query_pairs = vec![("action", "query"),
|
||||
("format", "json"),
|
||||
("list", "categorymembers"),
|
||||
("cmlimit", "500"),
|
||||
("cmtitle", &cat),
|
||||
("continue", "")];
|
||||
base_url.query_pairs_mut().extend_pairs(query_pairs.into_iter());
|
||||
base_url
|
||||
}
|
||||
|
||||
fn construct_query_task_content(task_id: &str) -> Url {
|
||||
let mut base_url = Url::parse("http://rosettacode.org/mw/api.php").unwrap();
|
||||
let mut query_pairs =
|
||||
vec![("action", "query"), ("format", "json"), ("prop", "revisions"), ("rvprop", "content")];
|
||||
query_pairs.push(("pageids", task_id));
|
||||
base_url.query_pairs_mut().extend_pairs(query_pairs.into_iter());
|
||||
base_url
|
||||
}
|
||||
|
||||
fn query_api(url: Url) -> Result<Json, ParseError> {
|
||||
let mut response = try!(reqwest::get(url.as_str()));
|
||||
// Build JSON
|
||||
let mut body = String::new();
|
||||
response.read_to_string(&mut body).unwrap();
|
||||
|
||||
Ok(try!(Json::from_str(&body)))
|
||||
}
|
||||
|
||||
fn parse_all_tasks(reply: &Json) -> Result<Vec<Task>, ParseError> {
|
||||
let json_to_task = |json: &Json| -> Result<Task, ParseError> {
|
||||
let page_id: u64 = try!(json.find("pageid")
|
||||
.and_then(|id| id.as_u64())
|
||||
.ok_or(ParseError::UnexpectedFormat));
|
||||
let title: &str = try!(json.find("title")
|
||||
.and_then(|title| title.as_string())
|
||||
.ok_or(ParseError::UnexpectedFormat));
|
||||
|
||||
Ok(Task {
|
||||
page_id: page_id,
|
||||
title: title.to_owned(),
|
||||
})
|
||||
};
|
||||
let tasks_json = try!(reply.find_path(&["query", "categorymembers"])
|
||||
.and_then(|tasks| tasks.as_array())
|
||||
.ok_or(ParseError::UnexpectedFormat));
|
||||
|
||||
// Convert into own type
|
||||
tasks_json.iter().map(json_to_task).collect()
|
||||
}
|
||||
fn count_number_examples(task: &Json, task_id: u64) -> Result<u32, ParseError> {
|
||||
let revisions =
|
||||
try!(task.find_path(&["query", "pages", task_id.to_string().as_str(), "revisions"])
|
||||
.and_then(|content| content.as_array())
|
||||
.ok_or(ParseError::UnexpectedFormat));
|
||||
let content = try!(revisions[0]
|
||||
.find("*")
|
||||
.and_then(|content| content.as_string())
|
||||
.ok_or(ParseError::UnexpectedFormat));
|
||||
Ok(content.split("=={{header").count() as u32)
|
||||
}
|
||||
|
||||
pub fn query_all_tasks() -> Vec<Task> {
|
||||
let query = construct_query_category("Programming_Tasks");
|
||||
let json: Json = query_api(query).unwrap();
|
||||
parse_all_tasks(&json).unwrap()
|
||||
}
|
||||
|
||||
pub fn query_a_task(task: &Task) -> u32 {
|
||||
let query = construct_query_task_content(&task.page_id.to_string());
|
||||
let json: Json = query_api(query).unwrap();
|
||||
count_number_examples(&json, task.page_id).unwrap()
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
extern crate count_examples;
|
||||
fn main() {
|
||||
let all_tasks = count_examples::query_all_tasks();
|
||||
for task in &all_tasks {
|
||||
let count = count_examples::query_a_task(task);
|
||||
println!("Task: {} has {} examples", task.title, count);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
import scala.language.postfixOps
|
||||
|
||||
object TaskCount extends App {
|
||||
import java.net.{ URL, URLEncoder }
|
||||
import scala.io.Source.fromURL
|
||||
|
||||
System.setProperty("http.agent", "*")
|
||||
val allTasksURL =
|
||||
"http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
|
||||
val allTasks = xml.parsing.XhtmlParser(fromURL(new URL(allTasksURL)))
|
||||
|
||||
val regexExpr = "(?i)==\\{\\{header\\|".r
|
||||
|
||||
def oneTaskURL(title: String) = {
|
||||
println(s"Check $title")
|
||||
"http://www.rosettacode.org/w/index.php?title=%s&action=raw" format URLEncoder.encode(title.replace(' ', '_'), "UTF-8")
|
||||
}
|
||||
|
||||
def count(title: String) = regexExpr findAllIn fromURL(new URL(oneTaskURL(title)))(io.Codec.UTF8).mkString length
|
||||
|
||||
val counts = for (task <- allTasks \\ "cm" \\ "@title" map (_.text)) yield scala.actors.Futures.future((task, count(task)))
|
||||
|
||||
counts map (_.apply) map Function.tupled("%s: %d examples." format (_, _)) foreach println
|
||||
println("\nTotal: %d examples." format (counts map (_.apply._2) sum))
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
var lwp = require('LWP::UserAgent').new(agent => 'Mozilla/5.0');
|
||||
|
||||
var site = 'http://rosettacode.org';
|
||||
var list_url = '/mw/api.php?action=query&list=categorymembers&'+
|
||||
'cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml';
|
||||
|
||||
var content = lwp.get(site + list_url).decoded_content;
|
||||
|
||||
while (var m = content.match(/cm.*?title="(.*?)"/g)) {
|
||||
(var slug = m[0]).gsub!(' ', '_');
|
||||
var count = lwp.get("#{site}/wiki/#{slug}").decoded_content.count(/toclevel-1/g);
|
||||
say "#{m[0]}: #{count} examples";
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
$$ MODE TUSCRIPT
|
||||
url="http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
|
||||
data=REQUEST (url)
|
||||
|
||||
BUILD S_TABLE beg=*
|
||||
DATA :title=":
|
||||
BUILD S_TABLE end=*
|
||||
DATA :":
|
||||
|
||||
titles=EXTRACT (data,beg|,end,1,0,"~~")
|
||||
titles=SPLIT (titles,":~~:")
|
||||
sz_titles=SIZE (titles)
|
||||
|
||||
BUILD R_TABLE header=":==\{\{header|:"
|
||||
all=*
|
||||
|
||||
ERROR/STOP CREATE ("tasks",seq-e,-std-)
|
||||
|
||||
COMPILE
|
||||
LOOP title=titles
|
||||
ask=*
|
||||
ask =SET_VALUE(ask,"title",title)
|
||||
ask =SET_VALUE(ask,"action","raw")
|
||||
ask =ENCODE (ask,cgi)
|
||||
http ="http://www.rosettacode.org/mw/index.php"
|
||||
url =CONCAT (http,"?",ask)
|
||||
data =REQUEST (url)
|
||||
header =FILTER_INDEX (data,header,-)
|
||||
sz_header=SIZE(header)
|
||||
line =CONCAT (title,"=",sz_header," members")
|
||||
FILE "tasks" = line
|
||||
all =APPEND(all,sz_header)
|
||||
ENDLOOP
|
||||
|
||||
ENDCOMPILE
|
||||
all =JOIN(all),sum=SUM(all),time=time()
|
||||
line=CONCAT (time,": ", sz_titles, " Programing Tasks: ", sum, " solutions")
|
||||
|
||||
FILE "tasks" = line
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
package require Tcl 8.5
|
||||
package require http
|
||||
package require json
|
||||
|
||||
fconfigure stdout -buffering none
|
||||
|
||||
proc get_tasks {category} {
|
||||
set start [clock milliseconds]
|
||||
puts -nonewline "getting $category members..."
|
||||
set base_url http://www.rosettacode.org/w/api.php
|
||||
set query {action query list categorymembers cmtitle Category:%s format json cmlimit 500}
|
||||
set this_query [dict create {*}[split [format $query $category]]]
|
||||
set tasks [list]
|
||||
|
||||
while {1} {
|
||||
set url [join [list $base_url [http::formatQuery {*}$this_query]] ?]
|
||||
set response [http::geturl $url]
|
||||
if {[set s [http::status $response]] ne "ok" || [http::ncode $response] != 200} {
|
||||
error "Oops: url=$url\nstatus=$s\nhttp code=[http::code $response]"
|
||||
}
|
||||
set data [json::json2dict [http::data $response]]
|
||||
http::cleanup $response
|
||||
|
||||
# add tasks to list
|
||||
foreach task [dict get $data query categorymembers] {
|
||||
lappend tasks [dict get [dict create {*}$task] title]
|
||||
}
|
||||
|
||||
if {[catch {dict get $data query-continue categorymembers cmcontinue} continue_task] != 0} {
|
||||
# no more continuations, we're done
|
||||
break
|
||||
}
|
||||
dict set this_query cmcontinue $continue_task
|
||||
}
|
||||
puts " found [llength $tasks] tasks in [expr {[clock milliseconds] - $start}] milliseconds"
|
||||
return $tasks
|
||||
}
|
||||
|
||||
# This proc can be replaced by a single regexp command:
|
||||
# set count [regexp -all "***=$needle" $haystack]
|
||||
# However this proc is more efficient -- we're dealing with plain strings only.
|
||||
proc count_substrings {needle haystack} {
|
||||
set count 0
|
||||
set idx 0
|
||||
while {[set idx [string first $needle $haystack $idx]] != -1} {
|
||||
incr count
|
||||
incr idx
|
||||
}
|
||||
return $count
|
||||
}
|
||||
|
||||
set total 0
|
||||
foreach task [get_tasks Programming_Tasks] {
|
||||
set url [format "http://www.rosettacode.org/w/index.php?title=%s&action=raw" [string map {{ } _} $task]]
|
||||
set response [http::geturl $url]
|
||||
if {[set s [http::status $response]] ne "ok" || [http::ncode $response] != 200} {
|
||||
error "Oops: url=$url\nstatus=$s\nhttp code=[http::code $response]"
|
||||
}
|
||||
set count [count_substrings "\{\{header|" [http::data $response]]
|
||||
puts [format "%3d examples in %s" $count $task]
|
||||
http::cleanup $response
|
||||
incr total $count
|
||||
}
|
||||
|
||||
puts "\nTotal: $total examples"
|
||||
|
|
@ -0,0 +1,20 @@
|
|||
#!/usr/bin/env bash
|
||||
SITE=https://www.rosettacode.org
|
||||
API=$SITE/mw/api.php
|
||||
PAGES=$SITE/mw/index.php
|
||||
query="$API?action=query"
|
||||
query+=$(printf '&%s' \
|
||||
list=categorymembers \
|
||||
cmtitle=Category:Programming_Tasks \
|
||||
cmlimit=500)
|
||||
total=0
|
||||
while read title; do
|
||||
t=${title// /_}
|
||||
tasks=$(curl -s "$PAGES?title=$t&action=raw" \
|
||||
| grep -c '{{header|')
|
||||
printf '%s: %d examples.\n' "$title" "$tasks"
|
||||
let total+=tasks
|
||||
done < <(curl -s "$query&format=json" \
|
||||
| jq -r '.query.categorymembers[].title')
|
||||
|
||||
printf '\nTotal: %d examples.\n' "$total"
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
/* rc_count_examples.wren */
|
||||
|
||||
import "./pattern" for Pattern
|
||||
|
||||
var CURLOPT_URL = 10002
|
||||
var CURLOPT_FOLLOWLOCATION = 52
|
||||
var CURLOPT_WRITEFUNCTION = 20011
|
||||
var CURLOPT_WRITEDATA = 10001
|
||||
|
||||
foreign class Buffer {
|
||||
construct new() {} // C will allocate buffer of a suitable size
|
||||
|
||||
foreign value // returns buffer contents as a string
|
||||
}
|
||||
|
||||
foreign class Curl {
|
||||
construct easyInit() {}
|
||||
|
||||
foreign easySetOpt(opt, param)
|
||||
|
||||
foreign easyPerform()
|
||||
|
||||
foreign easyCleanup()
|
||||
}
|
||||
|
||||
var curl = Curl.easyInit()
|
||||
|
||||
var getContent = Fn.new { |url|
|
||||
var buffer = Buffer.new()
|
||||
curl.easySetOpt(CURLOPT_URL, url)
|
||||
curl.easySetOpt(CURLOPT_FOLLOWLOCATION, 1)
|
||||
curl.easySetOpt(CURLOPT_WRITEFUNCTION, 0) // write function to be supplied by C
|
||||
curl.easySetOpt(CURLOPT_WRITEDATA, buffer)
|
||||
curl.easyPerform()
|
||||
return buffer.value
|
||||
}
|
||||
|
||||
var url = "https://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
|
||||
var content = getContent.call(url)
|
||||
var p = Pattern.new("title/=\"[+1^\"]\"")
|
||||
var matches = p.findAll(content)
|
||||
for (m in matches) {
|
||||
var title = m.capsText[0].replace("'", "'").replace(""", "\"")
|
||||
var title2 = title.replace(" ", "_").replace("+", "\%2B")
|
||||
var taskUrl = "https://www.rosettacode.org/w/index.php?title=%(title2)&action=raw"
|
||||
var taskContent = getContent.call(taskUrl)
|
||||
var lines = taskContent.split("\n")
|
||||
var count = lines.count { |line| line.trim().startsWith("=={{header|") }
|
||||
System.print("%(title) : %(count) examples")
|
||||
}
|
||||
|
||||
curl.easyCleanup()
|
||||
|
|
@ -0,0 +1,191 @@
|
|||
/* gcc rc_count_examples.c -o rc_count_examples -lcurl -lwren -lm */
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <curl/curl.h>
|
||||
#include "wren.h"
|
||||
|
||||
struct MemoryStruct {
|
||||
char *memory;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
/* C <=> Wren interface functions */
|
||||
|
||||
static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) {
|
||||
size_t realsize = size * nmemb;
|
||||
struct MemoryStruct *mem = (struct MemoryStruct *)userp;
|
||||
|
||||
char *ptr = realloc(mem->memory, mem->size + realsize + 1);
|
||||
if(!ptr) {
|
||||
/* out of memory! */
|
||||
printf("not enough memory (realloc returned NULL)\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
mem->memory = ptr;
|
||||
memcpy(&(mem->memory[mem->size]), contents, realsize);
|
||||
mem->size += realsize;
|
||||
mem->memory[mem->size] = 0;
|
||||
return realsize;
|
||||
}
|
||||
|
||||
void C_bufferAllocate(WrenVM* vm) {
|
||||
struct MemoryStruct *ms = (struct MemoryStruct *)wrenSetSlotNewForeign(vm, 0, 0, sizeof(struct MemoryStruct));
|
||||
ms->memory = malloc(1);
|
||||
ms->size = 0;
|
||||
}
|
||||
|
||||
void C_bufferFinalize(void* data) {
|
||||
struct MemoryStruct *ms = (struct MemoryStruct *)data;
|
||||
free(ms->memory);
|
||||
}
|
||||
|
||||
void C_curlAllocate(WrenVM* vm) {
|
||||
CURL** pcurl = (CURL**)wrenSetSlotNewForeign(vm, 0, 0, sizeof(CURL*));
|
||||
*pcurl = curl_easy_init();
|
||||
}
|
||||
|
||||
void C_value(WrenVM* vm) {
|
||||
struct MemoryStruct *ms = (struct MemoryStruct *)wrenGetSlotForeign(vm, 0);
|
||||
wrenSetSlotString(vm, 0, ms->memory);
|
||||
}
|
||||
|
||||
void C_easyPerform(WrenVM* vm) {
|
||||
CURL* curl = *(CURL**)wrenGetSlotForeign(vm, 0);
|
||||
curl_easy_perform(curl);
|
||||
}
|
||||
|
||||
void C_easyCleanup(WrenVM* vm) {
|
||||
CURL* curl = *(CURL**)wrenGetSlotForeign(vm, 0);
|
||||
curl_easy_cleanup(curl);
|
||||
}
|
||||
|
||||
void C_easySetOpt(WrenVM* vm) {
|
||||
CURL* curl = *(CURL**)wrenGetSlotForeign(vm, 0);
|
||||
CURLoption opt = (CURLoption)wrenGetSlotDouble(vm, 1);
|
||||
if (opt < 10000) {
|
||||
long lparam = (long)wrenGetSlotDouble(vm, 2);
|
||||
curl_easy_setopt(curl, opt, lparam);
|
||||
} else if (opt < 20000) {
|
||||
if (opt == CURLOPT_WRITEDATA) {
|
||||
struct MemoryStruct *ms = (struct MemoryStruct *)wrenGetSlotForeign(vm, 2);
|
||||
curl_easy_setopt(curl, opt, (void *)ms);
|
||||
} else if (opt == CURLOPT_URL) {
|
||||
const char *url = wrenGetSlotString(vm, 2);
|
||||
curl_easy_setopt(curl, opt, url);
|
||||
}
|
||||
} else if (opt < 30000) {
|
||||
if (opt == CURLOPT_WRITEFUNCTION) {
|
||||
curl_easy_setopt(curl, opt, &WriteMemoryCallback);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
WrenForeignClassMethods bindForeignClass(WrenVM* vm, const char* module, const char* className) {
|
||||
WrenForeignClassMethods methods;
|
||||
methods.allocate = NULL;
|
||||
methods.finalize = NULL;
|
||||
if (strcmp(module, "main") == 0) {
|
||||
if (strcmp(className, "Buffer") == 0) {
|
||||
methods.allocate = C_bufferAllocate;
|
||||
methods.finalize = C_bufferFinalize;
|
||||
} else if (strcmp(className, "Curl") == 0) {
|
||||
methods.allocate = C_curlAllocate;
|
||||
}
|
||||
}
|
||||
return methods;
|
||||
}
|
||||
|
||||
WrenForeignMethodFn bindForeignMethod(
|
||||
WrenVM* vm,
|
||||
const char* module,
|
||||
const char* className,
|
||||
bool isStatic,
|
||||
const char* signature) {
|
||||
if (strcmp(module, "main") == 0) {
|
||||
if (strcmp(className, "Buffer") == 0) {
|
||||
if (!isStatic && strcmp(signature, "value") == 0) return C_value;
|
||||
} else if (strcmp(className, "Curl") == 0) {
|
||||
if (!isStatic && strcmp(signature, "easySetOpt(_,_)") == 0) return C_easySetOpt;
|
||||
if (!isStatic && strcmp(signature, "easyPerform()") == 0) return C_easyPerform;
|
||||
if (!isStatic && strcmp(signature, "easyCleanup()") == 0) return C_easyCleanup;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static void writeFn(WrenVM* vm, const char* text) {
|
||||
printf("%s", text);
|
||||
}
|
||||
|
||||
void errorFn(WrenVM* vm, WrenErrorType errorType, const char* module, const int line, const char* msg) {
|
||||
switch (errorType) {
|
||||
case WREN_ERROR_COMPILE:
|
||||
printf("[%s line %d] [Error] %s\n", module, line, msg);
|
||||
break;
|
||||
case WREN_ERROR_STACK_TRACE:
|
||||
printf("[%s line %d] in %s\n", module, line, msg);
|
||||
break;
|
||||
case WREN_ERROR_RUNTIME:
|
||||
printf("[Runtime Error] %s\n", msg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
char *readFile(const char *fileName) {
|
||||
FILE *f = fopen(fileName, "r");
|
||||
fseek(f, 0, SEEK_END);
|
||||
long fsize = ftell(f);
|
||||
rewind(f);
|
||||
char *script = malloc(fsize + 1);
|
||||
fread(script, 1, fsize, f);
|
||||
fclose(f);
|
||||
script[fsize] = 0;
|
||||
return script;
|
||||
}
|
||||
|
||||
static void loadModuleComplete(WrenVM* vm, const char* module, WrenLoadModuleResult result) {
|
||||
if( result.source) free((void*)result.source);
|
||||
}
|
||||
|
||||
WrenLoadModuleResult loadModule(WrenVM* vm, const char* name) {
|
||||
WrenLoadModuleResult result = {0};
|
||||
if (strcmp(name, "random") != 0 && strcmp(name, "meta") != 0) {
|
||||
result.onComplete = loadModuleComplete;
|
||||
char fullName[strlen(name) + 6];
|
||||
strcpy(fullName, name);
|
||||
strcat(fullName, ".wren");
|
||||
result.source = readFile(fullName);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
WrenConfiguration config;
|
||||
wrenInitConfiguration(&config);
|
||||
config.writeFn = &writeFn;
|
||||
config.errorFn = &errorFn;
|
||||
config.bindForeignClassFn = &bindForeignClass;
|
||||
config.bindForeignMethodFn = &bindForeignMethod;
|
||||
config.loadModuleFn = &loadModule;
|
||||
WrenVM* vm = wrenNewVM(&config);
|
||||
const char* module = "main";
|
||||
const char* fileName = "rc_count_examples.wren";
|
||||
char *script = readFile(fileName);
|
||||
WrenInterpretResult result = wrenInterpret(vm, module, script);
|
||||
switch (result) {
|
||||
case WREN_RESULT_COMPILE_ERROR:
|
||||
printf("Compile Error!\n");
|
||||
break;
|
||||
case WREN_RESULT_RUNTIME_ERROR:
|
||||
printf("Runtime Error!\n");
|
||||
break;
|
||||
case WREN_RESULT_SUCCESS:
|
||||
break;
|
||||
}
|
||||
wrenFreeVM(vm);
|
||||
free(script);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
var [const] YAJL=Import("zklYAJL")[0], CURL=Import("zklCurl");
|
||||
|
||||
fcn getTasks(language){
|
||||
continueValue,tasks:="",Data(0,String); // "nm\0nm\0...."
|
||||
do{
|
||||
page:=CURL().get(("http://rosettacode.org/mw/api.php?"
|
||||
"action=query&cmlimit=500"
|
||||
"&format=json"
|
||||
"&list=categorymembers"
|
||||
"&cmtitle=Category:%s"
|
||||
"&cmcontinue=%s").fmt(language,continueValue));
|
||||
page=page[0].del(0,page[1]); // get rid of HTML header
|
||||
json:=YAJL().write(page).close();
|
||||
json["query"]["categorymembers"].pump(tasks,T("get","title"));
|
||||
continueValue=json.find("continue") //continue:-||,cmcontinue:page|954|19)
|
||||
.toList().apply("concat","=").concat("&");
|
||||
}while(continueValue);
|
||||
tasks
|
||||
}
|
||||
re:=RegExp(0'!\s+==\s*{{\s*header\s*\|!); // == {{ header | zkl
|
||||
foreach task in (getTasks("Programming_Tasks")){
|
||||
page:=CURL().get(
|
||||
"http://www.rosettacode.org/mw/index.php?title=%s&action=raw"
|
||||
.fmt(CURL.urlEncode(task)));
|
||||
page=page[0].del(0,page[1]); // get rid of HTML header
|
||||
cnt,n:=0,0; while(re.search(page,True,n)){ cnt+=1; n=re.matched[0].sum(0); }
|
||||
"%4d: %s".fmt(cnt,task).println();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue