Data commit

This commit is contained in:
Ingy döt Net 2023-07-01 11:58:00 -04:00
parent 7387c8f97b
commit cb5bb5e222
199093 changed files with 3378972 additions and 0 deletions

View file

@ -0,0 +1,3 @@
---
from: http://rosettacode.org/wiki/Rosetta_Code/Count_examples
note: Rosetta Code related

View file

@ -0,0 +1,16 @@
Find the total number of programming examples for each [[:Category:Programming Tasks|task]] and the total for all tasks.
Essentially, count the number of occurrences of <tt>=={{header|</tt> on each task page.
Output:
<pre>100 doors: 20 examples.
99 Bottles of Beer: 29 examples.
Abstract type: 10 examples.
Total: X examples.</pre>
For a full output, updated periodically, see [[Rosetta Code/Count examples/Full list]].
You'll need to use the Media Wiki API, which you can find out about locally, [http://rosettacode.org/mw/api.php here], or in Media Wiki's API documentation at, [http://www.mediawiki.org/wiki/API_Query API:Query]

View file

@ -0,0 +1,101 @@
with Aws.Client, Aws.Messages, Aws.Response, Aws.Resources, Aws.Url;
with Dom.Readers, Dom.Core, Dom.Core.Documents, Dom.Core.Nodes, Dom.Core.Attrs;
with Input_Sources.Strings, Unicode, Unicode.Ces.Utf8;
with Ada.Strings.Unbounded, Ada.Strings.Fixed, Ada.Text_IO, Ada.Command_Line;
with Ada.Containers.Vectors;
use Aws.Client, Aws.Messages, Aws.Response, Aws.Resources, Aws.Url;
use Dom.Readers, Dom.Core, Dom.Core.Documents, Dom.Core.Nodes, Dom.Core.Attrs;
use Aws, Ada.Strings.Unbounded, Ada.Strings.Fixed, Input_Sources.Strings;
use Ada.Text_IO, Ada.Command_Line;
procedure Count_Examples is
package Members_Vectors is new Ada.Containers.Vectors (
Index_Type => Positive,
Element_Type => Unbounded_String);
use Members_Vectors;
Exemples : Vector;
Nbr_Lg, Total : Natural := 0;
procedure Get_Vector (Category : in String; Mbr_Vector : in out Vector) is
Reader : Tree_Reader;
Doc : Document;
List : Node_List;
N : Node;
A : Attr;
Page : Aws.Response.Data;
Uri_Xml : constant String :=
"http://rosettacode.org/mw/api.php?action=query&list=categorymembers"
&
"&format=xml&cmlimit=500&cmtitle=Category:";
begin
Page := Client.Get (Uri_Xml & Category);
if Response.Status_Code (Page) not in Messages.Success then
raise Client.Connection_Error;
end if;
declare
Xml : constant String := Message_Body (Page);
Source : String_Input;
begin
Open
(Xml'Unrestricted_Access,
Unicode.Ces.Utf8.Utf8_Encoding,
Source);
Parse (Reader, Source);
Close (Source);
end;
Doc := Get_Tree (Reader);
List := Get_Elements_By_Tag_Name (Doc, "cm");
for Index in 1 .. Length (List) loop
N := Item (List, Index - 1);
A := Get_Named_Item (Attributes (N), "title");
Append (Mbr_Vector, To_Unbounded_String (Value (A)));
end loop;
Free (List);
Free (Reader);
end Get_Vector;
function Scan_Page (Title : String) return Natural is
Page : Aws.Response.Data;
File : Aws.Resources.File_Type;
Buffer : String (1 .. 1024);
Languages, Position, Last : Natural := 0;
begin
Page :=
Client.Get
("http://rosettacode.org/mw/index.php?title=" &
Aws.Url.Encode (Title) &
"&action=raw");
Response.Message_Body (Page, File);
while not End_Of_File (File) loop
Resources.Get_Line (File, Buffer, Last);
Position :=
Index
(Source => Buffer (Buffer'First .. Last),
Pattern => "=={{header|");
if Position > 0 then
Languages := Languages + 1;
end if;
end loop;
Close (File);
return Languages;
end Scan_Page;
begin
Get_Vector ("Programming_Tasks", Exemples);
for I in First_Index (Exemples) .. Last_Index (Exemples) loop
declare
Title : constant String :=
To_String (Members_Vectors.Element (Exemples, I));
begin
Nbr_Lg := Scan_Page (Title);
Total := Total + Nbr_Lg;
Put_Line (Title & " :" & Integer'Image (Nbr_Lg) & " exemples.");
end;
end loop;
Put_Line ("Total :" & Integer'Image (Total) & " exemples.");
end Count_Examples;

View file

@ -0,0 +1,20 @@
UrlDownloadToFile
, http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml
, tasks.xml
FileRead, tasks, tasks.xml
pos = 0
quote = " ; "
regtitle := "<cm.*?title=" . quote . "(.*?)" . quote
While, pos := RegExMatch(tasks, regtitle, title, pos + 1)
{
UrlDownloadToFile
, % "http://www.rosettacode.org/w/index.php?title=" . title1 . "&action=raw"
, task.xml
FileRead, task, task.xml
RegExReplace(task, "\{\{header\|", "", count)
current := title1 . ": " . count . " examples.`n"
output .= current
TrayTip, current, % current
}
MsgBox % output
Return

View file

@ -0,0 +1,84 @@
VDU 23,22,640;512;8,16,16,128+8 : REM Enable UTF-8 support
SYS "LoadLibrary", "URLMON.DLL" TO urlmon%
SYS "GetProcAddress", urlmon%, "URLDownloadToFileA" TO UDTF
special$ = "+()'"
url$ = "http://www.rosettacode.org/w/api.php?action=query" + \
\ "&list=categorymembers&cmtitle=Category:Programming_Tasks" + \
\ "&cmlimit=500&format=xml"
file$ = @tmp$ + "tasks.xml"
SYS UDTF, 0, url$, file$, 0, 0 TO fail%
IF fail% ERROR 100, "File download failed (tasks)"
Total% = 0
file% = OPENIN(file$)
WHILE NOT EOF#file%
a$ = GET$#file%
i% = 0
REPEAT
i% = INSTR(a$, "title=", i%+1)
IF i% THEN
j% = INSTR(a$, ">", i%)
title$ = MID$(a$, i%+7, j%-i%-10)
REM Replace HTML codes:
REPEAT
k% = INSTR(title$, "&")
IF k% THEN
l% = INSTR(title$, ";", k%)
title$ = LEFT$(title$,k%-1) + \
\ FNhtml(MID$(title$,k%,l%-k%+1)) + MID$(title$,l%+1)
ENDIF
UNTIL k% = 0
t$ = title$
REM Substitute characters not allowed in a URL:
FOR s% = 1 TO LEN(special$)
REPEAT
s$ = MID$(special$, s%, 1)
k% = INSTR(t$, s$)
IF k% t$ = LEFT$(t$,k%-1) + "%" + STR$~ASCs$ + MID$(t$,k%+1)
UNTIL k% = 0
NEXT
url$ = "http://www.rosettacode.org/w/index.php?title=" + t$ + \
\ "&action=raw"
file$ = @tmp$ + "title.htm"
SYS UDTF, 0, url$, file$, 0, 0 TO fail%
IF fail% ERROR 100, "File download failed " + t$
examples% = 0
task% = OPENIN(file$)
WHILE NOT EOF#task%
IF INSTR(GET$#task%, "=={{header|") examples% += 1
ENDWHILE
CLOSE #task%
Total% += examples%
PRINT title$ ": " ; examples% " examples."
ENDIF
UNTIL i% = 0
i% = INSTR(a$, "cmcontinue=")
IF i% THEN
CLOSE #file%
j% = INSTR(a$, """", i%+1)
k% = INSTR(a$, """", j%+1)
url$ = "http://www.rosettacode.org/w/api.php?action=query" + \
\ "&list=categorymembers&cmtitle=Category:Programming_Tasks" + \
\ "&cmlimit=500&format=xml&cmcontinue=" + MID$(a$,j%+1,k%-j%)
REPEAT
i% = INSTR(url$, "|")
IF i% url$ = LEFT$(url$,i%-1) + "%7C" + MID$(url$,i%+1)
UNTIL i% = 0
file$ = @tmp$ + "tasks.xml"
SYS UDTF, 0, url$, file$, 0, 0 TO fail%
IF fail% ERROR 100, "File download failed (continue)"
file% = OPENIN(file$)
ENDIF
ENDWHILE
CLOSE #file%
PRINT ' "Total: " ; Total% " examples."
END
DEF FNhtml(h$)
IF LEFT$(h$,2) = "&#" THEN = CHR$(VALMID$(h$,3))
CASE h$ OF
WHEN "&quot;": = """"
ENDCASE
= h$

View file

@ -0,0 +1,41 @@
( ( get-page
=
. sys$(str$("wget -q -O wget.out \"" !arg \"))
& get$("wget.out",HT ML)
)
& get-page$"http://rosettacode.org/wiki/Category:Programming_Tasks"
: ? (table.?) ?tasklist (.table.) ?
& 0:?list
& whl
' ( !tasklist
: ?
( a
. (href.@(?:"/wiki/" ?href)) (title.?title)
& get-page$(str$("http://rosettacode.org/wiki/" !href))
: ?task
& 0:?cnt
& whl
' ( !task
: ?
( (span.(class.mw-headline) (id.?))
?span
(.span.)
?task
& !span
: ?
( a
. (href.@(?:"/wiki/Category:" ?))
(title.@(?:"Category:" ?))
)
@
(.a.)
?
)
& 1+!cnt:?cnt
)
& (!cnt.!title)+!list:?list
)
?tasklist
)
& lst$(list,taskfreq,NEW)
)

View file

@ -0,0 +1,65 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Net;
class Task {
private string _task;
private int _examples;
public Task(string task, int examples) {
_task = task;
_examples = examples;
}
public string Name {
get { return _task; }
}
public int Examples {
get { return _examples; }
}
public override string ToString() {
return String.Format("{0}: {1} examples.", this._task, this._examples);
}
}
class Program {
static List<string> GetTitlesFromCategory(string category, WebClient wc) {
string content = wc.DownloadString(
String.Format("http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:{0}&cmlimit=500&format=json", category)
);
return new Regex("\"title\":\"(.+?)\"").Matches(content).Cast<Match>().Select(x => x.Groups[1].Value.Replace("\\/", "/")).ToList();
}
static string GetSourceCodeFromPage(string page, WebClient wc) {
return wc.DownloadString(
String.Format("http://www.rosettacode.org/w/index.php?title={0}&action=raw", page)
);
}
static void Main(string[] args) {
WebClient wc = new WebClient();
List<Task> tasks = new List<Task>();
List<string> tasknames = GetTitlesFromCategory("Programming_Tasks", wc);
foreach (string task in tasknames) {
try {
string content = GetSourceCodeFromPage(WebUtility.UrlEncode(task), wc);
int count = new Regex("=={{header", RegexOptions.IgnoreCase).Matches(content).Count;
Task t = new Task(task, count);
Console.WriteLine(t);
tasks.Add(t);
}
catch (Exception ex) {
Console.WriteLine("**** Unable to get task \"" + task + "\": " + ex.Message);
}
}
Console.WriteLine("\nTotal: {0} examples.", tasks.Select(x => x.Examples).Sum());
}
}

View file

@ -0,0 +1,39 @@
(ns count-examples
(:import [java.net URLEncoder])
(:use [clojure.contrib.http.agent :only (http-agent string)]
[clojure.contrib.json :only (read-json)]
[clojure.contrib.string :only (join)]))
(defn url-encode [v] (URLEncoder/encode (str v) "utf-8"))
(defn rosettacode-get [path params]
(let [param-string (join "&" (for [[n v] params] (str (name n) "=" (url-encode v))))]
(string (http-agent (format "http://www.rosettacode.org/w/%s?%s" path param-string)))))
(defn rosettacode-query [params]
(read-json (rosettacode-get "api.php" (merge {:action "query" :format "json"} params))))
(defn list-cm
([params] (list-cm params nil))
([params continue]
(let [cm-params (merge {:list "categorymembers"} params (or continue {}))
result (rosettacode-query cm-params)]
(concat (-> result (:query) (:categorymembers))
(if-let [cmcontinue (-> result (:query-continue) (:categorymembers))]
(list-cm params cmcontinue))))))
(defn programming-tasks []
(let [result (list-cm {:cmtitle "Category:Programming_Tasks" :cmlimit 50})]
(map #(:title %) result)))
(defn task-count [task]
[task (count
(re-seq #"==\{\{header"
(rosettacode-get "index.php" {:action "raw" :title task})))])
(defn print-result []
(let [task-counts (map task-count (programming-tasks))]
(doseq [[task count] task-counts]
(println (str task ":") count)
(flush))
(println "Total: " (reduce #(+ %1 (second %2)) 0 task-counts))))

View file

@ -0,0 +1,16 @@
count-examples> (print-result)
100 doors: 73
24 game: 18
24 game/Solve: 14
99 Bottles of Beer: 89
Abstract type: 27
Accumulator factory: 23
Ackermann function: 73
Active Directory/Connect: 4
Active Directory/Search for a user: 3
Active object: 14
Add a variable to a class instance at runtime: 21
Address of a variable: 20
...
Total: 11216
nil

View file

@ -0,0 +1,48 @@
import tango.io.Stdout;
import tango.net.http.HttpClient;
import tango.net.http.HttpHeaders;
import tango.text.xml.Document;
import tango.text.Util;
alias HttpHeader.ContentLength CL;
auto url = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
void main()
{
auto client = new HttpClient (HttpClient.Get, url);
client.open();
char[] mainData, tmp;
int total, i;
void cat(void[] content) { tmp ~= cast(char[]) content; }
if (client.isResponseOK) {
client.read(&cat, client.getResponseHeaders.getInt(CL));
mainData = tmp;
tmp = null;
auto doc = new Document!(char);
doc.parse(mainData);
foreach (n; doc.query.descendant("cm").attribute("title")) {
auto subClient = new HttpClient(HttpClient.Get,
"http://www.rosettacode.org/w/index.php?title=" ~
replace(n.value.dup, ' ', '_') ~ "&action=raw");
subClient.open();
if (! subClient.isResponseOK) {
Stderr (client.getResponse);
break;
}
subClient.read(&cat, subClient.getResponseHeaders.getInt(CL));
foreach (segment; patterns(cast(char[])tmp, "=={{header|")) i++;
--i;
if (i) --i;
Stdout.formatln ("{0,-40} - {}", n.value, i);
total += i;
tmp = null;
i = 0;
}
Stdout("total examples: ", total).newline;
} else {
Stderr (client.getResponse);
}
}

View file

@ -0,0 +1,76 @@
package com.eglexamples.client;
import org.eclipse.edt.rui.widgets.*;
handler RosettaCodeHandler type RUIhandler{initialUI =[ui], title = "Rosetta Code Tasks and Counts"}
ui GridLayout{columns = 3, rows = 4, cellPadding = 4, children = [ b1, dg1, l1, l2, l3, l4 ]};
b1 Button{ layoutData = new GridLayoutData{ row = 1, column = 1 }, text = "Go!", onClick ::= b1_onClick };
l1 TextLabel{ layoutData = new GridLayoutData{ row = 1, column = 2 }, text = "Total Tasks:" };
l2 TextLabel{ layoutData = new GridLayoutData{ row = 1, column = 3 }, text = "0" };
l3 TextLabel{ layoutData = new GridLayoutData{ row = 2, column = 2 }, text = "Total Implementations:" };
l4 TextLabel{ layoutData = new GridLayoutData{ row = 2, column = 3 }, text = "0" };
dg1 DataGrid{ layoutData = new GridLayoutData{ row = 3, column = 1, horizontalSpan = 3 },
pageSize = 10, showScrollbar = true,
columns = [ new DataGridColumn{name = "title", displayName = "Task", width=220},
new DataGridColumn{name = "count", displayName = "Count", width=100} ] };
cmcontinue string?;
title string?;
allTasks Task[];
restBindingTasks IHttp? = new HttpRest{
restType = eglx.rest.ServiceType.TrueRest,
request.uri = "http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=1&format=json"};
restBindingPageDetail IHttp? = new HttpRest{
restType = eglx.rest.ServiceType.TrueRest,
request.uri = "http://rosettacode.org/mw/index.php"};
function b1_onClick(event Event in)
call ProxyFunctions.listTasks("") using restBindingTasks
returning to listTasksCallBack onException exceptionHandler;
end
function listTasksCallBack(retResult RosettaCodeJSON in)
title = retResult.query.categorymembers[1].title;
cmcontinue = retResult.queryContinue.categorymembers.cmcontinue;
call ProxyFunctions.fetchPageDetail(title) using restBindingPageDetail
returning to pageDetailCallBack onException exceptionHandler;
end
function pageDetailCallBack(pageResults string in)
count int = countSubstring("=={{header", pageResults);
allTasks.appendElement(new Task { title = title, count = count });
l2.text = l2.text as int + 1;
l4.text = l4.text as int + count;
if(cmcontinue != null)
call ProxyFunctions.listTasks(cmcontinue) using restBindingTasks
returning to listTasksCallBack onException exceptionHandler;
else
dg1.data = allTasks as any[];
end
end
function countSubstring(substr string in, str string in) returns(int)
if(str.length() > 0 and substr.length() > 0)
return (str.length() - str.replaceStr(subStr, "").length()) / subStr.length();
else
return 0;
end
end
function exceptionHandler(exp AnyException in)
end
end
record Task
title string;
count int;
end

View file

@ -0,0 +1,37 @@
package com.eglexamples.client;
library ProxyFunctions
function listTasks(continueLocation String in) returns (RosettaCodeJSON) {
@Rest{method = _GET, uriTemplate = "&cmcontinue={continueLocation}",
requestFormat = None, responseFormat = JSON}
}
end
function fetchPageDetail(title String in) returns (String) {
@Rest{method = _GET, uriTemplate = "?title={title}&action=raw",
requestFormat = None, responseFormat = None}
}
end
end
record RosettaCodeJSON
query Query;
queryContinue QueryContinue{JSONName = "query-continue"};
end
record Query
categorymembers Categorymembers[];
end
record Categorymembers
cmcontinue string?;
pageid int?;
ns int?;
title string?;
end
record QueryContinue
categorymembers Categorymembers;
end

View file

@ -0,0 +1,62 @@
-module(rosseta_examples).
-include_lib("xmerl/include/xmerl.hrl").
-export([main/0]).
main() ->
application:start(inets),
Titles = read_titles(empty),
Result = lists:foldl(fun(Title,Acc) -> Acc + calculate_one(Title) end, 0, Titles),
io:format("Total: ~p examples.\n",[Result]),
application:stop(inets).
read_titles(CurrentContinue) ->
URL0 = "http://rosettacode.org/mw/api.php?" ++
"action=query&list=categorymembers&cmtitle=Category:Programming_Tasks" ++
"&cmlimit=500&format=xml",
URL =
case CurrentContinue of
empty -> URL0;
_ -> URL0 ++ "&cmcontinue=" ++ CurrentContinue
end,
{ok,Answer} = httpc:request(URL),
{Document,_} = xmerl_scan:string(lists:last(tuple_to_list(Answer))),
Continue =
[Value || #xmlAttribute{value = Value} <- xmerl_xpath:string("//@cmcontinue", Document)],
Titles =
[Value || #xmlAttribute{value = Value} <- xmerl_xpath:string("//@title", Document)],
case Continue of
[]->
Titles;
[ContValue | _] ->
Titles ++ read_titles(ContValue)
end.
calculate_one(Title0) ->
Title = replace_chars(Title0),
URL = "http://www.rosettacode.org/w/index.php?title=" ++
Title ++ "&action=raw",
case httpc:request(URL) of
{ok,Result} ->
{match,Found} =
re:run(lists:last(tuple_to_list(Result)), "\n=={{header(|)", [global]),
io:format("~ts: ~p examples.\n",[Title0,length(Found)]),
length(Found);
{error,socket_closed_remotely} ->
io:format("Socket closed remotely. Retry.\n"),
calculate_one(Title0)
end.
replace_chars(String) ->
replace_chars(String,[]).
replace_chars([$ | T],Acc) ->
replace_chars(T, [$_| Acc]);
replace_chars([$+| T],Acc) ->
replace_chars(T, lists:reverse("%2B") ++ Acc);
replace_chars([8211| T],Acc) ->
replace_chars(T, lists:reverse("%E2%80%93") ++ Acc);
replace_chars([Other| T],Acc) ->
replace_chars(T, [Other| Acc]);
replace_chars([],Acc) ->
lists:reverse(Acc).

View file

@ -0,0 +1,18 @@
#r "System.Xml.Linq.dll"
let uri1 = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
let uri2 task = sprintf "http://www.rosettacode.org/w/index.php?title=%s&action=raw" task
[|for xml in (System.Xml.Linq.XDocument.Load uri1).Root.Descendants() do
for attrib in xml.Attributes() do
if attrib.Name.LocalName = "title" then
yield async {
let uri = uri2 (attrib.Value.Replace(" ", "_") |> System.Web.HttpUtility.UrlEncode)
use client = new System.Net.WebClient()
let! html = client.AsyncDownloadString(System.Uri uri)
let sols' = html.Split([|"{{header|"|], System.StringSplitOptions.None).Length - 1
lock stdout (fun () -> printfn "%s: %d examples" attrib.Value sols')
return sols' }|]
|> Async.Parallel
|> Async.RunSynchronously
|> fun xs -> printfn "Total: %d examples" (Seq.sum xs)

View file

@ -0,0 +1,42 @@
USING: arrays assocs concurrency.combinators
concurrency.semaphores formatting hashtables http.client io
json.reader kernel math math.parser sequences splitting
urls.encoding ;
IN: rosetta-code.count-examples
CONSTANT: list-url "http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&cmprop=title&format=json"
: titles ( query -- titles )
"query" of "categorymembers" of [ "title" of ] map ;
: continued-url ( query -- url/f )
"query-continue" of "categorymembers" of
[ assoc>query list-url swap "&" glue ] [ f ] if* ;
: (all-programming-titles) ( titles url -- titles' url' )
http-get nip json> [ titles append ] [ continued-url ] bi
[ (all-programming-titles) ] [ f ] if* ;
: all-programming-titles ( -- titles ) { } list-url (all-programming-titles) drop ;
CONSTANT: content-base-url "http://rosettacode.org/mw/index.php?title=&action=raw"
: content-url ( title -- url )
" " "_" replace
"title" associate assoc>query
content-base-url swap "&" glue ;
: occurences ( seq subseq -- n ) split-subseq length 1 - ;
: count-examples ( title -- n )
content-url http-get nip "=={{header|" occurences ;
: print-task ( title n -- ) "%s: %d examples.\n" printf ;
: print-total ( assoc -- ) values sum "Total: %d examples.\n" printf ;
: fetch-counts ( titles -- assoc )
10 <semaphore> [
[ dup count-examples 2array ] with-semaphore
] curry parallel-map ;
: print-counts ( titles -- )
[ [ print-task ] assoc-each nl ] [ print-total ] bi ;
: rosetta-examples ( -- )
all-programming-titles fetch-counts print-counts ;
MAIN: rosetta-examples

View file

@ -0,0 +1,75 @@
package main
import (
"bytes"
"encoding/xml"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
func req(u string, foundCm func(string)) string {
resp, err := http.Get(u)
if err != nil {
fmt.Println(err) // connection or request fail
return ""
}
defer resp.Body.Close()
for p := xml.NewDecoder(resp.Body); ; {
t, err := p.RawToken()
switch s, ok := t.(xml.StartElement); {
case err == io.EOF:
return ""
case err != nil:
fmt.Println(err)
return ""
case !ok:
continue
case s.Name.Local == "cm":
for _, a := range s.Attr {
if a.Name.Local == "title" {
foundCm(a.Value)
}
}
case s.Name.Local == "categorymembers" && len(s.Attr) > 0 &&
s.Attr[0].Name.Local == "cmcontinue":
return url.QueryEscape(s.Attr[0].Value)
}
}
return ""
}
func main() {
taskQuery := "http://rosettacode.org/mw/api.php?action=query" +
"&format=xml&list=categorymembers&cmlimit=500" +
"&cmtitle=Category:Programming_Tasks"
continueAt := req(taskQuery, count)
for continueAt > "" {
continueAt = req(taskQuery+"&cmcontinue="+continueAt, count)
}
fmt.Printf("Total: %d examples.\n", total)
}
var marker = []byte("=={{header|")
var total int
func count(cm string) {
taskFmt := "http://rosettacode.org/mw/index.php?title=%s&action=raw"
taskEsc := url.QueryEscape(strings.Replace(cm, " ", "_", -1))
resp, err := http.Get(fmt.Sprintf(taskFmt, taskEsc))
var page []byte
if err == nil {
page, err = ioutil.ReadAll(resp.Body)
resp.Body.Close()
}
if err != nil {
fmt.Println(err)
return
}
examples := bytes.Count(page, marker)
fmt.Printf("%s: %d\n", cm, examples)
total += examples
}

View file

@ -0,0 +1,37 @@
import Network.Browser
import Network.HTTP
import Network.URI
import Data.List
import Data.Maybe
import Text.XML.Light
import Control.Arrow
justifyR w = foldl ((.return).(++).tail) (replicate w ' ')
showFormatted t n = t ++ ": " ++ justifyR 4 (show n)
getRespons url = do
rsp <- Network.Browser.browse $ do
setAllowRedirects True
setOutHandler $ const (return ()) -- quiet
request $ getRequest url
return $ rspBody $ snd rsp
getNumbOfExampels p = do
let pg = intercalate "_" $ words p
rsp <- getRespons $ "http://www.rosettacode.org/w/index.php?title=" ++ pg ++ "&action=raw"
let taskPage = rsp
countEx = length $ filter (=="=={{header|") $ takeWhile(not.null) $ unfoldr (Just. (take 11 &&& drop 1)) taskPage
return countEx
progTaskExamples = do
rsp <- getRespons "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
let xmls = onlyElems $ parseXML $ rsp
tasks = concatMap (map (fromJust.findAttr (unqual "title")). filterElementsName (== unqual "cm")) xmls
taskxx <- mapM getNumbOfExampels tasks
let ns = taskxx
tot = sum ns
mapM_ putStrLn $ zipWith showFormatted tasks ns
putStrLn $ ("Total: " ++) $ show tot

View file

@ -0,0 +1,10 @@
*Main> progTaskExamples
100 doors: 56
24 game: 11
24 game Player: 9
99 Bottles of Beer: 73
Abstract type: 23
Ackermann Function: 61
Active object: 9
...
Total: 9156

View file

@ -0,0 +1,82 @@
$define RCINDEX "http://rosettacode.org/mw/api.php?format=xml&action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500"
$define RCTASK "http://rosettacode.org/mw/index.php?action=raw&title="
$define RCUA "User-Agent: Unicon Rosetta 0.1"
$define RCXUA "X-Unicon: http://unicon.org/"
$define TASKTOT "* Total Tasks *"
$define TOTTOT "* Total Headers*"
link strings
link hexcvt
procedure main(A) # simple single threaded read all at once implementation
Tasks := table(0)
every task := taskNames() do {
Tasks[TASKTOT] +:= 1 # count tasks
every lang := languages(task) do { # count languages
Tasks[task] +:= 1
Tasks[TOTTOT] +:= 1
}
}
every insert(O := set(),key(Tasks)) # extract & sort keys
O := put(sort(O--set(TOTTOT,TASKTOT)),TASKTOT,TOTTOT) # move totals to end
every write(k := !O, " : ", Tasks[k]," examples.") # report
end
# Generate task names
procedure taskNames()
continue := ""
while \(txt := ReadURL(RCINDEX||continue)) do {
txt ? {
while tab(find("<cm ") & find(s :="title=\"")+*s) do
suspend tab(find("\""))\1
if tab(find("cmcontinue=")) then {
continue := "&"||tab(upto(' \t'))
}
else break
}
}
end
# Generate language headers in a task
procedure languages(task)
static WS
initial WS := ' \t'
page := ReadURL(RCTASK||CleanURI(task))
page ? while (tab(find("\n==")),tab(many(WS))|"",tab(find("{{"))) do {
header := tab(find("=="))
header ? {
while tab(find("{{header|")) do {
suspend 2(="{{header|",tab(find("}}")))\1
}
}
}
end
procedure CleanURI(u) #: clean up a URI
static tr,dxml # xml & http translation
initial {
tr := table()
every c := !string(~(&digits++&letters++'-_.!~*()/\'`')) do
tr[c] := "%"||hexstring(ord(c),2)
every /tr[c := !string(&cset)] := c
tr[" "] := "_" # wiki convention
every push(dxml := [],"&#"||right(ord(c := !"&<>'\""),3,"0")||";",c)
}
dxml[1] := u # insert URI as 1st arg
u := replacem!dxml # de-xml it
every (c := "") ||:= tr[!u] # reencode everything
c := replace(c,"%3E","'") # Hack to put single quotes back in
c := replace(c,"%26quot%3B","\"") # Hack to put double quotes back in
return c
end
procedure ReadURL(url) #: read URL into string
page := open(url,"m",RCUA,RCXUA) | stop("Unable to open ",url)
text := ""
if page["Status-Code"] < 300 then while text ||:= reads(page,-1)
else write(&errout,image(url),": ",
page["Status-Code"]," ",page["Reason-Phrase"])
close(page)
return text
end

View file

@ -0,0 +1,20 @@
require 'web/gethttp'
getAllTaskSolnCounts=: monad define
tasks=. getCategoryMembers 'Programming_Tasks'
counts=. getTaskSolnCounts &> tasks
tasks;counts
)
getTaskSolnCounts=: monad define
makeuri=. 'http://www.rosettacode.org/w/index.php?title=' , ,&'&action=raw'
wikidata=. gethttp makeuri urlencode y
([: +/ '{{header|'&E.) wikidata
)
formatSolnCounts=: monad define
'tasks counts'=. y
tasks=. tasks , &.>':'
res=. ;:^:_1 tasks ,. (8!:0 counts) ,. <'examples.'
res , 'Total examples: ' , ": +/counts
)

View file

@ -0,0 +1,6 @@
formatSolnCounts getAllTaskSolnCounts ''
100 doors: 61 examples.
24 game: 15 examples.
24 game Player: 11 examples.
99 Bottles of Beer: 76 examples.
...

View file

@ -0,0 +1,69 @@
import java.util.ArrayList;
import ScreenScrape;
public class CountProgramExamples {
private static final String baseURL = "http://rosettacode.org/wiki/";
private static final String rootURL = "http://www.rosettacode.org/w/"
+ "api.php?action=query&list=categorymembers"
+ "&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
private static final String taskBegin = "title=\"";
private static final String taskEnd = "\"";
private static final String exmplBegin = "<span class=\"tocnumber\">";
private static final String exmplEnd = "</span>";
private static final String editBegin = "<span class=\"editsection\">";
/**
* @param args
*/
public static void main(String[] args) {
int exTotal = 0;
try {
// Get root query results
ArrayList<String> tasks = new ArrayList<String>();
ScreenScrape ss = new ScreenScrape();
String rootPage = ss.read(rootURL);
while (rootPage.contains(taskBegin)) {
rootPage = rootPage.substring(rootPage.indexOf(taskBegin)
+ taskBegin.length());
String title = rootPage.substring(0, rootPage.indexOf(taskEnd));
if (!title.contains("Category:")) {
tasks.add(title);
}
rootPage = rootPage.substring(rootPage.indexOf(taskEnd));
}
// Loop through each task and print count
for (String task : tasks) {
String title = task.replaceAll("&#039;", "'");
String taskPage = ss.read(baseURL + title.replaceAll(" ", "_"));
int exSubTot;
if (taskPage.contains(exmplBegin)) {
int startPos = taskPage.lastIndexOf(exmplBegin)
+ exmplBegin.length();
String countStr = taskPage.substring(startPos,
taskPage.indexOf(exmplEnd, startPos));
exSubTot = Integer
.parseInt(countStr.contains(".") ? countStr
.substring(0, countStr.indexOf("."))
: countStr);
} else {
exSubTot = 0;
while (taskPage.contains(editBegin)) {
taskPage = taskPage.substring(taskPage
.indexOf(editBegin) + editBegin.length());
exSubTot++;
}
}
exTotal += exSubTot;
System.out.println(title + ": " + exSubTot + " examples.");
}
// Print total
System.out.println("\nTotal: " + exTotal + " examples.");
} catch (Exception e) {
System.out.println(title);
System.out.println(startPos + ":"
+ taskPage.indexOf(exmplEnd, startPos));
System.out.println(taskPage);
e.printStackTrace(System.out);
}
}
}

View file

@ -0,0 +1,25 @@
#!/bin/bash
# Produce lines of the form: URI TITLE
function titles {
local uri="http://www.rosettacode.org/mw/api.php?action=query&list=categorymembers"
uri+="&cmtitle=Category:Programming_Tasks&cmlimit=5000&format=json"
curl -Ss "$uri" |
jq -r '.query.categorymembers[] | .title | "\(@uri) \(.)"'
}
# Syntax: count URI
function count {
local uri="$1"
curl -Ss "http://rosettacode.org/mw/index.php?title=${uri}&action=raw" |
jq -R -n 'reduce (inputs|select(test("=={{header\\|"))) as $x(0; .+1)'
}
local n=0 i
while read uri title
do
i=$(count "$uri")
echo "$title: $i examples."
n=$((n + i))
done < <(titles)
echo Total: $n examples.

View file

@ -0,0 +1,45 @@
using HTTP, JSON, Dates
rosorg = "http://rosettacode.org"
qURI = "/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=json"
qdURI = "/w/api.php?action=query&list=categorymembers&cmtitle=Category:Draft_Programming_Tasks&cmlimit=500&format=json"
sqURI = rosorg * "/wiki/"
topages(js, v) = for d in js["query"]["categorymembers"] push!(v, sqURI * replace(d["title"], " " => "_")) end
function getpages(uri)
wikipages = Vector{String}()
response = HTTP.request("GET", rosorg * uri)
if response.status == 200
fromjson = JSON.parse(String(response.body))
topages(fromjson, wikipages)
while haskey(fromjson, "continue")
cmcont, cont = fromjson["continue"]["cmcontinue"], fromjson["continue"]["continue"]
response = HTTP.request("GET", rosorg * uri * "&cmcontinue=$cmcont&continue=$cont")
fromjson = JSON.parse(String(response.body))
topages(fromjson, wikipages)
end
end
wikipages
end
function processtaskpages(wpages, verbose=false)
totalexamples = 0
for pag in wpages
response = HTTP.request("GET", pag)
if response.status == 200
n = length(collect(eachmatch(r"span class=\"mw-headline\"", String(response.body))))
if verbose
println("Wiki page $pag => $n examples.")
end
totalexamples += n
end
end
println("Total of $totalexamples on $(length(wpages)) task pages.\n")
end
println("Programming examples at $(DateTime(now())):")
qURI |> getpages |> processtaskpages
println("Draft programming tasks:")
qdURI |> getpages |> processtaskpages

View file

@ -0,0 +1,17 @@
local(root = json_deserialize(curl('http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=10&format=json')->result))
local(tasks = array, title = string, urltitle = string, thiscount = 0, totalex = 0)
with i in #root->find('query')->find('categorymembers') do => {^
#thiscount = 0
#title = #i->find('title')
#urltitle = #i->find('title')
#urltitle->replace(' ','_')
#title+': '
local(src = curl('http://rosettacode.org/mw/index.php?title='+#urltitle->asBytes->encodeurl+'&action=raw')->result->asString)
#thiscount = (#src->split('=={{header|'))->size - 1
#thiscount < 0 ? #thiscount = 0
#thiscount + ' examples.'
#totalex += #thiscount
'\r'
^}
'Total: '+#totalex+' examples.'

View file

@ -0,0 +1,17 @@
on mouseUp
put empty into fld "taskurls"
put URL "http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=10&format=xml" into apixml
put revXMLCreateTree(apixml,true,true,false) into pDocID
put "/api/query/categorymembers/cm" into pXPathExpression
repeat for each line xmlnode in revXMLEvaluateXPath(pDocID, pXPathExpression)
put revXMLAttribute(pDocID,xmlnode,"title") into pgTitle
put revXMLAttribute(pDocID,xmlnode,"pageid") into pageId
put "http://www.rosettacode.org/w/index.php?title=" & urlEncode(pgTitle) & "&action=raw" into taskURL
put URL taskURL into taskPage
filter lines of taskPage with "=={{header|*"
put the number of lines of taskPage into taskTotal
put pgTitle & comma & taskTotal & cr after fld "tasks"
add taskTotal to allTaskTotal
end repeat
put "Total" & comma & allTaskTotal after fld "tasks"
end mouseUp

View file

@ -0,0 +1,18 @@
function c = count_examples(url)
c = 0;
[s, success] = urlread (url);
if ~success, return; end;
c = length(strfind(s,'<h2><span class='));
end;
% script
s = urlread ('http://rosettacode.org/wiki/Category:Programming_Tasks');
pat = '<li><a href="/wiki/';
ix = strfind(s,pat)+length(pat)-6;
for k = 1:length(ix);
% look through all tasks
e = find(s(ix(k):end)==34,1)-2;
t = s(ix(k)+[0:e]); % task
c = count_examples(['http://rosettacode.org',t]);
printf('Task "%s" has %i examples.\n',t(7:end), c);
end;

View file

@ -0,0 +1,37 @@
ConvertUTF8 := proc( str )
local i, tempstring, uniindex;
try
tempstring := str;
uniindex := [StringTools:-SearchAll("\u",str)];
if uniindex <> [] then
for i in uniindex do
tempstring := StringTools:-Substitute(tempstring, str[i..i+5], UTF8:-unicode(str[i+2..i+5]));
end do:
end if;
return tempstring;
catch:
return str;
end try;
end proc:
print_examples := proc(lst)
local task, count, url, headers, item;
for task in lst do
count := 0:
url := cat("http://www.rosettacode.org/mw/index.php?title=", StringTools:-Encode(StringTools:-SubstituteAll(task["title"], " ", "_"), 'percent'), "&action=raw"):
headers := [StringTools:-SearchAll("=={{header|",URL:-Get(url))]:
for item in headers do
count++:
end do:
printf("%s has %d examples\n",ConvertUTF8(task["title"]), count);
end do:
end proc:
x := JSON:-ParseFile("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=20&format=json"):
print_examples(x["query"]["categorymembers"]);
while(assigned(x["continue"]["cmcontinue"])) do
continue := x["continue"]["cmcontinue"]:
more_tasks:= cat("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=20&format=json", "&continue=", x["continue"]["continue"], "&cmcontinue=", x["continue"]["cmcontinue"]):
x := JSON:-ParseFile(more_tasks):
print_examples(x["query"]["categorymembers"]);
end do:

View file

@ -0,0 +1,6 @@
TaskList = Flatten[
Import["http://rosettacode.org/wiki/Category:Programming_Tasks", "Data"][[1, 1]]];
Print["Task \"", StringReplace[#, "_" -> " "], "\" has ",
Length@Select[Import["http://rosettacode.org/wiki/" <> #, "Data"][[1,2]],
StringFreeQ[#, __ ~~ "Programming Task" | __ ~~ "Omit"]& ], " example(s)"]&
~Map~ StringReplace[TaskList, " " -> "_"]

View file

@ -0,0 +1,23 @@
import httpclient, strutils, xmltree, xmlparser, cgi
proc count(s, sub: string): int =
var i = 0
while true:
i = s.find(sub, i)
if i < 0: break
inc i
inc result
const
mainSite = "http://www.rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
subSite = "http://www.rosettacode.org/mw/index.php?title=$#&action=raw"
var client = newHttpClient()
var sum = 0
for node in client.getContent(mainSite).parseXml().findAll("cm"):
let t = node.attr("title").replace(" ", "_")
let c = client.getContent(subSite % encodeUrl(t)).toLower().count("{{header|")
echo t.replace("_", " "), ": ", c, " examples."
inc sum, c
echo "\nTotal: ", sum, " examples."

View file

@ -0,0 +1,62 @@
open Http_client.Convenience
let repl_quote s =
let reg = Str.regexp_string "&#039;" in
(Str.global_replace reg "%27" s)
let repl_space s =
let s = String.copy s in
for i = 0 to pred(String.length s) do
if s.[i] = ' ' then s.[i] <- '_'
done;
(s)
(* or in OCaml 4.00+:
let repl_space = String.map (fun c -> if c = ' ' then '_' else c)
*)
let count_ex s =
let pat = Str.regexp_string "=={{header|" in
let rec aux n p =
try
let p = Str.search_forward pat s p in
aux (n+1) (p+1)
with Not_found -> (n)
in
aux 0 0
let get_child child xml =
let child =
List.find
(function Xml.Element (tag,_,_) when tag = child -> true | _ -> false) xml
in
Xml.children child
let () =
let url = "http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&\
cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml" in
let xml = Xml.parse_string (http_get url) in
let total = ref 0 in
at_exit (fun () -> Printf.printf "\n Total: %d\n" !total);
let f = function
| Xml.Element ("cm", attrs, _) ->
(try
let _title = List.assoc "title" attrs in
let title = repl_quote (repl_space _title) in
let url = "http://www.rosettacode.org/w/index.php?title="^ title ^"&action=raw" in
let n = count_ex (http_get url) in
Printf.printf "%s: %d\n%!" _title n;
total := n + !total;
with Http_client.Http_error (404, _) -> ())
| _ -> ()
in
match xml with
| Xml.Element ("api", _, ch) ->
let query = get_child "query" ch in
let catmb = get_child "categorymembers" query in
List.iter f catmb
| _ -> ()

View file

@ -0,0 +1,38 @@
use HTTP;
use XML;
class RosettaCount {
function : Main(args : String[]) ~ Nil {
taks_xml := HttpGet("http://rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml");
parser := XmlParser->New(taks_xml);
if(parser->Parse()) {
task_names := parser->FindElements("/api/query/categorymembers/cm");
if(task_names <> Nil) {
each(i : task_names) {
task_name := task_names->Get(i)->As(XmlElement)->GetAttribute("title")->GetValue();
task_url := "http://rosettacode.org/mw/index.php?title=";
task_url->Append(task_name);
task_url->Append("&action=raw");
task := HttpGet(task_url);
counts := task->FindAll("=={{header|");
if(counts->Size() > 0) {
IO.Console->Print(UrlUtility->Decode(task_name))->Print(": ")->PrintLine(counts->Size());
};
};
};
};
}
function : HttpGet(url : String) ~ String {
xml := "";
client := HttpClient->New();
lines := client->Get(url);
each(i : lines) {
xml->Append(lines->Get(i)->As(String));
};
return xml;
}
}

View file

@ -0,0 +1,85 @@
declare
[HTTPClient] = {Module.link ['x-ozlib://mesaros/net/HTTPClient.ozf']}
[XMLParser] = {Module.link ['x-oz://system/xml/Parser.ozf']}
[StringX] = {Module.link ['x-oz://system/String.ozf']}
[Regex] = {Module.link ['x-oz://contrib/regex']}
AllTasksUrl = "http://rosettacode.org/mw/api.php?action=query&list="#
"categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
proc {Main}
AllTasks = {Parse {GetPage AllTasksUrl}}
TaskTitles = {GetTitles AllTasks}
Total = {NewCell 0}
in
for Task in TaskTitles do
TaskPage = {GetPage {TaskUrl Task}}
RE = {Regex.compile "{{header\\|" [extended newline icase]}
NumMatches = {Length {Regex.allMatches RE TaskPage}}
in
{System.showInfo Task#": "#NumMatches#" examples."}
Total := @Total + NumMatches
end
{System.showInfo "Total: "#@Total#" examples."}
end
fun {TaskUrl Task}
"http://rosettacode.org/mw/index.php?"#
"title="#{PercentEncode {StringX.replace Task " " "_"}}#
"&action=raw"
end
%% GetPage
local
Client = {New HTTPClient.urlGET init(inPrms(toFile:false toStrm:true) _)}
in
fun {GetPage RawUrl}
Url = {VirtualString.toString RawUrl}
OutParams
HttpResponseParams
in
{Client getService(Url ?OutParams ?HttpResponseParams)}
OutParams.sOut
end
end
%% Parse
local
Parser = {New XMLParser.parser init}
in
fun {Parse Xs} {Parser parseVS(Xs $)} end
end
fun {GetTitles Doc}
CMs = Doc.2.1.children.1.children.1.children
fun {Attributes element(attributes:As ...)} As end
fun {IsTitle attribute(name:N ...)} N == title end
in
{Map {Filter {Flatten {Map CMs Attributes}} IsTitle}
fun {$ A} {Atom.toString A.value} end}
end
fun {PercentEncode Xs}
case Xs of nil then nil
[] X|Xr then
if {Char.isDigit X} orelse {Member X [&- &_ &. &~]}
orelse X >= &a andthen X =< &z
orelse X >= &z andthen X =< &Z then
X|{PercentEncode Xr}
else
{Append &%|{ToHex2 X} {PercentEncode Xr}}
end
end
end
fun {ToHex2 X}
[{ToHex1 X div 16} {ToHex1 X mod 16}]
end
fun {ToHex1 X}
if X >= 0 andthen X =< 9 then &0 + X
elseif X >= 10 andthen X =< 15 then &A + X - 10
end
end
in
{Main}

View file

@ -0,0 +1,12 @@
use HTTP::Tiny;
my $site = "http://rosettacode.org";
my $list_url = "/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
my $response = HTTP::Tiny->new->get("$site$list_url");
for ($response->{content} =~ /cm.*?title="(.*?)"/g) {
(my $slug = $_) =~ tr/ /_/;
my $response = HTTP::Tiny->new->get("$site/wiki/$slug");
my $count = () = $response->{content} =~ /toclevel-1/g;
print "$_: $count examples\n";
}

View file

@ -0,0 +1,11 @@
use Mojo::UserAgent;
my $site = "http://rosettacode.org";
my $list_url = "/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml";
my $ua = Mojo::UserAgent->new;
$ua->get("$site$list_url")->res->dom->find('cm')->each(sub {
(my $slug = $_->{title}) =~ tr/ /_/;
my $count = $ua->get("$site/wiki/$slug")->res->dom->find("#toc .toclevel-1")->size;
say "$_->{title}: $count examples";
});

View file

@ -0,0 +1,151 @@
(notonline)-->
<span style="color: #000080;font-style:italic;">--
-- demo\rosetta\rosettacode_cache.e
-- ================================
--
-- Common routines for handling rc_cache etc.
--</span>
<span style="color: #008080;">without</span> <span style="color: #008080;">js</span> <span style="color: #000080;font-style:italic;">-- (libcurl, file i/o, peek, progress..)</span>
<span style="color: #008080;">include</span> <span style="color: #000000;">builtins</span><span style="color: #0000FF;">\</span><span style="color: #004080;">timedate</span><span style="color: #0000FF;">.</span><span style="color: #000000;">e</span>
<span style="color: #008080;">constant</span> <span style="color: #000000;">day</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">timedelta</span><span style="color: #0000FF;">(</span><span style="color: #000000;">days</span><span style="color: #0000FF;">:=</span><span style="color: #000000;">1</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">refresh_cache</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">21</span><span style="color: #0000FF;">*</span><span style="color: #000000;">day</span> <span style="color: #000080;font-style:italic;">-- 0 for always [NB refresh_cache += timedelta(days:=1) below]</span>
<span style="color: #008080;">function</span> <span style="color: #000000;">days</span><span style="color: #0000FF;">(</span><span style="color: #004080;">atom</span> <span style="color: #000000;">delta</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">d</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">ceil</span><span style="color: #0000FF;">(</span><span style="color: #000000;">delta</span><span style="color: #0000FF;">/</span><span style="color: #000000;">day</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">return</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%d day%s"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">d</span><span style="color: #0000FF;">,</span><span style="color: #008080;">iff</span><span style="color: #0000FF;">(</span><span style="color: #000000;">d</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span><span style="color: #0000FF;">?</span><span style="color: #008000;">""</span><span style="color: #0000FF;">:</span><span style="color: #008000;">"s"</span><span style="color: #0000FF;">)})</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
<span style="color: #008080;">constant</span> <span style="color: #0000FF;">{</span><span style="color: #000000;">hex</span><span style="color: #0000FF;">,</span><span style="color: #000000;">ascii</span><span style="color: #0000FF;">}</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">columnize</span><span style="color: #0000FF;">({{</span><span style="color: #008000;">"%22"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">`"`</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%27"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"'"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%2A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"*"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%2B"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"+"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%3A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">":"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%5E"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">`^`</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%E2%80%93"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"-"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%E2%80%99"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"'"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%C3%A8"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"e"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%C3%A9"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"e"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%C3%B6"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"o"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"%C5%91"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"o"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"&quot;"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">`"`</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"&#039;"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"'"</span><span style="color: #0000FF;">},</span>
<span style="color: #0000FF;">{</span><span style="color: #008000;">"_"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">" "</span><span style="color: #0000FF;">}})</span>
<span style="color: #008080;">global</span> <span style="color: #008080;">function</span> <span style="color: #000000;">html_clean</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">return</span> <span style="color: #7060A8;">substitute_all</span><span style="color: #0000FF;">(</span><span style="color: #000000;">s</span><span style="color: #0000FF;">,</span><span style="color: #000000;">hex</span><span style="color: #0000FF;">,</span><span style="color: #000000;">ascii</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
<span style="color: #008080;">include</span> <span style="color: #000000;">builtins</span><span style="color: #0000FF;">\</span><span style="color: #000000;">libcurl</span><span style="color: #0000FF;">.</span><span style="color: #000000;">e</span>
<span style="color: #004080;">atom</span> <span style="color: #000000;">curl</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">NULL</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">pErrorBuffer</span>
<span style="color: #008080;">function</span> <span style="color: #000000;">write_callback</span><span style="color: #0000FF;">(</span><span style="color: #004080;">atom</span> <span style="color: #000000;">pData</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">integer</span> <span style="color: #000000;">size</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">nmemb</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">fn</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">bytes_written</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">size</span> <span style="color: #0000FF;">*</span> <span style="color: #000000;">nmemb</span>
<span style="color: #7060A8;">puts</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fn</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">peek</span><span style="color: #0000FF;">({</span><span style="color: #000000;">pData</span><span style="color: #0000FF;">,</span><span style="color: #000000;">bytes_written</span><span style="color: #0000FF;">}))</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">bytes_written</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
<span style="color: #008080;">constant</span> <span style="color: #000000;">write_cb</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">call_back</span><span style="color: #0000FF;">({</span><span style="color: #008000;">'+'</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">write_callback</span><span style="color: #0000FF;">})</span>
<span style="color: #008080;">global</span> <span style="color: #004080;">string</span> <span style="color: #000000;">wastitle</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">""</span> <span style="color: #000080;font-style:italic;">-- don't clobber "NEED EDITING"/Downloading messages</span>
<span style="color: #008080;">global</span> <span style="color: #004080;">integer</span> <span style="color: #000000;">show_title</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">progress</span>
<span style="color: #008080;">global</span> <span style="color: #008080;">function</span> <span style="color: #000000;">open_download</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">filename</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">url</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">integer</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">n</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">object</span> <span style="color: #000000;">text</span>
<span style="color: #004080;">bool</span> <span style="color: #000000;">refetch</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">false</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"not found"</span>
<span style="color: #000000;">filename</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">join_path</span><span style="color: #0000FF;">({</span><span style="color: #008000;">"rc_cache"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">})</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">file_exists</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
<span style="color: #000080;font-style:italic;">-- use existing file if &lt;= refresh_cache days old</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">last_mod</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">get_file_date</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">atom</span> <span style="color: #000000;">delta</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">timedate_diff</span><span style="color: #0000FF;">(</span><span style="color: #000000;">last_mod</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">date</span><span style="color: #0000FF;">())</span>
<span style="color: #000000;">refetch</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">true</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">delta</span><span style="color: #0000FF;">></span><span style="color: #000000;">refresh_cache</span>
<span style="color: #008080;">and</span> <span style="color: #008080;">not</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">".hist"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">days</span><span style="color: #0000FF;">(</span><span style="color: #000000;">delta</span><span style="color: #0000FF;">)</span> <span style="color: #0000FF;">&</span> <span style="color: #008000;">" &gt; "</span> <span style="color: #0000FF;">&</span> <span style="color: #000000;">days</span><span style="color: #0000FF;">(</span><span style="color: #000000;">refresh_cache</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">elsif</span> <span style="color: #7060A8;">get_file_size</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"filesize of 0"</span>
<span style="color: #008080;">else</span>
<span style="color: #000000;">text</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">trim</span><span style="color: #0000FF;">(</span><span style="color: #7060A8;">get_text</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">))</span>
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #004080;">sequence</span><span style="color: #0000FF;">(</span><span style="color: #000000;">text</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"no text"</span>
<span style="color: #008080;">elsif</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">text</span><span style="color: #0000FF;">)<</span><span style="color: #000000;">10</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">why</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"&lt;10 bytes"</span>
<span style="color: #008080;">else</span>
<span style="color: #000000;">refetch</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">false</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">else</span>
<span style="color: #000000;">refetch</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">true</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">directory</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">get_file_path</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">get_file_type</span><span style="color: #0000FF;">(</span><span style="color: #000000;">directory</span><span style="color: #0000FF;">)!=</span><span style="color: #004600;">FILETYPE_DIRECTORY</span> <span style="color: #008080;">then</span>
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #000000;">create_directory</span><span style="color: #0000FF;">(</span><span style="color: #000000;">directory</span><span style="color: #0000FF;">,</span><span style="color: #000000;">make_parent</span><span style="color: #0000FF;">:=</span><span style="color: #004600;">true</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
<span style="color: #7060A8;">crash</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"cannot create %s directory"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">directory</span><span style="color: #0000FF;">})</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">refetch</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">wastitle</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"x"</span> <span style="color: #000080;font-style:italic;">-- don't clobber</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">nofn</span> <span style="color: #0000FF;">=</span> <span style="color: #008080;">iff</span><span style="color: #0000FF;">(</span><span style="color: #000000;">n</span><span style="color: #0000FF;">?</span><span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"(%d/%d, %.1f%%) "</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">i</span><span style="color: #0000FF;">,</span><span style="color: #000000;">n</span><span style="color: #0000FF;">,</span><span style="color: #000000;">i</span><span style="color: #0000FF;">/</span><span style="color: #000000;">n</span><span style="color: #0000FF;">*</span><span style="color: #000000;">100</span><span style="color: #0000FF;">}):</span><span style="color: #008000;">""</span><span style="color: #0000FF;">),</span>
<span style="color: #000000;">title</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Downloading %s%s (%s)..."</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">nofn</span><span style="color: #0000FF;">,</span><span style="color: #000000;">html_clean</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">),</span><span style="color: #000000;">why</span><span style="color: #0000FF;">})</span>
<span style="color: #000000;">show_title</span><span style="color: #0000FF;">(</span><span style="color: #000000;">title</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">curl</span><span style="color: #0000FF;">=</span><span style="color: #004600;">NULL</span> <span style="color: #008080;">then</span>
<span style="color: #7060A8;">curl_global_init</span><span style="color: #0000FF;">()</span>
<span style="color: #000000;">curl</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">curl_easy_init</span><span style="color: #0000FF;">()</span>
<span style="color: #000000;">pErrorBuffer</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">allocate</span><span style="color: #0000FF;">(</span><span style="color: #000000;">CURL_ERROR_SIZE</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">curl_easy_setopt</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">CURLOPT_ERRORBUFFER</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">pErrorBuffer</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">curl_easy_setopt</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">,</span> <span style="color: #004600;">CURLOPT_WRITEFUNCTION</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">write_cb</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #000000;">url</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">substitute</span><span style="color: #0000FF;">(</span><span style="color: #000000;">url</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%3A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">":"</span><span style="color: #0000FF;">)</span>
<span style="color: #000000;">url</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">substitute</span><span style="color: #0000FF;">(</span><span style="color: #000000;">url</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%2A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"*"</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">curl_easy_setopt</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">,</span> <span style="color: #004600;">CURLOPT_URL</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">url</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">fn</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">open</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"wb"</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">assert</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fn</span><span style="color: #0000FF;">!=-</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"cannot open "</span><span style="color: #0000FF;">&</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">curl_easy_setopt</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">CURLOPT_WRITEDATA</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">fn</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">while</span> <span style="color: #004600;">true</span> <span style="color: #008080;">do</span>
<span style="color: #004080;">CURLcode</span> <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">curl_easy_perform</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">res</span><span style="color: #0000FF;">=</span><span style="color: #004600;">CURLE_OK</span> <span style="color: #008080;">then</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">error</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%d"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">res</span><span style="color: #0000FF;">=</span><span style="color: #000000;">CURLE_COULDNT_RESOLVE_HOST</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">error</span> <span style="color: #0000FF;">&=</span> <span style="color: #008000;">" [CURLE_COULDNT_RESOLVE_HOST]"</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Error %s downloading file, retry?(Y/N):"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">error</span><span style="color: #0000FF;">})</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">lower</span><span style="color: #0000FF;">(</span><span style="color: #7060A8;">wait_key</span><span style="color: #0000FF;">())!=</span><span style="color: #008000;">'y'</span> <span style="color: #008080;">then</span> <span style="color: #7060A8;">abort</span><span style="color: #0000FF;">(</span><span style="color: #000000;">0</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #7060A8;">printf</span><span style="color: #0000FF;">(</span><span style="color: #000000;">1</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"Y\n"</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">while</span>
<span style="color: #7060A8;">close</span><span style="color: #0000FF;">(</span><span style="color: #000000;">fn</span><span style="color: #0000FF;">)</span>
<span style="color: #000000;">refresh_cache</span> <span style="color: #0000FF;">+=</span> <span style="color: #7060A8;">timedelta</span><span style="color: #0000FF;">(</span><span style="color: #000000;">days</span><span style="color: #0000FF;">:=</span><span style="color: #000000;">1</span><span style="color: #0000FF;">)</span> <span style="color: #000080;font-style:italic;">-- did I mention it is slow?</span>
<span style="color: #000000;">text</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">get_text</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">text</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
<span style="color: #008080;">global</span> <span style="color: #008080;">function</span> <span style="color: #000000;">open_category</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">filename</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">integer</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">n</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">open_download</span><span style="color: #0000FF;">(</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">&</span><span style="color: #008000;">".htm"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"http://rosettacode.org/wiki/Category:"</span><span style="color: #0000FF;">&</span><span style="color: #000000;">filename</span><span style="color: #0000FF;">,</span><span style="color: #000000;">i</span><span style="color: #0000FF;">,</span><span style="color: #000000;">n</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
<span style="color: #008080;">global</span> <span style="color: #008080;">function</span> <span style="color: #000000;">dewiki</span><span style="color: #0000FF;">(</span><span style="color: #004080;">string</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">,</span> <span style="color: #004080;">sequence</span> <span style="color: #000000;">exclude</span><span style="color: #0000FF;">={})</span>
<span style="color: #000080;font-style:italic;">-- extract tasks from eg `&lt;li&gt;&lt;a href="/wiki/100_doors"`</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">1</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">finish</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`&lt;div class="printfooter"&gt;`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">s</span><span style="color: #0000FF;">)</span>
<span style="color: #000000;">s</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">[</span><span style="color: #000000;">1</span><span style="color: #0000FF;">..</span><span style="color: #000000;">finish</span><span style="color: #0000FF;">-</span><span style="color: #000000;">1</span><span style="color: #0000FF;">]</span>
<span style="color: #008080;">while</span> <span style="color: #004600;">true</span> <span style="color: #008080;">do</span>
<span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`&lt;li&gt;&lt;a href="/wiki/`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">s</span><span style="color: #0000FF;">,</span><span style="color: #000000;">start</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">start</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #000000;">start</span> <span style="color: #0000FF;">+=</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`&lt;li&gt;&lt;a href="/wiki/`</span><span style="color: #0000FF;">)</span>
<span style="color: #000000;">finish</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">find</span><span style="color: #0000FF;">(</span><span style="color: #008000;">'"'</span><span style="color: #0000FF;">,</span><span style="color: #000000;">s</span><span style="color: #0000FF;">,</span><span style="color: #000000;">start</span><span style="color: #0000FF;">)</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">task</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">s</span><span style="color: #0000FF;">[</span><span style="color: #000000;">start</span><span style="color: #0000FF;">..</span><span style="color: #000000;">finish</span><span style="color: #0000FF;">-</span><span style="color: #000000;">1</span><span style="color: #0000FF;">]</span>
<span style="color: #000000;">task</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">substitute_all</span><span style="color: #0000FF;">(</span><span style="color: #000000;">task</span><span style="color: #0000FF;">,{</span><span style="color: #008000;">"*"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">":"</span><span style="color: #0000FF;">},{</span><span style="color: #008000;">"%2A"</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"%3A"</span><span style="color: #0000FF;">})</span>
<span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">append</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">,</span><span style="color: #000000;">task</span><span style="color: #0000FF;">)</span>
<span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">finish</span><span style="color: #0000FF;">+</span><span style="color: #000000;">1</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">while</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">tasks</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
<span style="color: #008080;">global</span> <span style="color: #008080;">procedure</span> <span style="color: #000000;">curl_cleanup</span><span style="color: #0000FF;">()</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">curl</span><span style="color: #0000FF;">!=</span><span style="color: #004600;">NULL</span> <span style="color: #008080;">then</span>
<span style="color: #7060A8;">curl_easy_cleanup</span><span style="color: #0000FF;">(</span><span style="color: #000000;">curl</span><span style="color: #0000FF;">)</span>
<span style="color: #7060A8;">free</span><span style="color: #0000FF;">(</span><span style="color: #000000;">pErrorBuffer</span><span style="color: #0000FF;">)</span>
<span style="color: #000000;">curl</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">NULL</span>
<span style="color: #000000;">pErrorBuffer</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">NULL</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">procedure</span>
<!--

View file

@ -0,0 +1,106 @@
(notonline)-->
<span style="color: #000080;font-style:italic;">--
-- demo\rosetta\Count_examples.exw
-- ===============================
--
-- (This uses a few '&' instead of/as well as 'a', fairly obviously for everyone's sanity..)
-- Counts no of "<nowiki>{{</nowiki>he&der|" (nb not "==<nowiki>{{</nowiki>he&der|") via web api (but gets tasks via scraping).
-- Since downloading all the pages can be very slow, this uses a cache.
-- Limiting (notdone) by "Phix" fairly obviously speeds it up tenfold :-)
--</span>
<span style="color: #008080;">without</span> <span style="color: #008080;">js</span> <span style="color: #000080;font-style:italic;">-- (fairly obviously this will never ever run in a browser!)</span>
<span style="color: #008080;">constant</span> <span style="color: #000000;">include_drafts</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">true</span><span style="color: #0000FF;">,</span>
<span style="color: #000000;">sort_by_count</span> <span style="color: #0000FF;">=</span> <span style="color: #004600;">false</span><span style="color: #0000FF;">,</span>
<span style="color: #000080;font-style:italic;">-- notlang = "Phix" -- or "" (ie a zero length string) for all</span>
<span style="color: #000000;">notlang</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">""</span>
<span style="color: #008080;">include</span> <span style="color: #000000;">rosettacode_cache</span><span style="color: #0000FF;">.</span><span style="color: #000000;">e</span>
<span style="color: #008080;">function</span> <span style="color: #000000;">count_tasks</span><span style="color: #0000FF;">()</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">get_file_type</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"rc_cache"</span><span style="color: #0000FF;">)!=</span><span style="color: #004600;">FILETYPE_DIRECTORY</span> <span style="color: #008080;">then</span>
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #000000;">create_directory</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"rc_cache"</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
<span style="color: #7060A8;">crash</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"cannot create rc_cache directory"</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #000080;font-style:italic;">-- note this lot use web scraping (as cribbed from a similar task) ...</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">dewiki</span><span style="color: #0000FF;">(</span><span style="color: #000000;">open_category</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Programming_Tasks"</span><span style="color: #0000FF;">))</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">include_drafts</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">tasks</span> <span style="color: #0000FF;">&=</span> <span style="color: #000000;">dewiki</span><span style="color: #0000FF;">(</span><span style="color: #000000;">open_category</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Draft_Programming_Tasks"</span><span style="color: #0000FF;">))</span>
<span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sort</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">notlang</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
<span style="color: #000080;font-style:italic;">-- filter already done in specified language</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">langurl</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">"http://rosettacode.org/wiki/Category:"</span><span style="color: #0000FF;">&</span><span style="color: #000000;">notlang</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">done</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">dewiki</span><span style="color: #0000FF;">(</span><span style="color: #000000;">open_download</span><span style="color: #0000FF;">(</span><span style="color: #000000;">notlang</span><span style="color: #0000FF;">&</span><span style="color: #008000;">".htm"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">langurl</span><span style="color: #0000FF;">))</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">k</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">do</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">ti</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">d</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">find</span><span style="color: #0000FF;">(</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">,</span><span style="color: #000000;">done</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #000000;">d</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">k</span> <span style="color: #0000FF;">+=</span> <span style="color: #000000;">1</span>
<span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">k</span><span style="color: #0000FF;">]</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">ti</span>
<span style="color: #008080;">else</span>
<span style="color: #000000;">done</span><span style="color: #0000FF;">[</span><span style="color: #000000;">d</span><span style="color: #0000FF;">..</span><span style="color: #000000;">d</span><span style="color: #0000FF;">]</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #000000;">tasks</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">1</span><span style="color: #0000FF;">..</span><span style="color: #000000;">k</span><span style="color: #0000FF;">]</span>
<span style="color: #000000;">done</span> <span style="color: #0000FF;">=</span> <span style="color: #0000FF;">{}</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%d tasks found\n"</span><span style="color: #0000FF;">,{</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)})</span>
<span style="color: #000080;font-style:italic;">-- ... whereas the individual tasks use the web api instead (3x smaller/faster)</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">total_count</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">task_counts</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">repeat</span><span style="color: #0000FF;">(</span><span style="color: #000000;">0</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">))</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">do</span>
<span style="color: #004080;">string</span> <span style="color: #000000;">ti</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">],</span>
<span style="color: #000000;">url</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">sprintf</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"http://rosettacode.org/mw/index.php?title=%s&action=raw"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">}),</span>
<span style="color: #000000;">contents</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">open_download</span><span style="color: #0000FF;">(</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">&</span><span style="color: #008000;">".raw"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">url</span><span style="color: #0000FF;">),</span>
<span style="color: #000000;">prev</span> <span style="color: #0000FF;">=</span> <span style="color: #008000;">""</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">curr</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">count</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">0</span><span style="color: #0000FF;">,</span> <span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">1</span>
<span style="color: #008080;">while</span> <span style="color: #004600;">true</span> <span style="color: #008080;">do</span>
<span style="color: #000000;">start</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<nowiki>{{</nowiki>hea`</span><span style="color: #0000FF;">&</span><span style="color: #008000;">`der|`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">contents</span><span style="color: #0000FF;">,</span><span style="color: #000000;">start</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">start</span><span style="color: #0000FF;">=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #000080;font-style:italic;">--
-- skip duplicates/we also have to cope with eg
-- ==<nowiki>{{</nowiki>he&der|Python<nowiki>}}</nowiki>== \
-- ===<nowiki>{{</nowiki>he&der|Python<nowiki>}}</nowiki> Original=== } count
-- ===<nowiki>{{</nowiki>he&der|Python<nowiki>}}</nowiki> Succinct=== } once
-- ===<nowiki>{{</nowiki>he&der|Python<nowiki>}}</nowiki> Recursive === /
-- ==<nowiki>{{</nowiki>he&der|Mathematica<nowiki>}}</nowiki> / <nowiki>{{</nowiki>he&der|Wolfram Language<nowiki>}}</nowiki>== \
-- ==<nowiki>{{</nowiki>he&der|Icon<nowiki>}}</nowiki> and <nowiki>{{</nowiki>he&der|Unicon<nowiki>}}</nowiki>== } count
-- == <nowiki>{{</nowiki>he&der|Icon<nowiki>}}</nowiki> and <nowiki>{{</nowiki>he&der|Unicon<nowiki>}}</nowiki> == / both
-- == <nowiki>{{</nowiki>he&der|Java<nowiki>}}</nowiki>==
-- etc. Note however that this /does/ count eg
-- ===<nowiki>{{</nowiki>he&der|Applesoft BASIC<nowiki>}}</nowiki>=== \
-- ===<nowiki>{{</nowiki>he&der|BASIC256<nowiki>}}</nowiki>=== } count
-- ===<nowiki>{{</nowiki>he&der|Commodore BASIC<nowiki>}}</nowiki>=== } 'em
-- ===<nowiki>{{</nowiki>he&der|IS-BASIC<nowiki>}}</nowiki>=== } all
-- ===<nowiki>{{</nowiki>he&der|Sinclair ZX81 BASIC<nowiki>}}</nowiki>=== /
--</span>
<span style="color: #000000;">curr</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">contents</span><span style="color: #0000FF;">[</span><span style="color: #000000;">start</span><span style="color: #0000FF;">..</span><span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<nowiki>}}</nowiki>`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">contents</span><span style="color: #0000FF;">,</span><span style="color: #000000;">start</span><span style="color: #0000FF;">+</span><span style="color: #000000;">1</span><span style="color: #0000FF;">)]</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">curr</span><span style="color: #0000FF;">!=</span><span style="color: #000000;">prev</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">count</span> <span style="color: #0000FF;">+=</span> <span style="color: #000000;">1</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #000000;">prev</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">curr</span>
<span style="color: #000000;">start</span> <span style="color: #0000FF;">+=</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`<nowiki>{{</nowiki>hea`</span><span style="color: #0000FF;">&</span><span style="color: #008000;">`der|`</span><span style="color: #0000FF;">)</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">while</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">sort_by_count</span> <span style="color: #008080;">then</span>
<span style="color: #000000;">task_counts</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">count</span>
<span style="color: #008080;">elsif</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">notlang</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">or</span> <span style="color: #000000;">i</span><span style="color: #0000FF;"><=</span><span style="color: #000000;">2</span> <span style="color: #008080;">or</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">>=</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)-</span><span style="color: #000000;">1</span> <span style="color: #008080;">or</span> <span style="color: #7060A8;">mod</span><span style="color: #0000FF;">(</span><span style="color: #000000;">i</span><span style="color: #0000FF;">,</span><span style="color: #000000;">200</span><span style="color: #0000FF;">)=</span><span style="color: #000000;">0</span> <span style="color: #008080;">then</span>
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%s: %d\n"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">html_clean</span><span style="color: #0000FF;">(</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">),</span><span style="color: #000000;">count</span><span style="color: #0000FF;">})</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #000000;">total_count</span> <span style="color: #0000FF;">+=</span> <span style="color: #000000;">count</span>
<span style="color: #008080;">if</span> <span style="color: #7060A8;">get_key</span><span style="color: #0000FF;">()=</span><span style="color: #000000;">#1B</span> <span style="color: #008080;">then</span> <span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"escape keyed\n"</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">exit</span> <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #000000;">curl_cleanup</span><span style="color: #0000FF;">()</span>
<span style="color: #008080;">if</span> <span style="color: #000000;">sort_by_count</span> <span style="color: #008080;">then</span>
<span style="color: #004080;">sequence</span> <span style="color: #000000;">tags</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">custom_sort</span><span style="color: #0000FF;">(</span><span style="color: #000000;">task_counts</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">tagset</span><span style="color: #0000FF;">(</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">)))</span>
<span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tags</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">to</span> <span style="color: #000000;">1</span> <span style="color: #008080;">by</span> <span style="color: #0000FF;">-</span><span style="color: #000000;">1</span> <span style="color: #008080;">do</span>
<span style="color: #004080;">integer</span> <span style="color: #000000;">ti</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">tags</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]</span>
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"%s: %d\n"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">html_clean</span><span style="color: #0000FF;">(</span><span style="color: #000000;">tasks</span><span style="color: #0000FF;">[</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">]),</span><span style="color: #000000;">task_counts</span><span style="color: #0000FF;">[</span><span style="color: #000000;">ti</span><span style="color: #0000FF;">]})</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
<span style="color: #008080;">return</span> <span style="color: #000000;">total_count</span>
<span style="color: #008080;">end</span> <span style="color: #008080;">function</span>
<span style="color: #7060A8;">progress</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"Total: %d\n"</span><span style="color: #0000FF;">,{</span><span style="color: #000000;">count_tasks</span><span style="color: #0000FF;">()})</span>
<!--

View file

@ -0,0 +1,12 @@
(load "@lib/http.l")
(client "rosettacode.org" 80
"mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
(while (from " title=\"")
(let Task (till "\"")
(client "rosettacode.org" 80 (pack "wiki/" (replace Task " " "_"))
(let Cnt 0
(while (from "<span class=\"tocnumber\">")
(unless (sub? "." (till "<" T))
(inc 'Cnt) ) )
(out NIL (prinl (ht:Pack Task) ": " Cnt)) ) ) ) ) )

View file

@ -0,0 +1,64 @@
Procedure handleError(value, msg.s)
If value = 0
MessageRequester("Error", msg)
End
EndIf
EndProcedure
handleError(InitNetwork(), "Unable to initialize network functions.")
If OpenConsole()
Define url$, x1$, y1$, title$, unescapedTitle$, encodedURL$
Define x2, i, j, totalExamples, totalTasks
url$ = "http://www.rosettacode.org/mw/api.php?action=query" +
"&list=categorymembers&cmtitle=Category:Programming_Tasks" +
"&cmlimit=500&format=xml"
Repeat
handleError(ReceiveHTTPFile(url$, "tasks.xml"), "Unable to access tasks URL.")
handleError(ReadFile(0, "tasks.xml"), "Unable to read 'task.xml' file.")
x1$ = ReadString(0)
CloseFile(0)
Repeat
x2 = FindString(x1$, "title=", x2 + 1)
If x2
title$ = Mid(x1$, x2 + 7, 99)
title$ = Left(title$, FindString(title$, ">", 1) - 4)
unescapedTitle$ = UnescapeString(ReplaceString(title$, "&#039;", "&apos;"), #PB_String_EscapeXML)
encodedURL$ = URLEncoder("http://www.rosettacode.org/mw/index.php?title=" + unescapedTitle$ + "&action=raw")
If ReceiveHTTPFile(encodedURL$, "task.xml")
ReadFile(0, "task.xml")
While Not Eof(0)
y1$ = ReadString(0)
If FindString(y1$, "=={{header|", 1, #PB_String_NoCase)
totalExamples + 1
EndIf
Wend
CloseFile(0)
PrintN(unescapedTitle$ +": " + Str(totalExamples) + " examples")
totalTasks + totalExamples
totalExamples = 0
EndIf
EndIf
Until x2 = 0
;check for additional pages of tasks
x2 = FindString(x1$, "cmcontinue=")
If x2
i = FindString(x1$, #DQUOTE$, x2 + 1)
j = FindString(x1$, #DQUOTE$, i + 1)
url$ = URLEncoder("http://www.rosettacode.org/mw/api.php?action=query" +
"&list=categorymembers&cmtitle=Category:Programming_Tasks" +
"&cmlimit=500&format=xml&cmcontinue=" + Mid(x1$, i + 1, j - i))
Else
Break ;all done
EndIf
ForEver
PrintN("Total: " + Str(totalTasks) + " examples")
Input()
CloseConsole()
EndIf

View file

@ -0,0 +1,18 @@
from urllib.request import urlopen, Request
import xml.dom.minidom
r = Request(
'https://www.rosettacode.org/mw/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml',
headers={'User-Agent': 'Mozilla/5.0'})
x = urlopen(r)
tasks = []
for i in xml.dom.minidom.parseString(x.read()).getElementsByTagName('cm'):
t = i.getAttribute('title').replace(' ', '_')
r = Request(f'https://www.rosettacode.org/mw/index.php?title={t}&action=raw',
headers={'User-Agent': 'Mozilla/5.0'})
y = urlopen(r)
tasks.append( y.read().lower().count(b'{{header|') )
print(t.replace('_', ' ') + f': {tasks[-1]} examples.')
print(f'\nTotal: {sum(tasks)} examples.')

View file

@ -0,0 +1,394 @@
"""Count Rosetta Code tasks implementations using the Semantic MediaWiki API.
Works with Python >= 3.7."""
import json
import logging
from dataclasses import dataclass
from dataclasses import field
from datetime import datetime
from typing import Any
from typing import Dict
from typing import List
from typing import Optional
from typing import Set
from typing import Tuple
import requests
from requests.adapters import HTTPAdapter
from requests.adapters import Retry
logging.basicConfig(level=logging.WARN)
# See https://www.semantic-mediawiki.org/wiki/Help:API:ask
_SM_ASK: Dict[str, str] = {
"action": "ask",
"format": "json",
"formatversion": "2",
"api_version": "3",
}
_SM_ASK_REQUEST_BLOCK_SIZE = 500
@dataclass(frozen=True)
class Page:
# fulltext is the page's title, not the page content.
fulltext: str
fullurl: str
namespace: int
exists: str
displaytitle: str
@dataclass(frozen=True, eq=False)
class Lang(Page):
def __eq__(self, other: object) -> bool:
if isinstance(other, Lang):
return self.fullurl == other.fullurl
elif isinstance(other, str):
return self.fullurl == other
return False
def __hash__(self) -> int:
return hash(self.fullurl)
def unimplemented(self, tasks: Set["Task"]) -> Set["Task"]:
return {task for task in tasks if self.fullurl not in task.exclude}
def omitted_from(self, tasks: Set["Task"]) -> Set["Task"]:
return {task for task in tasks if self.fullurl in task.omitted_from}
@dataclass(frozen=True)
class Task(Page):
title: str
implemented_in: Set[Lang] = field(repr=False, compare=False)
omitted_from: Set[Lang] = field(repr=False, compare=False)
exclude: Set[Lang] = field(repr=False, compare=False)
def not_implemented_in(self, langs: Set[Lang]) -> Set[Lang]:
return langs.difference(self.exclude)
@dataclass
class TaskResponseBlock:
tasks: List[Task]
continue_offset: Optional[int] = None
@dataclass
class LanguageResponseBlock:
langs: List[Lang]
continue_offset: Optional[int] = None
def sm_ask_category(
session: requests.Session,
url: str,
category: str,
limit: int,
offset: int,
known_langs: Set[Lang],
) -> TaskResponseBlock:
query_params = {
**_SM_ASK,
"query": (
f"[[Category:{category}]]"
"|?Implemented in language"
"|?Omitted from language"
f"|limit={limit}"
f"|offset={offset}"
),
}
# Display some progress
log(f"ask [[Category:{category}]] offset={offset}")
response = session.get(url, params=query_params)
response.raise_for_status()
data = response.json()
handle_warnings_and_errors(data)
return _transform_implemented_in_response_data(data, known_langs)
def sm_ask_tasks(
session: requests.Session,
url: str,
limit: int,
offset: int,
known_langs: Set[Lang],
):
return sm_ask_category(
session, url, "Programming Tasks", limit, offset, known_langs
)
def sm_ask_drafts(
session: requests.Session,
url: str,
limit: int,
offset: int,
known_langs: Set[Lang],
):
return sm_ask_category(
session, url, "Draft Programming Tasks", limit, offset, known_langs
)
def sm_ask_languages(
session: requests.Session,
url: str,
limit: int,
offset: int,
) -> LanguageResponseBlock:
query_params = {
**_SM_ASK,
"query": (
"[[Is language::+]]"
"|?Implemented in language"
"|?Omitted from language"
f"|limit={limit}"
f"|offset={offset}"
),
}
# Display some progress
log(f"ask [[Is language::+]] offset={offset}")
response = session.get(url, params=query_params)
response.raise_for_status()
data = response.json()
handle_warnings_and_errors(data)
return _transform_language_response_data(data)
def sm_ask_all_tasks(
session: requests.Session, url: str, known_langs: Set[Lang]
) -> List[Task]:
block = sm_ask_tasks(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=0,
known_langs=known_langs,
)
tasks = block.tasks
while block.continue_offset:
block = sm_ask_tasks(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=block.continue_offset,
known_langs=known_langs,
)
tasks.extend(block.tasks)
return tasks
def sm_ask_all_drafts(
session: requests.Session, url: str, known_langs: Set[Lang]
) -> List[Task]:
block = sm_ask_drafts(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=0,
known_langs=known_langs,
)
tasks = block.tasks
while block.continue_offset:
block = sm_ask_drafts(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=block.continue_offset,
known_langs=known_langs,
)
tasks.extend(block.tasks)
return tasks
def sm_ask_all_languages(session: requests.Session, url: str) -> List[Lang]:
block = sm_ask_languages(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=0,
)
langs = block.langs
while block.continue_offset:
block = sm_ask_languages(
session,
url,
limit=_SM_ASK_REQUEST_BLOCK_SIZE,
offset=block.continue_offset,
)
langs.extend(block.langs)
return langs
def _transform_implemented_in_response_data(
data: Any, known_langs: Set[Lang]
) -> TaskResponseBlock:
tasks: List[Task] = []
for result in data["query"]["results"]:
for task_title, task_page in result.items():
# We're excluding implementations that don't have a corresponding
# category page with an "Is Language" property.
implemented_in = {
Lang(**lang)
for lang in task_page["printouts"]["Implemented in language"]
}.intersection(known_langs)
omitted_from = (
{
Lang(**lang)
for lang in task_page["printouts"]["Omitted from language"]
}
.intersection(known_langs)
.difference(implemented_in)
)
tasks.append(
Task(
title=task_title,
implemented_in=implemented_in,
omitted_from=omitted_from,
fulltext=task_page["fulltext"],
fullurl=task_page["fullurl"],
namespace=task_page["namespace"],
exists=task_page["exists"],
displaytitle=task_page["displaytitle"],
exclude=implemented_in.union(omitted_from),
)
)
return TaskResponseBlock(
tasks=tasks, continue_offset=data.get("query-continue-offset", None)
)
def _transform_language_response_data(data: Any) -> LanguageResponseBlock:
langs: List[Lang] = []
for result in data["query"]["results"]:
for _, task_page in result.items():
langs.append(
Lang(
fulltext=task_page["fulltext"],
fullurl=task_page["fullurl"],
namespace=task_page["namespace"],
exists=task_page["exists"],
displaytitle=task_page["displaytitle"],
)
)
return LanguageResponseBlock(
langs=langs, continue_offset=data.get("query-continue-offset", None)
)
def get_session() -> requests.Session:
"""Setup a requests.Session with retries."""
retry_strategy = Retry(
total=5,
status_forcelist=[429, 500, 502, 503, 504],
allowed_methods=["HEAD", "GET", "OPTIONS"],
)
adapter = HTTPAdapter(max_retries=retry_strategy)
session = requests.Session()
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
def log(msg: str) -> None:
print(f"{datetime.now().isoformat(' ', 'seconds')}: {msg}")
def handle_warnings_and_errors(data: Any) -> None:
if data.get("errors"):
for error in data["errors"]:
logging.error(json.dumps(error))
# legacy format
if data.get("error"):
logging.error(json.dumps(data["error"]))
if data.get("warnings"):
for warning in data["warnings"]:
logging.warning(json.dumps(warning))
def count_examples(url: str, n: int = 30) -> None:
"""Print a table to stdout containing implementation counts for the first
`n` tasks, sorted by number implementations (most to least)."""
session = get_session()
langs = set(sm_ask_all_languages(session, url))
tasks = sm_ask_all_tasks(session, url, langs)
drafts = sm_ask_all_drafts(session, url, langs)
all_tasks = [*tasks, *drafts]
# Map of task to (implemented in, not implemented in, omitted from)
counts: Dict[Task, Tuple[int, int, int]] = {}
# Running total of examples for all tasks. Where a language has multiple examples
# for a single tasks, we only count one example.
total: int = 0
for task in all_tasks:
total += len(task.implemented_in)
counts[task] = (
len(task.implemented_in),
len(task.not_implemented_in(langs)),
len(task.omitted_from),
)
# Pretty print
top = sorted(counts.items(), key=lambda it: it[1][0], reverse=True)[:n]
pad = max([len(task.fulltext) for task, _ in top])
print("\nKnown languages:", len(langs))
print("Total tasks:", len(all_tasks))
print("Total examples:", total)
print(f"{'Task':>{pad}} | Implemented In | Not Implemented In | Omitted From")
print("-" * (pad + 1), "+", "-" * 16, "+", "-" * 20, "+", "-" * 13, sep="")
for task, _counts in top:
implemented_in, not_implemented_in, omitted_from = _counts
print(
f"{task.fulltext:>{pad}} |"
f"{implemented_in:>15} |"
f"{not_implemented_in:>19} |"
f"{omitted_from:>13}"
)
if __name__ == "__main__":
import argparse
URL = "https://rosettacode.org/w/api.php"
parser = argparse.ArgumentParser(description="Count tasks on Rosetta Code.")
parser.add_argument(
"--rows",
"-n",
type=int,
default=30,
dest="n",
help="number of rows to display in the output table (default: 30)",
)
parser.add_argument(
"--url",
default=URL,
help=f"target MediaWiki URL (default: {URL})",
)
args = parser.parse_args()
count_examples(args.url, args.n)

View file

@ -0,0 +1,14 @@
library(XML)
library(RCurl)
doc <- xmlInternalTreeParse("http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml")
nodes <- getNodeSet(doc,"//cm")
titles = as.character( sapply(nodes, xmlGetAttr, "title") )
headers <- list()
counts <- list()
for (i in 1:length(titles)){
headers[[i]] <- getURL( paste("http://rosettacode.org/mw/index.php?title=", gsub(" ", "_", titles[i]), "&action=raw", sep="") )
counts[[i]] <- strsplit(headers[[i]],split=" ")[[1]]
counts[[i]] <- grep("\\{\\{header", counts[[i]])
cat(titles[i], ":", length(counts[[i]]), "examples\n")
}
cat("Total: ", length(unlist(counts)), "examples\n")

View file

@ -0,0 +1,25 @@
#lang racket
(require net/url net/uri-codec json (only-in racket/dict [dict-ref ref]))
(define (RC-get verb params)
((compose1 get-pure-port string->url format)
"http://rosettacode.org/mw/~a.php?~a" verb (alist->form-urlencoded params)))
(define (get-category catname)
(let loop ([c #f])
(define t
((compose1 read-json RC-get) 'api
`([action . "query"] [format . "json"]
[list . "categorymembers"] [cmtitle . ,(format "Category:~a" catname)]
[cmcontinue . ,(and c (ref c 'cmcontinue))] [cmlimit . "500"])))
(define (c-m key) (ref (ref t key '()) 'categorymembers #f))
(append (for/list ([page (c-m 'query)]) (ref page 'title))
(cond [(c-m 'query-continue) => loop] [else '()]))))
(printf "Total: ~a\n"
(for/sum ([task (get-category 'Programming_Tasks)])
(define s ((compose1 length regexp-match-positions*)
#rx"=={{" (RC-get 'index `([action . "raw"] [title . ,task]))))
(printf "~a: ~a\n" task s)
s))

View file

@ -0,0 +1,175 @@
use HTTP::UserAgent;
use URI::Escape;
use JSON::Fast;
use Lingua::EN::Numbers :short;
unit sub MAIN ( Bool :nf(:$no-fetch) = False, :t(:$tier) = 1 );
# Friendlier descriptions for task categories
my %cat = (
'Programming_Tasks' => 'Task',
'Draft_Programming_Tasks' => 'Draft'
);
my $client = HTTP::UserAgent.new;
$client.timeout = 10;
my $url = 'https://rosettacode.org/w';
my $hashfile = './RC_Task_count.json';
my $tablefile = "./RC_Task_count-{$tier}.txt";
my %tasks;
my @places = <① ② ③ ④ ⑤ ⑥ ⑦ ⑧ ⑨ ⑩ ⑪ ⑫ ⑬ ⑭ ⑮ ⑯ ⑰ ⑱ ⑲ ⑳ ㉑ ㉒ ㉓ ㉔ ㉕
㉖ ㉗ ㉘ ㉙ ㉚ ㉛ ㉜ ㉝ ㉞ ㉟ ㊱ ㊲ ㊳ ㊴ ㊵ ㊶ ㊷ ㊸ ㊹ ㊺ ㊺ ㊻ ㊼ ㊽ ㊾ ㊿>;
# clear screen
run($*DISTRO.is-win ?? 'cls' !! 'clear') unless $no-fetch;
my %counts =
mediawiki-query(
$url, 'pages',
:generator<categorymembers>,
:gcmtitle<Category:Programming Languages>,
:gcmlimit<350>,
:rawcontinue(),
:prop<categoryinfo>
)
.map({ .<title>.subst(/^'Category:'/, '') => .<categoryinfo><pages> || 0 });
my $per-tier = 10;
my $which = (^$per-tier) »+» $per-tier * ($tier - 1);
my @top-n = %counts.sort( {-.value, .key} )[|$which].map: *.key.trans(' ' => '_');
# dump a copy to STDOUT, mostly for debugging purposes
say "<pre>{tc $tier.&ord-n} {$per-tier.&card} programming languages by number of task examples completed:";
say ' ', join ' ', .map( {("{(@places[|$which])[$_]} {@top-n[$_]}").fmt("%-15s")} ) for (^@top-n).batch(5);
say "</pre>\n";
unless $no-fetch {
note 'Retrieving task information...';
mkdir('./pages') unless './pages'.IO.e;
@top-n = %counts.sort( {-.value, .key} )[^@places].map: *.key.trans(' ' => '_');;
for %cat.keys.sort -> $cat {
mediawiki-query(
$url, 'pages',
:generator<categorymembers>,
:gcmtitle("Category:$cat"),
:gcmlimit<350>,
:rawcontinue(),
:prop<title>
).map({
my $page;
my $response;
loop {
$response = $client.get("{ $url }/index.php?title={ uri-escape .<title> }&action=raw");
if $response.is-success {
$page = $response.content;
last;
} else {
redo;
}
}
"./pages/{ uri-escape .<title>.subst(/' '/, '_', :g) }".IO.spurt($page);
my $lc = $page.lc.trans(' ' => '_');
my $count = +$lc.comb(/ ^^'==' <-[\n=]>* '{{header|' <-[}]>+? '}}==' \h* $$ /);
%tasks{.<title>} = {'cat' => %cat{$cat}, :$count};
%tasks{.<title>}<top-n> = (^@top-n).map( {
($lc.contains("==\{\{header|{@top-n[$_].lc}}}") or
# Deal with 3 part headers - {{header|F_Sharp|F#}}, {{header|C_Sharp|C#}}, etc.
$lc.contains("==\{\{header|{@top-n[$_].lc}|") or
# Icon and Unicon are their own special flowers
$lc.contains("}}_and_\{\{header|{@top-n[$_].lc}}}==") or
# Language1 / Language2 for shared entries (e.g. C / C++)
$lc.contains(rx/'}}''_'*'/''_'*'{{header|'$(@top-n[$_].lc)'}}=='/)) ??
(@places[$_]) !!
# Check if the task was omitted
$lc.contains("\{\{omit_from|{@top-n[$_].lc}") ?? 'O' !!
# The task is neither done or omitted
' '
} ).join;
print clear, 1 + $++, ' ', %cat{$cat}, ' ', .<title>;
})
}
print clear;
note "\nTask information saved to local file: {$hashfile.IO.absolute}";
$hashfile.IO.spurt(%tasks.&to-json);
}
# Load information from local file
%tasks = $hashfile.IO.e ?? $hashfile.IO.slurp.&from-json !! ( );
@top-n = %counts.sort( {-.value, .key} )[|$which].map: *.key.trans(' ' => '_');
# Convert saved task info to a table
note "\nBuilding table...";
my $count = +%tasks;
my $taskcnt = +%tasks.grep: *.value.<cat> eq %cat<Programming_Tasks>;
my $draftcnt = $count - $taskcnt;
my $total = sum %tasks{*}»<count>;
# Dump table to a file
my $out = open($tablefile, :w) or die "$!\n";
$out.say: "<pre>{tc $tier.&ord-n} {$per-tier.&card} programming languages by number of task examples completed:";
$out.say: ' ', join ' ', .map( {("{(@places[|$which])[$_]} {@top-n[$_]}").fmt("%-15s")} ) for (^@top-n).batch(5);
$out.say: "</pre>\n\n<div style=\"height:40em;overflow:scroll;\">";
# Add table boilerplate and caption
$out.say:
'{|class="wikitable sortable"', "\n",
"|+ As of { DateTime.new(time) } :: Tasks: { $taskcnt } ::<span style=\"background-color:#ffd\"> Draft Tasks:",
"{ $draftcnt } </span>:: Total Tasks: { $count } :: Total Examples: { $total }\n",
"!Count!!Task!!{(@places[|$which]).join('!!')}"
;
# Sort tasks by count then add row
for %tasks.sort: { [-.value<count>, .key] } -> $task {
$out.say:
( $task.value<cat> eq 'Draft'
?? "|- style=\"background-color: #ffc\"\n"
!! "|-\n"
),
"| { $task.value<count> }\n",
( $task.key ~~ /\d/
?? "|data-sort-value=\"{ $task.key.&naturally }\"| [[{uri-escape $task.key}|{$task.key}]]\n"
!! "| [[{uri-escape $task.key}|{$task.key}]]\n"
),
"|{ $task.value<top-n>.comb[|$which].join('||') }"
}
$out.say( "|}\n</div>" );
$out.close;
note "Table file saved as: {$tablefile.IO.absolute}";
sub mediawiki-query ($site, $type, *%query) {
my $url = "$site/api.php?" ~ uri-query-string(
:action<query>, :format<json>, :formatversion<2>, |%query);
my $continue = '';
gather loop {
my $response = $client.get("$url&$continue");
my $data = from-json($response.content);
take $_ for $data.<query>.{$type}.values;
$continue = uri-query-string |($data.<query-continue>{*}».hash.hash or last);
}
}
sub uri-query-string (*%fields) { %fields.map({ "{.key}={uri-escape .value}" }).join("&") }
sub naturally ($a) { $a.lc.subst(/(\d+)/, ->$/ {0~(65+$0.chars).chr~$0},:g) }
sub clear { "\r" ~ ' ' x 116 ~ "\r" }

View file

@ -0,0 +1,55 @@
# Project: Rosetta Code/Count examples
load "stdlib.ring"
ros= download("http://rosettacode.org/wiki/Category:Programming_Tasks")
pos = 1
num = 0
totalros = 0
rosname = ""
rostitle = ""
for n = 1 to len(ros)
nr = searchstring(ros,'<li><a href="/wiki/',pos)
if nr = 0
exit
else
pos = nr + 1
ok
nr = searchname(nr)
nr = searchtitle(nr)
next
see nl
see "Total: " + totalros + " examples." + nl
func searchstring(str,substr,n)
newstr=right(str,len(str)-n+1)
nr = substr(newstr, substr)
if nr = 0
return 0
else
return n + nr -1
ok
func searchname(sn)
nr2 = searchstring(ros,'">',sn)
nr3 = searchstring(ros,"</a></li>",sn)
rosname = substr(ros,nr2+2,nr3-nr2-2)
return sn
func searchtitle(sn)
st = searchstring(ros,"title=",sn)
rostitle = substr(ros,sn+19,st-sn-21)
rostitle = "rosettacode.org/wiki/" + rostitle
rostitle = download(rostitle)
sum = count(rostitle,"Edit section:")
num = num + 1
see "" + num + ". " + rosname + ": " + sum + " examples." + nl
totalros = totalros + sum
return sn
func count(cstring,dstring)
sum = 0
while substr(cstring,dstring) > 0
sum = sum + 1
cstring = substr(cstring,substr(cstring,dstring)+len(string(sum)))
end
return sum

View file

@ -0,0 +1,51 @@
require 'open-uri'
require 'rexml/document'
module RosettaCode
URL_ROOT = "http://rosettacode.org/mw"
def self.get_url(page, query)
begin
# Ruby 1.9.2
pstr = URI.encode_www_form_component(page)
qstr = URI.encode_www_form(query)
rescue NoMethodError
require 'cgi'
pstr = CGI.escape(page)
qstr = query.map {|k,v|
"%s=%s" % [CGI.escape(k.to_s), CGI.escape(v.to_s)]}.join("&")
end
url = "#{URL_ROOT}/#{pstr}?#{qstr}"
p url if $DEBUG
url
end
def self.get_api_url(query)
get_url "api.php", query
end
def self.category_members(category)
query = {
"action" => "query",
"list" => "categorymembers",
"cmtitle" => "Category:#{category}",
"format" => "xml",
"cmlimit" => 500,
}
while true
url = get_api_url query
doc = REXML::Document.new open(url)
REXML::XPath.each(doc, "//cm") do |task|
yield task.attribute("title").value
end
continue = REXML::XPath.first(doc, "//query-continue")
break if continue.nil?
cm = REXML::XPath.first(continue, "categorymembers")
query["cmcontinue"] = cm.attribute("cmcontinue").value
end
end
end

View file

@ -0,0 +1,13 @@
require 'rosettacode'
total_examples = 0
RosettaCode.category_members("Programming_Tasks") do |task|
url = RosettaCode.get_url("index.php", {"action" => "raw", "title" => task})
examples = open(url).read.scan("=={{header").length
puts "#{task}: #{examples}"
total_examples += examples
end
puts
puts "Total: #{total_examples}"

View file

@ -0,0 +1,32 @@
html "<table border=1><tr bgcolor=wheat align=center><td>Num</td><td>Task</td><td>Examples</td></tr>"
a$ = httpGet$("http://rosettacode.org/wiki/Category:Programming_Tasks")
a$ = word$(a$,1,"</table></div>")
i = instr(a$,"<a href=""/wiki/")
i = instr(a$,"<a href=""/wiki/",i+1)
while i > 0
count = count + 1
i = instr(a$,"<a href=""/wiki/",i+1)
j = instr(a$,">",i+5)
a1$ = mid$(a$,i+15,j-i)
taskId$ = word$(a1$,1,"""")
task$ = word$(a1$,3,"""")
url$ = "http://rosettacode.org/wiki/";taskId$
a2$ = httpGet$(url$)
ii = instr(a2$,"<span class=""tocnumber"">")
jj = 0
while ii > 0
jj = ii
ii = instr(a2$,"<span class=""tocnumber"">",ii+10)
wend
if jj = 0 then
examp = 0
else
kk = instr(a2$,"<",jj+24)
examp = int(val(mid$(a2$,jj+24,kk-jj-24)))
end if
html "<tr><td align=right>";count;"</td><td>";task$;"</td><td align=right>";examp;"</td></tr>"
totExamp = totExamp + examp
wend
html "<tr bgcolor=wheat><td>**</td><td>** Total **</td><td align=right>";totExamp;"</td></tr></table>"
end

View file

@ -0,0 +1,112 @@
extern crate reqwest;
extern crate url;
extern crate rustc_serialize;
use std::io::Read;
use self::url::Url;
use rustc_serialize::json::{self, Json};
pub struct Task {
page_id: u64,
pub title: String,
}
#[derive(Debug)]
enum ParseError {
/// Something went wrong with the HTTP request to the API.
Http(reqwest::Error),
/// There was a problem parsing the API response into JSON.
Json(json::ParserError),
/// Unexpected JSON format from response
UnexpectedFormat,
}
impl From<json::ParserError> for ParseError {
fn from(error: json::ParserError) -> Self {
ParseError::Json(error)
}
}
impl From<reqwest::Error> for ParseError {
fn from(error: reqwest::Error) -> Self {
ParseError::Http(error)
}
}
fn construct_query_category(category: &str) -> Url {
let mut base_url = Url::parse("http://rosettacode.org/mw/api.php").unwrap();
let cat = format!("Category:{}", category);
let query_pairs = vec![("action", "query"),
("format", "json"),
("list", "categorymembers"),
("cmlimit", "500"),
("cmtitle", &cat),
("continue", "")];
base_url.query_pairs_mut().extend_pairs(query_pairs.into_iter());
base_url
}
fn construct_query_task_content(task_id: &str) -> Url {
let mut base_url = Url::parse("http://rosettacode.org/mw/api.php").unwrap();
let mut query_pairs =
vec![("action", "query"), ("format", "json"), ("prop", "revisions"), ("rvprop", "content")];
query_pairs.push(("pageids", task_id));
base_url.query_pairs_mut().extend_pairs(query_pairs.into_iter());
base_url
}
fn query_api(url: Url) -> Result<Json, ParseError> {
let mut response = try!(reqwest::get(url.as_str()));
// Build JSON
let mut body = String::new();
response.read_to_string(&mut body).unwrap();
Ok(try!(Json::from_str(&body)))
}
fn parse_all_tasks(reply: &Json) -> Result<Vec<Task>, ParseError> {
let json_to_task = |json: &Json| -> Result<Task, ParseError> {
let page_id: u64 = try!(json.find("pageid")
.and_then(|id| id.as_u64())
.ok_or(ParseError::UnexpectedFormat));
let title: &str = try!(json.find("title")
.and_then(|title| title.as_string())
.ok_or(ParseError::UnexpectedFormat));
Ok(Task {
page_id: page_id,
title: title.to_owned(),
})
};
let tasks_json = try!(reply.find_path(&["query", "categorymembers"])
.and_then(|tasks| tasks.as_array())
.ok_or(ParseError::UnexpectedFormat));
// Convert into own type
tasks_json.iter().map(json_to_task).collect()
}
fn count_number_examples(task: &Json, task_id: u64) -> Result<u32, ParseError> {
let revisions =
try!(task.find_path(&["query", "pages", task_id.to_string().as_str(), "revisions"])
.and_then(|content| content.as_array())
.ok_or(ParseError::UnexpectedFormat));
let content = try!(revisions[0]
.find("*")
.and_then(|content| content.as_string())
.ok_or(ParseError::UnexpectedFormat));
Ok(content.split("=={{header").count() as u32)
}
pub fn query_all_tasks() -> Vec<Task> {
let query = construct_query_category("Programming_Tasks");
let json: Json = query_api(query).unwrap();
parse_all_tasks(&json).unwrap()
}
pub fn query_a_task(task: &Task) -> u32 {
let query = construct_query_task_content(&task.page_id.to_string());
let json: Json = query_api(query).unwrap();
count_number_examples(&json, task.page_id).unwrap()
}

View file

@ -0,0 +1,8 @@
extern crate count_examples;
fn main() {
let all_tasks = count_examples::query_all_tasks();
for task in &all_tasks {
let count = count_examples::query_a_task(task);
println!("Task: {} has {} examples", task.title, count);
}
}

View file

@ -0,0 +1,25 @@
import scala.language.postfixOps
object TaskCount extends App {
import java.net.{ URL, URLEncoder }
import scala.io.Source.fromURL
System.setProperty("http.agent", "*")
val allTasksURL =
"http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
val allTasks = xml.parsing.XhtmlParser(fromURL(new URL(allTasksURL)))
val regexExpr = "(?i)==\\{\\{header\\|".r
def oneTaskURL(title: String) = {
println(s"Check $title")
"http://www.rosettacode.org/w/index.php?title=%s&action=raw" format URLEncoder.encode(title.replace(' ', '_'), "UTF-8")
}
def count(title: String) = regexExpr findAllIn fromURL(new URL(oneTaskURL(title)))(io.Codec.UTF8).mkString length
val counts = for (task <- allTasks \\ "cm" \\ "@title" map (_.text)) yield scala.actors.Futures.future((task, count(task)))
counts map (_.apply) map Function.tupled("%s: %d examples." format (_, _)) foreach println
println("\nTotal: %d examples." format (counts map (_.apply._2) sum))
}

View file

@ -0,0 +1,13 @@
var lwp = require('LWP::UserAgent').new(agent => 'Mozilla/5.0');
var site = 'http://rosettacode.org';
var list_url = '/mw/api.php?action=query&list=categorymembers&'+
'cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml';
var content = lwp.get(site + list_url).decoded_content;
while (var m = content.match(/cm.*?title="(.*?)"/g)) {
(var slug = m[0]).gsub!(' ', '_');
var count = lwp.get("#{site}/wiki/#{slug}").decoded_content.count(/toclevel-1/g);
say "#{m[0]}: #{count} examples";
}

View file

@ -0,0 +1,39 @@
$$ MODE TUSCRIPT
url="http://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
data=REQUEST (url)
BUILD S_TABLE beg=*
DATA :title=":
BUILD S_TABLE end=*
DATA :":
titles=EXTRACT (data,beg|,end,1,0,"~~")
titles=SPLIT (titles,":~~:")
sz_titles=SIZE (titles)
BUILD R_TABLE header=":==\{\{header|:"
all=*
ERROR/STOP CREATE ("tasks",seq-e,-std-)
COMPILE
LOOP title=titles
ask=*
ask =SET_VALUE(ask,"title",title)
ask =SET_VALUE(ask,"action","raw")
ask =ENCODE (ask,cgi)
http ="http://www.rosettacode.org/mw/index.php"
url =CONCAT (http,"?",ask)
data =REQUEST (url)
header =FILTER_INDEX (data,header,-)
sz_header=SIZE(header)
line =CONCAT (title,"=",sz_header," members")
FILE "tasks" = line
all =APPEND(all,sz_header)
ENDLOOP
ENDCOMPILE
all =JOIN(all),sum=SUM(all),time=time()
line=CONCAT (time,": ", sz_titles, " Programing Tasks: ", sum, " solutions")
FILE "tasks" = line

View file

@ -0,0 +1,65 @@
package require Tcl 8.5
package require http
package require json
fconfigure stdout -buffering none
proc get_tasks {category} {
set start [clock milliseconds]
puts -nonewline "getting $category members..."
set base_url http://www.rosettacode.org/w/api.php
set query {action query list categorymembers cmtitle Category:%s format json cmlimit 500}
set this_query [dict create {*}[split [format $query $category]]]
set tasks [list]
while {1} {
set url [join [list $base_url [http::formatQuery {*}$this_query]] ?]
set response [http::geturl $url]
if {[set s [http::status $response]] ne "ok" || [http::ncode $response] != 200} {
error "Oops: url=$url\nstatus=$s\nhttp code=[http::code $response]"
}
set data [json::json2dict [http::data $response]]
http::cleanup $response
# add tasks to list
foreach task [dict get $data query categorymembers] {
lappend tasks [dict get [dict create {*}$task] title]
}
if {[catch {dict get $data query-continue categorymembers cmcontinue} continue_task] != 0} {
# no more continuations, we're done
break
}
dict set this_query cmcontinue $continue_task
}
puts " found [llength $tasks] tasks in [expr {[clock milliseconds] - $start}] milliseconds"
return $tasks
}
# This proc can be replaced by a single regexp command:
# set count [regexp -all "***=$needle" $haystack]
# However this proc is more efficient -- we're dealing with plain strings only.
proc count_substrings {needle haystack} {
set count 0
set idx 0
while {[set idx [string first $needle $haystack $idx]] != -1} {
incr count
incr idx
}
return $count
}
set total 0
foreach task [get_tasks Programming_Tasks] {
set url [format "http://www.rosettacode.org/w/index.php?title=%s&action=raw" [string map {{ } _} $task]]
set response [http::geturl $url]
if {[set s [http::status $response]] ne "ok" || [http::ncode $response] != 200} {
error "Oops: url=$url\nstatus=$s\nhttp code=[http::code $response]"
}
set count [count_substrings "\{\{header|" [http::data $response]]
puts [format "%3d examples in %s" $count $task]
http::cleanup $response
incr total $count
}
puts "\nTotal: $total examples"

View file

@ -0,0 +1,20 @@
#!/usr/bin/env bash
SITE=https://www.rosettacode.org
API=$SITE/mw/api.php
PAGES=$SITE/mw/index.php
query="$API?action=query"
query+=$(printf '&%s' \
list=categorymembers \
cmtitle=Category:Programming_Tasks \
cmlimit=500)
total=0
while read title; do
t=${title// /_}
tasks=$(curl -s "$PAGES?title=$t&action=raw" \
| grep -c '{{header|')
printf '%s: %d examples.\n' "$title" "$tasks"
let total+=tasks
done < <(curl -s "$query&format=json" \
| jq -r '.query.categorymembers[].title')
printf '\nTotal: %d examples.\n' "$total"

View file

@ -0,0 +1,52 @@
/* rc_count_examples.wren */
import "./pattern" for Pattern
var CURLOPT_URL = 10002
var CURLOPT_FOLLOWLOCATION = 52
var CURLOPT_WRITEFUNCTION = 20011
var CURLOPT_WRITEDATA = 10001
foreign class Buffer {
construct new() {} // C will allocate buffer of a suitable size
foreign value // returns buffer contents as a string
}
foreign class Curl {
construct easyInit() {}
foreign easySetOpt(opt, param)
foreign easyPerform()
foreign easyCleanup()
}
var curl = Curl.easyInit()
var getContent = Fn.new { |url|
var buffer = Buffer.new()
curl.easySetOpt(CURLOPT_URL, url)
curl.easySetOpt(CURLOPT_FOLLOWLOCATION, 1)
curl.easySetOpt(CURLOPT_WRITEFUNCTION, 0) // write function to be supplied by C
curl.easySetOpt(CURLOPT_WRITEDATA, buffer)
curl.easyPerform()
return buffer.value
}
var url = "https://www.rosettacode.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Programming_Tasks&cmlimit=500&format=xml"
var content = getContent.call(url)
var p = Pattern.new("title/=\"[+1^\"]\"")
var matches = p.findAll(content)
for (m in matches) {
var title = m.capsText[0].replace("&#039;", "'").replace("&quot;", "\"")
var title2 = title.replace(" ", "_").replace("+", "\%2B")
var taskUrl = "https://www.rosettacode.org/w/index.php?title=%(title2)&action=raw"
var taskContent = getContent.call(taskUrl)
var lines = taskContent.split("\n")
var count = lines.count { |line| line.trim().startsWith("=={{header|") }
System.print("%(title) : %(count) examples")
}
curl.easyCleanup()

View file

@ -0,0 +1,191 @@
/* gcc rc_count_examples.c -o rc_count_examples -lcurl -lwren -lm */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <curl/curl.h>
#include "wren.h"
struct MemoryStruct {
char *memory;
size_t size;
};
/* C <=> Wren interface functions */
static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) {
size_t realsize = size * nmemb;
struct MemoryStruct *mem = (struct MemoryStruct *)userp;
char *ptr = realloc(mem->memory, mem->size + realsize + 1);
if(!ptr) {
/* out of memory! */
printf("not enough memory (realloc returned NULL)\n");
return 0;
}
mem->memory = ptr;
memcpy(&(mem->memory[mem->size]), contents, realsize);
mem->size += realsize;
mem->memory[mem->size] = 0;
return realsize;
}
void C_bufferAllocate(WrenVM* vm) {
struct MemoryStruct *ms = (struct MemoryStruct *)wrenSetSlotNewForeign(vm, 0, 0, sizeof(struct MemoryStruct));
ms->memory = malloc(1);
ms->size = 0;
}
void C_bufferFinalize(void* data) {
struct MemoryStruct *ms = (struct MemoryStruct *)data;
free(ms->memory);
}
void C_curlAllocate(WrenVM* vm) {
CURL** pcurl = (CURL**)wrenSetSlotNewForeign(vm, 0, 0, sizeof(CURL*));
*pcurl = curl_easy_init();
}
void C_value(WrenVM* vm) {
struct MemoryStruct *ms = (struct MemoryStruct *)wrenGetSlotForeign(vm, 0);
wrenSetSlotString(vm, 0, ms->memory);
}
void C_easyPerform(WrenVM* vm) {
CURL* curl = *(CURL**)wrenGetSlotForeign(vm, 0);
curl_easy_perform(curl);
}
void C_easyCleanup(WrenVM* vm) {
CURL* curl = *(CURL**)wrenGetSlotForeign(vm, 0);
curl_easy_cleanup(curl);
}
void C_easySetOpt(WrenVM* vm) {
CURL* curl = *(CURL**)wrenGetSlotForeign(vm, 0);
CURLoption opt = (CURLoption)wrenGetSlotDouble(vm, 1);
if (opt < 10000) {
long lparam = (long)wrenGetSlotDouble(vm, 2);
curl_easy_setopt(curl, opt, lparam);
} else if (opt < 20000) {
if (opt == CURLOPT_WRITEDATA) {
struct MemoryStruct *ms = (struct MemoryStruct *)wrenGetSlotForeign(vm, 2);
curl_easy_setopt(curl, opt, (void *)ms);
} else if (opt == CURLOPT_URL) {
const char *url = wrenGetSlotString(vm, 2);
curl_easy_setopt(curl, opt, url);
}
} else if (opt < 30000) {
if (opt == CURLOPT_WRITEFUNCTION) {
curl_easy_setopt(curl, opt, &WriteMemoryCallback);
}
}
}
WrenForeignClassMethods bindForeignClass(WrenVM* vm, const char* module, const char* className) {
WrenForeignClassMethods methods;
methods.allocate = NULL;
methods.finalize = NULL;
if (strcmp(module, "main") == 0) {
if (strcmp(className, "Buffer") == 0) {
methods.allocate = C_bufferAllocate;
methods.finalize = C_bufferFinalize;
} else if (strcmp(className, "Curl") == 0) {
methods.allocate = C_curlAllocate;
}
}
return methods;
}
WrenForeignMethodFn bindForeignMethod(
WrenVM* vm,
const char* module,
const char* className,
bool isStatic,
const char* signature) {
if (strcmp(module, "main") == 0) {
if (strcmp(className, "Buffer") == 0) {
if (!isStatic && strcmp(signature, "value") == 0) return C_value;
} else if (strcmp(className, "Curl") == 0) {
if (!isStatic && strcmp(signature, "easySetOpt(_,_)") == 0) return C_easySetOpt;
if (!isStatic && strcmp(signature, "easyPerform()") == 0) return C_easyPerform;
if (!isStatic && strcmp(signature, "easyCleanup()") == 0) return C_easyCleanup;
}
}
return NULL;
}
static void writeFn(WrenVM* vm, const char* text) {
printf("%s", text);
}
void errorFn(WrenVM* vm, WrenErrorType errorType, const char* module, const int line, const char* msg) {
switch (errorType) {
case WREN_ERROR_COMPILE:
printf("[%s line %d] [Error] %s\n", module, line, msg);
break;
case WREN_ERROR_STACK_TRACE:
printf("[%s line %d] in %s\n", module, line, msg);
break;
case WREN_ERROR_RUNTIME:
printf("[Runtime Error] %s\n", msg);
break;
}
}
char *readFile(const char *fileName) {
FILE *f = fopen(fileName, "r");
fseek(f, 0, SEEK_END);
long fsize = ftell(f);
rewind(f);
char *script = malloc(fsize + 1);
fread(script, 1, fsize, f);
fclose(f);
script[fsize] = 0;
return script;
}
static void loadModuleComplete(WrenVM* vm, const char* module, WrenLoadModuleResult result) {
if( result.source) free((void*)result.source);
}
WrenLoadModuleResult loadModule(WrenVM* vm, const char* name) {
WrenLoadModuleResult result = {0};
if (strcmp(name, "random") != 0 && strcmp(name, "meta") != 0) {
result.onComplete = loadModuleComplete;
char fullName[strlen(name) + 6];
strcpy(fullName, name);
strcat(fullName, ".wren");
result.source = readFile(fullName);
}
return result;
}
int main(int argc, char **argv) {
WrenConfiguration config;
wrenInitConfiguration(&config);
config.writeFn = &writeFn;
config.errorFn = &errorFn;
config.bindForeignClassFn = &bindForeignClass;
config.bindForeignMethodFn = &bindForeignMethod;
config.loadModuleFn = &loadModule;
WrenVM* vm = wrenNewVM(&config);
const char* module = "main";
const char* fileName = "rc_count_examples.wren";
char *script = readFile(fileName);
WrenInterpretResult result = wrenInterpret(vm, module, script);
switch (result) {
case WREN_RESULT_COMPILE_ERROR:
printf("Compile Error!\n");
break;
case WREN_RESULT_RUNTIME_ERROR:
printf("Runtime Error!\n");
break;
case WREN_RESULT_SUCCESS:
break;
}
wrenFreeVM(vm);
free(script);
return 0;
}

View file

@ -0,0 +1,28 @@
var [const] YAJL=Import("zklYAJL")[0], CURL=Import("zklCurl");
fcn getTasks(language){
continueValue,tasks:="",Data(0,String); // "nm\0nm\0...."
do{
page:=CURL().get(("http://rosettacode.org/mw/api.php?"
"action=query&cmlimit=500"
"&format=json"
"&list=categorymembers"
"&cmtitle=Category:%s"
"&cmcontinue=%s").fmt(language,continueValue));
page=page[0].del(0,page[1]); // get rid of HTML header
json:=YAJL().write(page).close();
json["query"]["categorymembers"].pump(tasks,T("get","title"));
continueValue=json.find("continue") //continue:-||,cmcontinue:page|954|19)
.toList().apply("concat","=").concat("&");
}while(continueValue);
tasks
}
re:=RegExp(0'!\s+==\s*{{\s*header\s*\|!); // == {{ header | zkl
foreach task in (getTasks("Programming_Tasks")){
page:=CURL().get(
"http://www.rosettacode.org/mw/index.php?title=%s&action=raw"
.fmt(CURL.urlEncode(task)));
page=page[0].del(0,page[1]); // get rid of HTML header
cnt,n:=0,0; while(re.search(page,True,n)){ cnt+=1; n=re.matched[0].sum(0); }
"%4d: %s".fmt(cnt,task).println();
}