Data commit

Task/Web-scraping/00-META.yaml

@@ -0,0 +1,5 @@
+---
+category:
+- Input_Output
+from: http://rosettacode.org/wiki/Web_scraping
+note: Networking and Web Interaction

Task/Web-scraping/00-TASK.txt

@@ -0,0 +1,28 @@
+;Task:
+Create a program that downloads the time from this URL:   [http://tycho.usno.navy.mil/cgi-bin/timer.pl http://tycho.usno.navy.mil/cgi-bin/timer.pl]   and then prints the current UTC time by extracting just the UTC time from the web page's [[HTML]]. Alternatively, if the above url is not working, grab the first date/time off this page's talk page.
+
+<!-- As of March 2014, the page is available
+{{task|Networking and Web Interaction}}
+
+The page http://tycho.usno.navy.mil/cgi-bin/timer.pl is no longer available since July 2011. 
+The relevant part of that page source looked like this:
+<pre>
+...
+
+<TITLE>What time is it?</TITLE>
+<H2> US Naval Observatory Master Clock Time</H2> <H3><PRE>
+<BR>Jul. 27, 22:57:22 UTC   Universal Time
+<BR>Jul. 27, 06:57:22 PM EDT  Eastern Time
+<BR>Jul. 27, 05:57:22 PM CDT  Central Time
+<BR>Jul. 27, 04:57:22 PM MDT  Mountain Time
+<BR>Jul. 27, 03:57:22 PM PDT  Pacific Time
+<BR>Jul. 27, 02:57:22 PM AKDT Alaska Time
+<BR>Jul. 27, 12:57:22 PM HAST Hawaii-Aleutian Time
+
+...
+</pre>
+End of comment -->
+
+If possible, only use libraries that come at no ''extra'' monetary cost with the programming language and that are widely available and popular such as [http://www.cpan.org/ CPAN] for Perl or [[Boost]] for C++.
+<br><br>
+

Task/Web-scraping/8th/web-scraping.8th

@@ -0,0 +1,14 @@
+\ Web-scrape sample: get UTC time from the US Naval Observatory:
+: read-url \ -- s
+ "http://tycho.usno.navy.mil/cgi-bin/timer.pl" net:get
+ not if "Could not connect" throw then
+ >s ;
+
+: get-time
+  read-url
+  /<BR>.*?(\d{2}:\d{2}:\d{2})\sUTC/
+  tuck r:match if
+    1 r:@ . cr
+  then ;
+
+get-time bye

Task/Web-scraping/ALGOL-68/web-scraping.alg

@@ -0,0 +1,42 @@
+STRING
+   domain="tycho.usno.navy.mil",
+   page="cgi-bin/timer.pl";
+
+STRING # search for the needle in the haystack #
+   needle = "UTC",
+   hay stack = "http://"+domain+"/"+page,
+
+   re success="^HTTP/[0-9.]* 200",
+   re result description="^HTTP/[0-9.]* [0-9]+ [a-zA-Z ]*",
+   re doctype ="\s\s<![Dd][Oo][Cc][Tt][Yy][Pp][Ee] [^>]+>\s+";
+
+PROC raise error = (STRING msg)VOID: ( put(stand error, (msg, new line)); stop);
+
+PROC is html page = (REF STRING page) BOOL: (
+     BOOL out=grep in string(re success, page, NIL, NIL) = 0;
+     IF INT start, end;
+        grep in string(re result description, page, start, end) = 0
+     THEN
+        page:=page[end+1:];
+        IF grep in string(re doctype, page, start, end) = 0
+        THEN page:=page[start+2:]
+        ELSE raise error("unknown format retrieving page")
+        FI
+     ELSE raise error("unknown error retrieving page")
+     FI;
+     out
+);
+
+STRING reply;
+INT rc = http content (reply, domain, haystack, 0);
+IF rc = 0 AND is html page (reply)
+THEN
+  STRING line; FILE freply; associate(freply, reply);
+  on logical file end(freply, (REF FILE freply)BOOL: (done; SKIP));
+  DO
+    get(freply,(line, new line));
+    IF string in string(needle, NIL, line) THEN print((line, new line)) FI
+  OD;
+  done: SKIP
+ELSE raise error (strerror (rc))
+FI

Task/Web-scraping/AWK/web-scraping.awk

@@ -0,0 +1,21 @@
+#! /usr/bin/awk -f
+
+BEGIN {
+  purl = "/inet/tcp/0/tycho.usno.navy.mil/80"
+  ORS = RS = "\r\n\r\n"
+  print "GET /cgi-bin/timer.pl HTTP/1.0" |& purl
+  purl |& getline header
+  while ( (purl |& getline ) > 0 )
+  {
+     split($0, a, "\n")
+     for(i=1; i <= length(a); i++)
+     {
+        if ( a[i] ~ /UTC/ )
+        {
+          sub(/^<BR>/, "", a[i])
+          printf "%s\n", a[i]
+        }
+     }
+  }
+  close(purl)
+}

Task/Web-scraping/Ada/web-scraping.ada

@@ -0,0 +1,34 @@
+with AWS.Client, AWS.Response, AWS.Resources, AWS.Messages;
+with Ada.Text_IO, Ada.Strings.Fixed;
+use  Ada, AWS, AWS.Resources, AWS.Messages;
+
+procedure Get_UTC_Time is
+
+   Page           : Response.Data;
+   File           : Resources.File_Type;
+   Buffer         : String (1 .. 1024);
+   Position, Last : Natural := 0;
+   S              : Messages.Status_Code;
+begin
+   Page := Client.Get ("http://tycho.usno.navy.mil/cgi-bin/timer.pl");
+   S    := Response.Status_Code (Page);
+   if S not  in Success then
+      Text_IO.Put_Line
+        ("Unable to retrieve data => Status Code :" & Image (S) &
+         " Reason :" & Reason_Phrase (S));
+      return;
+   end if;
+
+   Response.Message_Body (Page, File);
+   while not End_Of_File (File) loop
+      Resources.Get_Line (File, Buffer, Last);
+      Position :=
+         Strings.Fixed.Index
+           (Source  => Buffer (Buffer'First .. Last),
+            Pattern => "UTC");
+      if Position > 0 then
+         Text_IO.Put_Line (Buffer (5 .. Position + 2));
+         return;
+      end if;
+   end loop;
+end Get_UTC_Time;

Task/Web-scraping/App-Inventor/web-scraping.app

@@ -0,0 +1,9 @@
+when ScrapeButton.Click do
+  set ScrapeWeb.Url to SourceTextBox.Text
+  call ScrapeWeb.Get
+
+when ScrapeWeb.GotText url,responseCode,responseType,responseContent do
+  initialize local Left to split at first text (text: get responseContent, at: PreTextBox.Text)
+  initialize local Right to "" in
+    set Right to select list item (list: get Left, index: 2)
+    set ResultLabel.Text to select list item (list: split at first (text:get Right, at: PostTextBox.Text), index: 1)

Task/Web-scraping/AppleScript/web-scraping-1.applescript

@@ -0,0 +1,33 @@
+use AppleScript version "2.4" -- OS X 10.10 (Yosemite) or later
+use framework "Foundation"
+
+on firstDateonWebPage(URLText)
+    set |⌘| to current application
+    set pageURL to |⌘|'s class "NSURL"'s URLWithString:(URLText)
+    -- Fetch the page HTML as data.
+    -- The Xcode documentation advises against using dataWithContentsOfURL: over a network,
+    -- but I'm guessing this applies to downloading large files rather than a Web page's HTML.
+    -- If in doubt, the HTML can be fetched as text instead and converted to data in house.
+    set HTMLData to |⌘|'s class "NSData"'s dataWithContentsOfURL:(pageURL)
+    -- Or:
+    (*
+    set {HTMLText, encoding} to |⌘|'s class "NSString"'s stringWithContentsOfURL:(pageURL) ¬
+        usedEncoding:(reference) |error|:(missing value)
+    set HTMLData to HTMLText's dataUsingEncoding:(encoding)
+    *)
+
+    -- Extract the page's visible text from the HTML.
+    set straightText to (|⌘|'s class "NSAttributedString"'s alloc()'s initWithHTML:(HTMLData) ¬
+        documentAttributes:(missing value))'s |string|()
+
+    -- Use an NSDataDetector to locate the first date in the text. (It's assumed here there'll be one.)
+    set dateDetector to |⌘|'s class "NSDataDetector"'s dataDetectorWithTypes:(|⌘|'s NSTextCheckingTypeDate) ¬
+        |error|:(missing value)
+    set matchRange to dateDetector's rangeOfFirstMatchInString:(straightText) options:(0) ¬
+        range:({0, straightText's |length|()})
+
+    -- Return the date text found.
+    return (straightText's substringWithRange:(matchRange)) as text
+end firstDateonWebPage
+
+firstDateonWebPage("https://www.rosettacode.org/wiki/Talk:Web_scraping")

Task/Web-scraping/AppleScript/web-scraping-2.applescript

@@ -0,0 +1 @@
+"20:59, 30 May 2020"

Task/Web-scraping/AutoHotkey/web-scraping.ahk

@@ -0,0 +1,4 @@
+UrlDownloadToFile, http://tycho.usno.navy.mil/cgi-bin/timer.pl, time.html
+FileRead, timefile, time.html
+pos := InStr(timefile, "UTC")
+msgbox % time := SubStr(timefile, pos - 9, 8)

Task/Web-scraping/BBC-BASIC/web-scraping.basic

@@ -0,0 +1,18 @@
+      SYS "LoadLibrary", "URLMON.DLL" TO urlmon%
+      SYS "GetProcAddress", urlmon%, "URLDownloadToFileA" TO UDTF%
+      SYS "LoadLibrary", "WININET.DLL" TO wininet%
+      SYS "GetProcAddress", wininet%, "DeleteUrlCacheEntryA" TO DUCE%
+
+      url$ = "http://tycho.usno.navy.mil/cgi-bin/timer.pl"
+      file$ = @tmp$+"navytime.txt"
+
+      SYS DUCE%, url$
+      SYS UDTF%, 0, url$, file$, 0, 0 TO result%
+      IF result% ERROR 100, "Download failed"
+
+      file% = OPENIN(file$)
+      REPEAT
+        text$ = GET$#file%
+        IF INSTR(text$, "UTC") PRINT MID$(text$, 5)
+      UNTIL EOF#file%
+      CLOSE #file%

Task/Web-scraping/C++/web-scraping.cpp

@@ -0,0 +1,23 @@
+#include <iostream>
+#include <string>
+#include <boost/asio.hpp>
+#include <boost/regex.hpp>
+int main()
+{
+    boost::asio::ip::tcp::iostream s("tycho.usno.navy.mil", "http");
+    if(!s)
+        std::cout << "Could not connect to tycho.usno.navy.mil\n";
+    s  << "GET /cgi-bin/timer.pl HTTP/1.0\r\n"
+       << "Host: tycho.usno.navy.mil\r\n"
+       << "Accept: */*\r\n"
+       << "Connection: close\r\n\r\n" ;
+    for(std::string line; getline(s, line); )
+    {
+        boost::smatch matches;
+        if(regex_search(line, matches, boost::regex("<BR>(.+\\s+UTC)") ) )
+        {
+            std::cout << matches[1] << '\n';
+            break;
+        }
+    }
+}

Task/Web-scraping/C-sharp/web-scraping.cs

@@ -0,0 +1,24 @@
+class Program
+    {
+        static void Main(string[] args)
+        {
+            WebClient wc = new WebClient();
+            Stream myStream = wc.OpenRead("http://tycho.usno.navy.mil/cgi-bin/timer.pl");
+            string html = "";
+            using (StreamReader sr = new StreamReader(myStream))
+            {
+                while (sr.Peek() >= 0)
+                {
+                    html = sr.ReadLine();
+                    if (html.Contains("UTC"))
+                    {
+                        break;
+                    }
+                }
+
+            }
+            Console.WriteLine(html.Remove(0, 4));
+
+            Console.ReadLine();
+        }
+    }

Task/Web-scraping/C/web-scraping.c

@@ -0,0 +1,48 @@
+#include <stdio.h>
+#include <string.h>
+#include <curl/curl.h>
+#include <sys/types.h>
+#include <regex.h>
+
+#define BUFSIZE 16384
+
+size_t lr = 0;
+
+size_t filterit(void *ptr, size_t size, size_t nmemb, void *stream)
+{
+  if ( (lr + size*nmemb) > BUFSIZE ) return BUFSIZE;
+  memcpy(stream+lr, ptr, size*nmemb);
+  lr += size*nmemb;
+  return size*nmemb;
+}
+
+int main()
+{
+  CURL *curlHandle;
+  char buffer[BUFSIZE];
+  regmatch_t amatch;
+  regex_t cregex;
+
+  curlHandle = curl_easy_init();
+  curl_easy_setopt(curlHandle, CURLOPT_URL, "http://tycho.usno.navy.mil/cgi-bin/timer.pl");
+  curl_easy_setopt(curlHandle, CURLOPT_FOLLOWLOCATION, 1);
+  curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, filterit);
+  curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, buffer);
+  int success = curl_easy_perform(curlHandle);
+  curl_easy_cleanup(curlHandle);
+
+  buffer[lr] = 0;
+
+  regcomp(&cregex, " UTC", REG_NEWLINE);
+  regexec(&cregex, buffer, 1, &amatch, 0);
+  int bi = amatch.rm_so;
+  while ( bi-- > 0 )
+    if ( memcmp(&buffer[bi], "<BR>", 4) == 0 ) break;
+
+  buffer[amatch.rm_eo] = 0;
+
+  printf("%s\n", &buffer[bi+4]);
+
+  regfree(&cregex);
+  return 0;
+}

Task/Web-scraping/Ceylon/web-scraping.ceylon

@@ -0,0 +1,31 @@
+import ceylon.uri {
+    parse
+}
+import ceylon.http.client {
+    get
+}
+
+shared void run() {
+
+    // apparently the cgi link is deprecated?
+    value oldUri = "http://tycho.usno.navy.mil/cgi-bin/timer.pl";
+    value newUri = "http://tycho.usno.navy.mil/timer.pl";
+
+    value contents = downloadContents(newUri);
+    value time = extractTime(contents);
+    print(time else "nothing found");
+}
+
+String downloadContents(String uriString) {
+    value uri = parse(uriString);
+    value request = get(uri);
+    value response = request.execute();
+    return response.contents;
+}
+
+String? extractTime(String contents) =>
+        contents
+        .lines
+        .filter((String element) => element.contains("UTC"))
+        .first
+        ?.substring(4, 21);

Task/Web-scraping/Clojure/web-scraping.clj

@@ -0,0 +1 @@
+(second (re-find #" (\d{1,2}:\d{1,2}:\d{1,2}) UTC" (slurp "http://tycho.usno.navy.mil/cgi-bin/timer.pl")))

Task/Web-scraping/CoffeeScript/web-scraping.coffee

@@ -0,0 +1,39 @@
+http = require 'http'
+
+CONFIG =
+  host: 'tycho.usno.navy.mil'
+  path: '/cgi-bin/timer.pl'
+
+# Web scraping code tends to be brittle, and this is no exception.
+# The tycho time page does not use highly structured markup, so
+# we do a real dirty scrape.
+scrape_tycho_ust_time = (text) ->
+  for line in text.split '\n'
+    matches = line.match /(.*:\d\d UTC)/
+    if matches
+      console.log matches[0].replace '<BR>', ''
+      return
+  throw Error("unscrapable page!")
+
+# This is low-level-ish code to get data from a URL. It's
+# pretty general purpose, so you'd normally tuck this away
+# in a library (or use somebody else's library).
+wget = (host, path, cb) ->
+  options =
+    host: host
+    path: path
+    headers:
+      "Cache-Control": "max-age=0"
+
+  req = http.request options, (res) ->
+    s = ''
+    res.on 'data', (chunk) ->
+      s += chunk
+    res.on 'end', ->
+      cb s
+  req.end()
+
+# Do our web scrape
+do ->
+  wget CONFIG.host, CONFIG.path, (data) ->
+    scrape_tycho_ust_time data

Task/Web-scraping/Common-Lisp/web-scraping-1.lisp

@@ -0,0 +1,10 @@
+BOA> (let* ((url "http://tycho.usno.navy.mil/cgi-bin/timer.pl")
+            (regexp (load-time-value
+                     (cl-ppcre:create-scanner "(?m)^.{4}(.+? UTC)")))
+            (data (drakma:http-request url)))
+       (multiple-value-bind (start end start-regs end-regs)
+           (cl-ppcre:scan regexp data)
+         (declare (ignore end))
+         (when start
+           (subseq data (aref start-regs 0) (aref end-regs 0)))))
+"Aug. 12, 04:29:51 UTC"

Task/Web-scraping/Common-Lisp/web-scraping-2.lisp

@@ -0,0 +1,5 @@
+CL-USER> (cl-ppcre:do-matches-as-strings
+             (m ".*<BR>(.*)UTC.*"
+                (drakma:http-request "http://tycho.usno.navy.mil/cgi-bin/timer.pl"))
+           (print (cl-ppcre:regex-replace "<BR>(.*UTC).*" m "\\1")))
+"Jul. 13, 06:32:01 UTC"

Task/Web-scraping/D/web-scraping.d

@@ -0,0 +1,7 @@
+void main() {
+    import std.stdio, std.string, std.net.curl, std.algorithm;
+
+    foreach (line; "http://tycho.usno.navy.mil/cgi-bin/timer.pl".byLine)
+        if (line.canFind(" UTC"))
+            line[4 .. $].writeln;
+}

Task/Web-scraping/Delphi/web-scraping-1.delphi

@@ -0,0 +1,206 @@
+program WebScrape;
+
+{$APPTYPE CONSOLE}
+
+{.$DEFINE DEBUG}
+
+uses
+  Classes,
+  Winsock;
+
+
+{ Function to connect to host, send HTTP request and retrieve response }
+function DoHTTPGET(const hostName: PAnsiChar; const resource: PAnsiChar; HTTPResponse: TStrings): Boolean;
+const
+  Port: integer = 80;
+  CRLF = #13#10; // carriage return/line feed
+var
+  WSAData: TWSAData;
+  Sock: TSocket;
+  SockAddrIn: TSockAddrIn;
+  IPAddress: PHostEnt;
+  bytesIn: integer;
+  inBuffer: array [0..1023] of char;
+  Req: string;
+begin
+  Result := False;
+  HTTPResponse.Clear;
+
+  { Initialise use of the Windows Sockets DLL.
+    Older Windows versions support Winsock 1.1 whilst newer Windows
+    include Winsock 2 but support 1.1.  Therefore, we'll specify
+    version 1.1 ($101) as being the highest version of Windows Sockets
+    that we can use to provide greatest flexibility.
+    WSAData receives details of the Windows Sockets implementation }
+  Winsock.WSAStartUp($101, WSAData);
+  try
+
+    { Create a socket for TCP/IP usage passing in
+      Address family spec: AF_INET (TCP, UDP, etc.)
+      Type specification: SOCK_STREAM
+      Protocol: IPPROTO_TCP (TCP) }
+    Sock := WinSock.Socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
+    try
+
+      // Check we have a valid socket
+      if (Sock <> INVALID_SOCKET) then
+        begin
+          // Populate socket address structure
+          with SockAddrIn do
+            begin
+              // Address family specification
+              sin_family := AF_INET;
+              // Port
+              sin_port := htons(Port);
+              // Address
+              sin_addr.s_addr := inet_addr(hostName);
+            end;
+
+          if (SockAddrIn.sin_addr.s_addr = INADDR_NONE) then
+            begin
+              { As we're using a domain name instead of an
+                IP Address, we need to resolve the domain name }
+              IPAddress := Winsock.gethostbyname(hostName);
+
+              // Quit if we didn't get an IP Address
+              if (IPAddress = nil) then
+                Exit;
+
+              // Update the structure with the IP Address
+              SockAddrIn.sin_addr.s_addr := PLongint(IPAddress^.h_addr_list^)^;
+            end;
+
+          // Try to connect to host
+          if (Winsock.connect(Sock, SockAddrIn, SizeOf(SockAddrIn)) <> SOCKET_ERROR) then
+            begin
+              // OK - Connected
+
+              // Compose our request
+              // Each line of the request must be terminated with a carriage return/line feed
+
+              {  The First line specifies method (e.g. GET, POST), path to required resource,
+                 and the HTTP version being used.  These three fields are space separated. }
+               Req := 'GET '+resource+' HTTP/1.1' + CRLF +
+
+                     // Host: is the only Required header in HTTP 1.1
+                     'Host: '+hostName + CRLF +
+
+                     { Persistent connections are the default in HTTP 1.1 but, as we don't want
+                       or need one for this exercise, we must include the "Connection: close"
+                       header in our request }
+                     'Connection: close' + CRLF +
+
+                     CRLF; // Request must end with an empty line!
+
+              // Try to send the request to the host
+              if (Winsock.send(Sock,Req[1],Length(Req),0) <> SOCKET_ERROR) then
+                begin
+                  // Initialise incoming data buffer (i.e. fill array with nulls)
+                  FillChar(inBuffer,SizeOf(inBuffer),#0);
+                  // Loop until nothing left to read
+                  repeat
+                    // Read incoming data from socket
+                    bytesIn := Winsock.recv(Sock, inBuffer, SizeOf(inBuffer), 0);
+                    // Assign buffer to Stringlist
+                    HTTPResponse.Text := HTTPResponse.Text + Copy(string(inBuffer),1,bytesIn);
+                  until
+                    (bytesIn <= 0) or (bytesIn = SOCKET_ERROR);
+
+                  { Our list of response strings should
+                    contain at least 1 line }
+                  Result := HTTPResponse.Count > 0;
+                end;
+
+             end;
+
+        end;
+
+    finally
+      // Close our socket
+      Winsock.closesocket(Sock);
+    end;
+
+  finally
+    { This causes our application to deregister itself from this
+      Windows Sockets implementation and allows the implementation
+      to free any resources allocated on our behalf. }
+    Winsock.WSACleanup;
+  end;
+
+end;
+
+{ Simple function to locate and return the UTC time from the
+  request sent to http://tycho.usno.navy.mil/cgi-bin/timer.pl
+  The HTTPResponse parameter contains both the HTTP Headers and
+  the HTML served up by the requested resource. }
+function ParseResponse(HTTPResponse: TStrings): string;
+var
+  i: Integer;
+begin
+  Result := '';
+
+  { Check first line for server response code
+    We want something like this: HTTP/1.1 200 OK }
+  if Pos('200',HTTPResponse[0]) > 0 then
+    begin
+      for i := 0 to Pred(HTTPResponse.Count) do
+        begin
+          { The line we're looking for is something like this:
+            <BR>May. 04. 21:55:19 UTC Universal Time }
+
+          // Check each line
+          if Pos('UTC',HTTPResponse[i]) > 0 then
+            begin
+              Result := Copy(HTTPResponse[i],5,Pos('UTC',HTTPResponse[i])-1);
+              Break;
+            end;
+
+        end;
+    end
+    else
+    Result := 'HTTP Error: '+HTTPResponse[0];
+end;
+
+
+const
+  host: PAnsiChar = 'tycho.usno.navy.mil';
+  res : PAnsiChar = '/cgi-bin/timer.pl';
+
+
+var
+  Response: TStrings;
+
+begin
+  { A TStringList is a TStrings descendant class
+    that is used to store and manipulate a list
+    of strings.
+
+    Instantiate a stringlist class to
+    hold the results of our HTTP GET }
+  Response := TStringList.Create;
+  try
+    // Try an HTTP GET request
+    if DoHTTPGET(host,res,Response) then
+      begin
+        {$IFDEF DEBUG}
+        { Write the entire response to
+          the console window }
+        Writeln(Response.text);
+        {$ELSE}
+        { Parse the response and write the
+          result to the console window }
+        Writeln(ParseResponse(Response));
+        {$ENDIF DEBUG}
+      end
+      else
+      Writeln('Error retrieving data');
+
+  finally
+    Response.Free;
+  end;
+
+  // Keep console window open
+  Readln;
+
+
+end.

Task/Web-scraping/Delphi/web-scraping-2.delphi

@@ -0,0 +1,28 @@
+program ReadUTCTime;
+
+{$APPTYPE CONSOLE}
+
+uses SysUtils, Classes, IdHTTP;
+
+var
+  s: string;
+  lHTTP: TIdHTTP;
+  lReader: TStringReader;
+begin
+  lHTTP := TIdHTTP.Create(nil);
+  try
+    lReader := TStringReader.Create(lHTTP.Get('http://tycho.usno.navy.mil/cgi-bin/timer.pl'));
+    while lReader.Peek > 0 do
+    begin
+      s := lReader.ReadLine;
+      if Pos('UTC', s) > 0 then
+      begin
+        Writeln(s);
+        Break;
+      end;
+    end;
+  finally
+    lHTTP.Free;
+    lReader.Free;
+  end;
+end.

Task/Web-scraping/E/web-scraping.e

@@ -0,0 +1,4 @@
+interp.waitAtTop(when (def html := <http://tycho.usno.navy.mil/cgi-bin/timer.pl>.getText()) -> {
+    def rx`(?s).*>(@time.*? UTC).*` := html
+    println(time)
+})

Task/Web-scraping/Erlang/web-scraping.erl

@@ -0,0 +1,10 @@
+-module(scraping).
+-export([main/0]).
+-define(Url, "http://tycho.usno.navy.mil/cgi-bin/timer.pl").
+-define(Match, "<BR>(.+ UTC)").
+
+main() ->
+  inets:start(),
+  {ok, {_Status, _Header, HTML}} = httpc:request(?Url),
+  {match, [Time]} = re:run(HTML, ?Match, [{capture, all_but_first, binary}]),
+  io:format("~s~n",[Time]).

Task/Web-scraping/F-Sharp/web-scraping.fs

@@ -0,0 +1,11 @@
+open System
+open System.Net
+open System.Text.RegularExpressions
+
+async {
+    use wc = new WebClient()
+    let! html = wc.AsyncDownloadString(Uri("http://tycho.usno.navy.mil/cgi-bin/timer.pl"))
+    return Regex.Match(html, @"<BR>(.+ UTC)").Groups.[1].Value
+}
+|> Async.RunSynchronously
+|> printfn "%s"

Task/Web-scraping/Factor/web-scraping.factor

@@ -0,0 +1,4 @@
+USING: http.client io sequences ;
+
+"http://tycho.usno.navy.mil/cgi-bin/timer.pl" http-get nip
+[ "UTC" swap start [ 9 - ] [ 1 - ] bi ] keep subseq print

Task/Web-scraping/Forth/web-scraping.fth

@@ -0,0 +1,15 @@
+include unix/socket.fs
+
+: extract-time ( addr len type len -- time len )
+  dup >r
+  search 0= abort" that time not present!"
+  dup >r
+  begin -1 /string  over 1- c@ [char] > = until       \ seek back to <BR> at start of line
+  r> - r> + ;
+
+s" tycho.usno.navy.mil" 80 open-socket
+dup s\" GET /cgi-bin/timer.pl HTTP/1.0\n\n" rot write-socket
+dup pad 4096 read-socket
+s\" \r\n\r\n" search 0= abort" can't find headers!"   \ skip headers
+s" UTC" extract-time type cr
+close-socket

Task/Web-scraping/FunL/web-scraping.funl

@@ -0,0 +1,5 @@
+import io.Source
+
+case Source.fromURL( 'http://tycho.usno.navy.mil/cgi-bin/timer.pl', 'UTF-8' ).getLines().find( ('Eastern' in) ) of
+  Some( time ) -> println( time.substring(4) )
+  None         -> error( 'Easter time not found' )

Task/Web-scraping/Gambas/web-scraping.gambas

@@ -0,0 +1,17 @@
+Public Sub Main()
+Dim sWeb, sTemp, sOutput As String                                          'Variables
+
+Shell "wget -O /tmp/web http://tycho.usno.navy.mil/cgi-bin/timer.pl" Wait   'Use 'wget' to save the web file in /tmp/
+
+sWeb = File.Load("/tmp/web")                                                'Open file and store in sWeb
+
+For Each sTemp In Split(sWeb, gb.NewLine)                                   'Split the file by NewLines..
+  If InStr(sTemp, "UTC") Then                                               'If the line contains "UTC" then..
+    sOutPut = sTemp                                                         'Extract the line into sOutput
+    Break                                                                   'Get out of here
+  End If
+Next
+
+Print Mid(sOutput, 5)                                                       'Print the result without the '<BR>' tag
+
+End

Task/Web-scraping/Go/web-scraping.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+    "bytes"
+    "encoding/xml"
+    "fmt"
+    "io"
+    "net/http"
+    "regexp"
+    "time"
+)
+
+func main() {
+    resp, err := http.Get("http://tycho.usno.navy.mil/cgi-bin/timer.pl")
+    if err != nil {
+        fmt.Println(err) // connection or request fail
+        return
+    }
+    defer resp.Body.Close()
+    var us string
+    var ux int
+    utc := []byte("UTC")
+    for p := xml.NewDecoder(resp.Body); ; {
+        t, err := p.RawToken()
+        switch err {
+        case nil:
+        case io.EOF:
+            fmt.Println("UTC not found")
+            return
+        default:
+            fmt.Println(err) // read or parse fail
+            return
+        }
+        if ub, ok := t.(xml.CharData); ok {
+            if ux = bytes.Index(ub, utc); ux != -1 {
+                // success: found a line with the string "UTC"
+                us = string([]byte(ub))
+                break
+            }
+        }
+    }
+    // first thing to try: parsing the expected date format
+    if t, err := time.Parse("Jan. 2, 15:04:05 UTC", us[:ux+3]); err == nil {
+        fmt.Println("parsed UTC:", t.Format("January 2, 15:04:05"))
+        return
+    }
+    // fallback: search for anything looking like a time and print that
+    tx := regexp.MustCompile("[0-2]?[0-9]:[0-5][0-9]:[0-6][0-9]")
+    if justTime := tx.FindString(us); justTime > "" {
+        fmt.Println("found UTC:", justTime)
+        return
+    }
+    // last resort: just print the whole element containing "UTC" and hope
+    // there is a human readable time in there somewhere.
+    fmt.Println(us)
+}

Task/Web-scraping/Groovy/web-scraping.groovy

@@ -0,0 +1,8 @@
+def time = "unknown"
+def text = new URL('http://tycho.usno.navy.mil/cgi-bin/timer.pl').eachLine { line ->
+    def matcher = (line =~ "<BR>(.+) UTC")
+    if (matcher.find()) {
+        time = matcher[0][1]
+    }
+}
+println "UTC Time was '$time'"

Task/Web-scraping/Haskell/web-scraping-1.hs

@@ -0,0 +1,7 @@
+import Data.List
+import Network.HTTP (simpleHTTP, getResponseBody, getRequest)
+
+tyd = "http://tycho.usno.navy.mil/cgi-bin/timer.pl"
+
+readUTC = simpleHTTP (getRequest tyd)>>=
+            fmap ((!!2).head.dropWhile ("UTC"`notElem`).map words.lines). getResponseBody>>=putStrLn

Task/Web-scraping/Haskell/web-scraping-2.hs

@@ -0,0 +1,2 @@
+*Main> readUTC
+08:30:23

Task/Web-scraping/J/web-scraping.j

@@ -0,0 +1,4 @@
+   require 'web/gethttp'
+
+   _8{. ' UTC' taketo gethttp 'http://tycho.usno.navy.mil/cgi-bin/timer.pl'
+04:32:44

Task/Web-scraping/Java/web-scraping-1.java

@@ -0,0 +1,15 @@
+String scrapeUTC() throws URISyntaxException, IOException {
+    String address = "http://tycho.usno.navy.mil/cgi-bin/timer.pl";
+    URL url = new URI(address).toURL();
+    try (BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()))) {
+        Pattern pattern = Pattern.compile("^.+? UTC");
+        Matcher matcher;
+        String line;
+        while ((line = reader.readLine()) != null) {
+            matcher = pattern.matcher(line);
+            if (matcher.find())
+                return matcher.group().replaceAll("<.+?>", "");
+        }
+    }
+    return null;
+}

Task/Web-scraping/Java/web-scraping-2.java

@@ -0,0 +1,38 @@
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public final class WebScraping {
+	
+    public static void main(String[] aArgs) {
+	    try {
+	        URI uri = new URI("https://www.rosettacode.org/wiki/Talk:Web_scraping").parseServerAuthority();	
+	        URL address = uri.toURL();
+	        HttpURLConnection connection = (HttpURLConnection) address.openConnection();
+	        BufferedReader reader = new BufferedReader( new InputStreamReader(connection.getInputStream()) );
+	
+	        final int responseCode = connection.getResponseCode();
+	        System.out.println("Response code: " + responseCode);
+	
+	        String line;
+	        while ( ! ( line = reader.readLine() ).contains("UTC") ) {
+	        	/* Empty block */
+	        }
+	
+	        final int index = line.indexOf("UTC");	
+	        System.out.println(line.substring(index - 16, index + 4));
+	
+	        reader.close();
+	        connection.disconnect();
+	    } catch (IOException ioe) {
+	        System.err.println("Error connecting to server: " + ioe.getCause());
+	    } catch (URISyntaxException use) {
+	    	System.err.println("Unable to connect to URI: " + use.getCause());
+		}
+	}
+
+}

Task/Web-scraping/JavaScript/web-scraping.js

@@ -0,0 +1,7 @@
+var req = new XMLHttpRequest();
+req.onload = function () {
+  var re = /[JFMASOND].+ UTC/; //beginning of month name to 'UTC'
+  console.log(this.responseText.match(re)[0]);
+};
+req.open('GET', 'http://tycho.usno.navy.mil/cgi-bin/timer.pl', true);
+req.send();

Task/Web-scraping/Jq/web-scraping-1.jq

@@ -0,0 +1,3 @@
+#!/bin/bash
+  curl -Ss 'http://tycho.usno.navy.mil/cgi-bin/timer.pl' |\
+    jq -R -r 'if index(" UTC") then .[4:] else empty end'

Task/Web-scraping/Jq/web-scraping-2.jq

@@ -0,0 +1,2 @@
+$ ./Web_scraping.jq
+Apr. 21, 05:19:32 UTC   Universal Time

Task/Web-scraping/Julia/web-scraping-1.julia

@@ -0,0 +1,22 @@
+using Requests, Printf
+
+function getusnotime()
+    const url = "http://tycho.usno.navy.mil/timer.pl"
+    s = try
+        get(url)
+        catch err
+        @sprintf "get(%s)\n   => %s" url err
+    end
+    isa(s, Requests.Response) || return (s, false)
+    t = match(r"(?<=<BR>)(.*?UTC)", readstring(s))
+    isa(t, RegexMatch) || return (@sprintf("raw html:\n %s", readstring(s)), false)
+    return (t.match, true)
+end
+
+(t, issuccess) = getusnotime();
+
+if issuccess
+    println("The USNO time is ", t)
+else
+    println("Failed to fetch UNSO time:\n", t)
+end

Task/Web-scraping/Julia/web-scraping-2.julia

@@ -0,0 +1,17 @@
+Failed to fetch UNSO time:
+raw html:
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final"//EN>
+<html>
+<body>
+<TITLE>What time is it?</TITLE>
+<H2> US Naval Observatory Master Clock Time</H2> <H3><PRE>
+<BR>Apr. 20, 17:55:31 UTC               Universal Time
+<BR>Apr. 20, 01:55:31 PM EDT            Eastern Time
+<BR>Apr. 20, 12:55:31 PM CDT            Central Time
+<BR>Apr. 20, 11:55:31 AM MDT            Mountain Time
+<BR>Apr. 20, 10:55:31 AM PDT            Pacific Time
+<BR>Apr. 20, 09:55:31 AM AKDT   Alaska Time
+<BR>Apr. 20, 07:55:31 AM HAST   Hawaii-Aleutian Time
+</PRE></H3><P><A HREF="http://www.usno.navy.mil"> US Naval Observatory</A>
+
+</body></html>

Task/Web-scraping/Kotlin/web-scraping.kotlin

@@ -0,0 +1,19 @@
+// version 1.1.3
+
+import java.net.URL
+import java.io.InputStreamReader
+import java.util.Scanner
+
+fun main(args: Array<String>) {
+    val url = URL("http://tycho.usno.navy.mil/cgi-bin/timer.pl")
+    val isr = InputStreamReader(url.openStream())
+    val sc = Scanner(isr)
+    while (sc.hasNextLine()) {
+        val line = sc.nextLine()
+        if ("UTC" in line) {
+            println(line.drop(4).take(17))
+            break
+        }
+    }
+    sc.close()
+}

Task/Web-scraping/Lasso/web-scraping.lasso

@@ -0,0 +1,27 @@
+/* have to be used
+local(raw_htmlstring = '<TITLE>What time is it?</TITLE>
+<H2> US Naval Observatory Master Clock Time</H2> <H3><PRE>
+<BR>Jul. 27, 22:57:22 UTC   Universal Time
+<BR>Jul. 27, 06:57:22 PM EDT  Eastern Time
+<BR>Jul. 27, 05:57:22 PM CDT  Central Time
+<BR>Jul. 27, 04:57:22 PM MDT  Mountain Time
+<BR>Jul. 27, 03:57:22 PM PDT  Pacific Time
+<BR>Jul. 27, 02:57:22 PM AKDT Alaska Time
+<BR>Jul. 27, 12:57:22 PM HAST Hawaii-Aleutian Time
+</PRE></H3>
+')
+*/
+
+// should be used
+local(raw_htmlstring = string(include_url('http://tycho.usno.navy.mil/cgi-bin/timer.pl')))
+
+local(
+  reg_exp = regexp(-find = `<br>(.*?) UTC`, -input = #raw_htmlstring, -ignorecase),
+  datepart_txt =  #reg_exp -> find ? #reg_exp -> matchstring(1) | string
+)
+
+#datepart_txt
+'<br />'
+// added bonus showing how parsed string can be converted to date object
+local(mydate = date(#datepart_txt, -format = `MMM'.' dd',' HH:mm:ss`))
+#mydate -> format(`YYYY-MM-dd HH:mm:ss`)

Task/Web-scraping/Liberty-BASIC/web-scraping.basic

@@ -0,0 +1,23 @@
+if DownloadToFile("http://tycho.usno.navy.mil/cgi-bin/timer.pl", DefaultDir$ + "\timer.htm") = 0 then
+        open DefaultDir$ + "\timer.htm" for input as #f
+                html$ = lower$(input$(#f, LOF(#f)))
+        close #f
+
+        a= instr( html$, "utc" )-1
+        print "UTC";mid$( html$, a-9,9)
+
+    end if
+
+end
+
+function DownloadToFile(urlfile$, localfile$)
+    open "URLmon" for dll as #url
+    calldll #url, "URLDownloadToFileA",_
+    0 as long,_         'null
+    urlfile$ as ptr,_   'url to download
+    localfile$ as ptr,_ 'save file name
+    0 as long,_         'reserved, must be 0
+    0 as long,_         'callback address, can be 0
+    DownloadToFile as ulong  '0=success
+    close #url
+end function

Task/Web-scraping/Lua/web-scraping.lua

@@ -0,0 +1,14 @@
+local http = require("socket.http")    -- Debian package is 'lua-socket'
+
+function scrapeTime (pageAddress, timeZone)
+    local page = http.request(pageAddress)
+    if not page then return "Cannot connect" end
+    for line in page:gmatch("[^<BR>]*") do
+        if line:match(timeZone) then
+            return line:match("%d+:%d+:%d+")
+        end
+    end
+end
+
+local url = "http://tycho.usno.navy.mil/cgi-bin/timer.pl"
+print(scrapeTime(url, "UTC"))

Task/Web-scraping/M2000-Interpreter/web-scraping.m2000

@@ -0,0 +1,29 @@
+Module Web_scraping {
+	Print "Web scraping"
+	function GetTime$(a$, what$="UTC") {
+		document a$ ' change string to document
+		find a$, what$  ' place data to stack
+		Read find_pos
+		if find_pos>0 then
+			read par_order, par_pos
+			b$=paragraph$(a$, par_order)
+			k=instr(b$,">")
+			if k>0 then if k<par_pos then b$=mid$(b$,k+1) :par_pos-=k
+			k=rinstr(b$,"<")
+			if k>0 then if k>par_pos then b$=Left(b$,k-1)
+			=b$
+		end if
+	}
+	declare msxml2 "MSXML2.XMLHTTP.6.0"
+	rem print type$(msxml2)="IXMLHTTPRequest"
+	Url$ = "http://tycho.usno.navy.mil/cgi-bin/timer.pl"
+	try ok {
+		method msxml2, "Open", "GET", url$, false
+		method msxml2,"Send"
+		with msxml2,"responseText" as txt$
+		Print GetTime$(txt$)
+	}
+	If error or not ok then Print Error$
+	declare msxml2 nothing
+}
+Web_scraping

Task/Web-scraping/MATLAB/web-scraping.m

@@ -0,0 +1,8 @@
+s  = urlread('http://tycho.usno.navy.mil/cgi-bin/timer.pl');
+ix = [findstr(s,'<BR>'), length(s)+1];
+for k = 2:length(ix)
+     tok = s(ix(k-1)+4:ix(k)-1);
+     if findstr(tok,'UTC')
+  disp(tok);
+     end;
+end;

Task/Web-scraping/MIRC-Scripting-Language/web-scraping.mirc

@@ -0,0 +1,23 @@
+alias utc {
+  sockclose UTC
+  sockopen UTC tycho.usno.navy.mil 80
+}
+
+on *:SOCKOPEN:UTC: {
+  sockwrite -n UTC GET /cgi-bin/timer.pl HTTP/1.1
+  sockwrite -n UTC Host: tycho.usno.navy.mil
+  sockwrite UTC $crlf
+}
+
+on *:SOCKREAD:UTC: {
+  sockread %UTC
+  while ($sockbr) {
+    if (<BR>*Universal Time iswm %UTC) {
+      echo -ag $remove(%UTC,<BR>,$chr(9),Universal Time)
+      unset %UTC
+      sockclose UTC
+      return
+    }
+    sockread %UTC
+  }
+}

Task/Web-scraping/Maple/web-scraping.maple

@@ -0,0 +1,2 @@
+text := URL:-Get("http://tycho.usno.navy.mil/cgi-bin/timer.pl"):
+printf(StringTools:-StringSplit(text,"<BR>")[2]);

Task/Web-scraping/Mathematica/web-scraping.math

@@ -0,0 +1,2 @@
+test = StringSplit[Import["http://tycho.usno.navy.mil/cgi-bin/timer.pl"], "\n"];
+Extract[test, Flatten@Position[StringFreeQ[test, "UTC"], False]]

Task/Web-scraping/Microsoft-Small-Basic/web-scraping.basic

@@ -0,0 +1,14 @@
+'Entered by AykayayCiti -- Earl L. Montgomery
+url_name = "http://tycho.usno.navy.mil/cgi-bin/timer.pl"
+url_data = Network.GetWebPageContents(url_name)
+find = "UTC"
+' the length from the UTC to the time is -18 so we need
+' to subtract from the UTC position
+pos = Text.GetIndexOf(url_data,find)-18
+result = Text.GetSubText(url_data,pos,(18+3)) 'plus 3 to add the UTC
+TextWindow.WriteLine(result)
+
+'you can eleminate a line of code by putting the
+' GetIndexOf insde the GetSubText
+'result2 = Text.GetSubText(url_data,Text.GetIndexOf(url_data,find)-18,(18+3))
+'TextWindow.WriteLine(result2)

Task/Web-scraping/NetRexx/web-scraping.netrexx

@@ -0,0 +1,36 @@
+/* NetRexx */
+options replace format comments java crossref symbols binary
+
+parse arg full_short .
+if 'FULL'.abbrev(full_short.upper(), 1) then
+  dateFull = isTrue()
+else
+  dateFull = isFalse()
+do
+  timeURL = java.net.URL('http://tycho.usno.navy.mil/cgi-bin/timer.pl')
+  conn = timeURL.openConnection()
+  ibr = BufferedReader(InputStreamReader(conn.getInputStream()))
+  line = Rexx
+  loop label readLoop while ibr.ready()
+    line = ibr.readLine()
+    if line = null then leave readLoop
+    line = line.translate(' ', '\t')
+    if line.wordpos('UTC') > 0 then do
+      parse line . '>'       udatetime 'UTC' . -
+            0    . '>' . ',' utime     'UTC' .
+      if dateFull then
+        say udatetime.strip() 'UTC'
+      else
+        say utime.strip()
+      leave readLoop
+      end
+    end readLoop
+  ibr.close()
+catch ex = IOException
+  ex.printStackTrace()
+end
+
+method isTrue() public constant returns boolean
+  return 1 == 1
+method isFalse() public constant returns boolean
+  return \isTrue()

Task/Web-scraping/Nim/web-scraping.nim

@@ -0,0 +1,13 @@
+import httpclient, strutils
+
+var client = newHttpClient()
+
+var res: string
+for line in client.getContent("https://rosettacode.org/wiki/Talk:Web_scraping").splitLines:
+  let k = line.find("UTC")
+  if k >= 0:
+    res = line[0..(k - 3)]
+    let k = res.rfind("</a>")
+    res = res[(k + 6)..^1]
+    break
+echo if res.len > 0: res else: "No date/time found."

Task/Web-scraping/OCaml/web-scraping.ocaml

@@ -0,0 +1,14 @@
+let () =
+  let _,_, page_content = make_request ~url:Sys.argv.(1) ~kind:GET () in
+
+  let lines = Str.split (Str.regexp "\n") page_content in
+  let str =
+    List.find
+      (fun line ->
+        try ignore(Str.search_forward (Str.regexp "UTC") line 0); true
+        with Not_found -> false)
+      lines
+  in
+  let str = Str.global_replace (Str.regexp "<BR>") "" str in
+  print_endline str;
+;;

Task/Web-scraping/Objeck/web-scraping.objeck

@@ -0,0 +1,25 @@
+use Net;
+use IO;
+use Structure;
+
+bundle Default {
+  class Scrape {
+    function : Main(args : String[]) ~ Nil {
+      client := HttpClient->New();
+      lines := client->Get("http://tycho.usno.navy.mil/cgi-bin/timer.pl", 80);
+
+       i := 0;
+       found := false;
+      while(found <> true & i < lines->Size()) {
+         line := lines->Get(i)->As(String);
+        index := line->Find("UTC");
+        if(index > -1) {
+          time := line->SubString(index - 9, 9)->Trim();
+          time->PrintLine();
+          found := true;
+        };
+        i += 1;
+      };
+    }
+  }
+}

Task/Web-scraping/OoRexx/web-scraping.rexx

@@ -0,0 +1,23 @@
+/* load the RexxcURL library */
+Call RxFuncAdd 'CurlLoadFuncs', 'rexxcurl', 'CurlLoadFuncs'
+Call CurlLoadFuncs
+
+url = "http://tycho.usno.navy.mil/cgi-bin/timer.pl"
+
+/* get a curl session */
+curl = CurlInit()
+if curl \= ''
+then do
+   call CurlSetopt curl, 'URL', Url
+   if curlerror.intcode \= 0 then exit
+   call curlSetopt curl, 'OUTSTEM', 'stem.'
+   if curlerror.intcode \= 0 then exit
+   call CurlPerform curl
+
+   /* content is in a stem - lets get it all in a string */
+   content = stem.~allItems~makestring('l')
+   /* now parse out utc time */
+   parse var content content 'Universal Time' .
+   utcTime = content~substr(content~lastpos('<BR>') + 4)
+   say utcTime
+end

Task/Web-scraping/Oz/web-scraping.oz

@@ -0,0 +1,20 @@
+declare
+  [Regex] = {Module.link ['x-oz://contrib/regex']}
+
+  fun {GetPage Url}
+     F = {New Open.file init(url:Url)}
+     Contents = {F read(list:$ size:all)}
+  in
+     {F close}
+     Contents
+  end
+
+  fun {GetDateString Doc}
+     case {Regex.search "<BR>([A-Za-z0-9:., ]+ UTC)" Doc}
+     of match(1:S#E ...) then {List.take {List.drop Doc S} E-S+1}
+     end
+  end
+
+  Url = "http://tycho.usno.navy.mil/cgi-bin/timer.pl"
+in
+  {System.showInfo {GetDateString {GetPage Url}}}

Task/Web-scraping/PHP/web-scraping-1.php

@@ -0,0 +1,8 @@
+<?
+
+$contents = file('http://tycho.usno.navy.mil/cgi-bin/timer.pl');
+foreach ($contents as $line){
+  if (($pos = strpos($line, ' UTC')) === false) continue;
+  echo subStr($line, 4, $pos - 4); //Prints something like "Dec. 06, 16:18:03"
+  break;
+}

Task/Web-scraping/PHP/web-scraping-2.php

@@ -0,0 +1,7 @@
+<?
+
+echo preg_replace(
+  "/^.*<BR>(.*) UTC.*$/su",
+  "\\1",
+  file_get_contents('http://tycho.usno.navy.mil/cgi-bin/timer.pl')
+);

Task/Web-scraping/Peloton/web-scraping-1.peloton

@@ -0,0 +1,5 @@
+<@ DEFAREPRS>Rexx Parse</@>
+<@ DEFPRSLIT>Rexx Parse|'<BR>' UTCtime 'UTC'</@>
+<@ LETVARURL>timer|http://tycho.usno.navy.mil/cgi-bin/timer.pl</@>
+<@ ACTRPNPRSVAR>Rexx Parse|timer</@>
+<@ SAYVAR>UTCtime</@>

Task/Web-scraping/Peloton/web-scraping-2.peloton

@@ -0,0 +1,5 @@
+<# DEFINE WORKAREA PARSEVALUES>Rexx Parse</#>
+<# DEFINE PARSEVALUES LITERAL>Rexx Parse|'<BR>' UTCtime 'UTC'</#>
+<# LET VARIABLE URLSOURCE>timer|http://tycho.usno.navy.mil/cgi-bin/timer.pl</#>
+<# ACT REPLACEBYPATTERN PARSEVALUES VARIABLE>Rexx Parse|timer</#>
+<# SAY VARIABLE>UTCtime</#>

Task/Web-scraping/Peloton/web-scraping-3.peloton

@@ -0,0 +1 @@
+<@ SAY AFT BEF URL LIT LIT LIT >http://tycho.usno.navy.mil/cgi-bin/timer.pl| UTC|<BR></@>

Task/Web-scraping/Perl/web-scraping.pl

@@ -0,0 +1,5 @@
+use LWP::Simple;
+
+my $url = 'http://tycho.usno.navy.mil/cgi-bin/timer.pl';
+get($url) =~ /<BR>(.+? UTC)/
+    and print "$1\n";

Task/Web-scraping/Phix/web-scraping.phix

@@ -0,0 +1,33 @@
+(notonline)-->
+ <span style="color: #000080;font-style:italic;">--
+ -- demo\rosetta\web_scrape.exw
+ -- ===========================
+ --</span>
+ <span style="color: #008080;">without</span> <span style="color: #008080;">js</span> <span style="color: #000080;font-style:italic;">-- (libcurl)</span>
+ <span style="color: #008080;">include</span> <span style="color: #000000;">builtins</span><span style="color: #0000FF;">\</span><span style="color: #000000;">libcurl</span><span style="color: #0000FF;">.</span><span style="color: #000000;">e</span>
+ <span style="color: #008080;">include</span> <span style="color: #000000;">builtins</span><span style="color: #0000FF;">\</span><span style="color: #004080;">timedate</span><span style="color: #0000FF;">.</span><span style="color: #000000;">e</span>
+
+ <span style="color: #004080;">object</span> <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">curl_easy_perform_ex</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"https://rosettacode.org/wiki/Talk:Web_scraping"</span><span style="color: #0000FF;">)</span>
+ <span style="color: #008080;">if</span> <span style="color: #004080;">string</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
+     <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">split</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">,</span><span style="color: #008000;">'\n'</span><span style="color: #0000FF;">)</span>
+     <span style="color: #008080;">for</span> <span style="color: #000000;">i</span><span style="color: #0000FF;">=</span><span style="color: #000000;">1</span> <span style="color: #008080;">to</span> <span style="color: #7060A8;">length</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">do</span>
+       <span style="color: #008080;">if</span> <span style="color: #008080;">not</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">`&lt;div id="siteNotice"&gt;`</span><span style="color: #0000FF;">,</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">])</span> <span style="color: #008080;">then</span> <span style="color: #000080;font-style:italic;">-- (24/11/22, exclude notice)</span>
+         <span style="color: #004080;">integer</span> <span style="color: #000000;">k</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">match</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"UTC"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">])</span>
+         <span style="color: #008080;">if</span> <span style="color: #000000;">k</span> <span style="color: #008080;">then</span>
+             <span style="color: #004080;">string</span> <span style="color: #000000;">line</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">i</span><span style="color: #0000FF;">]</span> <span style="color: #000080;font-style:italic;">-- (debug aid)</span>
+             <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #000000;">line</span><span style="color: #0000FF;">[</span><span style="color: #000000;">1</span><span style="color: #0000FF;">..</span><span style="color: #000000;">k</span><span style="color: #0000FF;">-</span><span style="color: #000000;">3</span><span style="color: #0000FF;">]</span>
+             <span style="color: #000000;">k</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">rmatch</span><span style="color: #0000FF;">(</span><span style="color: #008000;">"&lt;/a&gt;"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)</span>
+             <span style="color: #000000;">res</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">trim</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">[</span><span style="color: #000000;">k</span><span style="color: #0000FF;">+</span><span style="color: #000000;">5</span><span style="color: #0000FF;">..$])</span>
+             <span style="color: #008080;">exit</span>
+         <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
+       <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
+     <span style="color: #008080;">end</span> <span style="color: #008080;">for</span>
+     <span style="color: #0000FF;">?</span><span style="color: #000000;">res</span>
+     <span style="color: #008080;">if</span> <span style="color: #004080;">string</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)</span> <span style="color: #008080;">then</span>
+         <span style="color: #004080;">timedate</span> <span style="color: #000000;">td</span> <span style="color: #0000FF;">=</span> <span style="color: #7060A8;">parse_date_string</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">,</span> <span style="color: #0000FF;">{</span><span style="color: #008000;">"hh:mm, d Mmmm yyyy"</span><span style="color: #0000FF;">})</span>
+         <span style="color: #0000FF;">?</span><span style="color: #7060A8;">format_timedate</span><span style="color: #0000FF;">(</span><span style="color: #000000;">td</span><span style="color: #0000FF;">,</span><span style="color: #008000;">"Dddd Mmmm ddth yyyy h:mpm"</span><span style="color: #0000FF;">)</span>
+     <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
+ <span style="color: #008080;">else</span>
+     <span style="color: #0000FF;">?{</span><span style="color: #008000;">"some error"</span><span style="color: #0000FF;">,</span><span style="color: #000000;">res</span><span style="color: #0000FF;">,</span><span style="color: #7060A8;">curl_easy_strerror</span><span style="color: #0000FF;">(</span><span style="color: #000000;">res</span><span style="color: #0000FF;">)}</span>
+ <span style="color: #008080;">end</span> <span style="color: #008080;">if</span>
+<!--

Task/Web-scraping/PicoLisp/web-scraping.l

@@ -0,0 +1,5 @@
+(load "@lib/http.l")
+
+(client "tycho.usno.navy.mil" 80 "cgi-bin/timer.pl"
+   (when (from "<BR>")
+      (pack (trim (till "U"))) ) )

Task/Web-scraping/PowerShell/web-scraping-1.psh

@@ -0,0 +1,4 @@
+$wc = New-Object Net.WebClient
+$html = $wc.DownloadString('http://tycho.usno.navy.mil/cgi-bin/timer.pl')
+$html -match ', (.*) UTC' | Out-Null
+Write-Host $Matches[1]

Task/Web-scraping/PowerShell/web-scraping-2.psh

@@ -0,0 +1 @@
+[System.DateTime]::UtcNow

Task/Web-scraping/PowerShell/web-scraping-3.psh

@@ -0,0 +1 @@
+[System.DateTime]::Now

Task/Web-scraping/PureBasic/web-scraping.basic

@@ -0,0 +1,4 @@
+URLDownloadToFile_( #Null, "http://tycho.usno.navy.mil/cgi-bin/timer.pl", "timer.htm", 0, #Null)
+ReadFile(0, "timer.htm")
+While Not Eof(0)    :    Text$ + ReadString(0)    :    Wend
+MessageRequester("Time", Mid(Text$, FindString(Text$, "UTC", 1) - 9 , 8))

Task/Web-scraping/Python/web-scraping.py

@@ -0,0 +1,7 @@
+import urllib
+page = urllib.urlopen('http://tycho.usno.navy.mil/cgi-bin/timer.pl')
+for line in page:
+    if ' UTC' in line:
+        print line.strip()[4:]
+        break
+page.close()

Task/Web-scraping/R/web-scraping-1.r

@@ -0,0 +1,4 @@
+all_lines <- readLines("http://tycho.usno.navy.mil/cgi-bin/timer.pl")
+utc_line <- grep("UTC", all_lines, value = TRUE)
+matched <- regexpr("(\\w{3}.*UTC)", utc_line)
+utc_time_str <- substring(line, matched, matched + attr(matched, "match.length") - 1L)

Task/Web-scraping/R/web-scraping-2.r

@@ -0,0 +1,3 @@
+library(stringr)
+utc_line <- all_lines[str_detect(all_lines, "UTC")]
+utc_time_str <- str_extract(utc_line, "\\w{3}.*UTC")

Task/Web-scraping/R/web-scraping-3.r

@@ -0,0 +1,2 @@
+utc_time <- strptime(utc_time_str, "%b. %d, %H:%M:%S UTC")
+strftime(utc_time, "%A, %d %B %Y, %H:%M:%S")

Task/Web-scraping/R/web-scraping-4.r

@@ -0,0 +1,2 @@
+library(RCurl)
+web_page <- getURL("http://tycho.usno.navy.mil/cgi-bin/timer.pl")

Task/Web-scraping/R/web-scraping-5.r

@@ -0,0 +1,5 @@
+library(XML)
+page_tree <- htmlTreeParse(webpage)
+times_node <- page_tree$children$html$children$body$children$h3$children$pre$children
+times_node <- times_node[names(times_node) == "text"]
+time_lines <- sapply(times_node, function(x) x$value)

Task/Web-scraping/R/web-scraping-6.r

@@ -0,0 +1,4 @@
+page_tree <- htmlTreeParse(web_page, useInternalNodes = TRUE)
+times_node <- xpathSApply(page_tree, "//pre")[[1]]
+times_node <- times_node[names(times_node) == "text"]
+time_lines <- sapply(times_node, function(x) as(x, "character"))

Task/Web-scraping/R/web-scraping-7.r

@@ -0,0 +1,2 @@
+utc_line <- time_lines[str_detect(time_lines, "UTC")]
+#etc.

Task/Web-scraping/REBOL/web-scraping.rebol

@@ -0,0 +1,25 @@
+REBOL [
+  Title: "Web Scraping"
+  URL: http://rosettacode.org/wiki/Web_Scraping
+]
+
+; Notice that REBOL understands unquoted URL's:
+
+service: http://tycho.usno.navy.mil/cgi-bin/timer.pl
+
+; The 'read' function can read from any data scheme that REBOL knows
+; about, which includes web URLs. NOTE: Depending on your security
+; settings, REBOL may ask you for permission to contact the service.
+
+html: read service
+
+; I parse the HTML to find the first <br> (note the unquoted HTML tag
+; -- REBOL understands those too), then copy the current time from
+; there to the "UTC" terminator.
+
+; I have the "to end" in the parse rule so the parse will succeed.
+; Not strictly necessary once I've got the time, but good practice.
+
+parse html [thru <br> copy current thru "UTC" to end]
+
+print ["Current UTC time:" current]

Task/Web-scraping/Racket/web-scraping.rkt

@@ -0,0 +1,5 @@
+#lang racket
+(require net/url)
+((compose1 car (curry regexp-match #rx"[^ <>][^<>]+ UTC")
+           port->string get-pure-port string->url)
+ "https://tycho.usno.navy.mil/cgi-bin/timer.pl")

Task/Web-scraping/Raku/web-scraping.raku

@@ -0,0 +1,21 @@
+# 20210301 Updated Raku programming solution
+
+use HTTP::Client; # https://github.com/supernovus/perl6-http-client/
+
+#`[ Site inaccessible since 2019 ?
+my $site = "http://tycho.usno.navy.mil/cgi-bin/timer.pl";
+HTTP::Client.new.get($site).content.match(/'<BR>'( .+? <ws> UTC )/)[0].say
+# ]
+
+my $site = "https://www.utctime.net/";
+my $matched = HTTP::Client.new.get($site).content.match(
+   /'<td>UTC</td><td>'( .*Z )'</td>'/
+)[0];
+
+say $matched;
+#$matched = '12321321:412312312 123';
+with DateTime.new($matched.Str) {
+   say 'The fetch result seems to be of a valid time format.'
+} else {
+   CATCH { put .^name, ': ', .Str }
+}

Task/Web-scraping/Ruby/web-scraping-1.rb

@@ -0,0 +1,10 @@
+require "open-uri"
+
+open('http://tycho.usno.navy.mil/cgi-bin/timer.pl') do |p|
+  p.each_line do |line|
+    if line =~ /UTC/
+      puts line.match(/ (\d{1,2}:\d{1,2}:\d{1,2}) /)
+      break
+    end
+  end
+end

Task/Web-scraping/Ruby/web-scraping-2.rb

@@ -0,0 +1,2 @@
+require 'open-uri'
+puts URI.parse('http://tycho.usno.navy.mil/cgi-bin/timer.pl').read.match(/ (\d{1,2}:\d{1,2}:\d{1,2}) UTC/)[1]

Task/Web-scraping/Run-BASIC/web-scraping.basic

@@ -0,0 +1 @@
+print word$(word$(httpget$("http://tycho.usno.navy.mil/cgi-bin/timer.pl"),1,"UTC"),2,"<BR>")

Task/Web-scraping/Rust/web-scraping.rust

@@ -0,0 +1,19 @@
+// 202100302 Rust programming solution
+
+use std::io::Read;
+use regex::Regex;
+
+fn main() {
+
+   let client   = reqwest::blocking::Client::new();
+   let site     = "https://www.utctime.net/";
+   let mut res  = client.get(site).send().unwrap();
+   let mut body = String::new();
+
+   res.read_to_string(&mut body).unwrap();
+
+   let re   = Regex::new(r#"<td>UTC</td><td>(.*Z)</td>"#).unwrap();
+   let caps = re.captures(&body).unwrap();
+
+   println!("Result : {:?}", caps.get(1).unwrap().as_str());
+}

Task/Web-scraping/Scala/web-scraping.scala

@@ -0,0 +1,10 @@
+import scala.io.Source
+
+object WebTime extends Application {
+  val text = Source.fromURL("http://tycho.usno.navy.mil/cgi-bin/timer.pl")
+  val utc = text.getLines.find(_.contains("UTC"))
+  utc match {
+    case Some(s) => println(s.substring(4))
+    case _ => println("error")
+  }
+}

Task/Web-scraping/Scheme/web-scraping.ss

@@ -0,0 +1,34 @@
+; Use the regular expression module to parse the url
+(use-modules (ice-9 regex) (ice-9 rdelim))
+
+; Variable to store result
+(define time "")
+
+; Set the url and parse the hostname, port, and path into variables
+(define url "http://tycho.usno.navy.mil/cgi-bin/timer.pl")
+(define r (make-regexp "^(http://)?([^:/]+)(:)?(([0-9])+)?(/.*)?" regexp/icase))
+(define host (match:substring (regexp-exec r url) 2))
+(define port (match:substring (regexp-exec r url) 4))
+(define path (match:substring (regexp-exec r url) 6))
+
+; Set port to 80 if it wasn't set above and convert from a string to a number
+(if (eq? port #f) (define port "80"))
+(define port (string->number port))
+
+; Connect to remote host on specified port
+(let ((s (socket PF_INET SOCK_STREAM 0)))
+        (connect s AF_INET (car (hostent:addr-list (gethostbyname host))) port)
+
+; Send a HTTP request for the specified path
+        (display "GET " s)
+        (display path s)
+        (display " HTTP/1.0\r\n\r\n" s)
+
+        (set! r (make-regexp "<BR>(.+? UTC)"))
+        (do ((line (read-line s) (read-line s))) ((eof-object? line))
+                (if (regexp-match? (regexp-exec r line))
+                        (set! time (match:substring (regexp-exec r line) 1)))))
+
+; Display result
+(display time)
+(newline)

Task/Web-scraping/Seed7/web-scraping.seed7

@@ -0,0 +1,20 @@
+$ include "seed7_05.s7i";
+  include "gethttp.s7i";
+
+const proc: main is func
+  local
+    var string: pageWithTime is "";
+    var integer: posOfUTC is 0;
+    var integer: posOfBR is 0;
+    var string: timeStri is "";
+  begin
+    pageWithTime := getHttp("tycho.usno.navy.mil/cgi-bin/timer.pl");
+    posOfUTC := pos(pageWithTime, "UTC");
+    if posOfUTC <> 0 then
+      posOfBR := rpos(pageWithTime, "<BR>", posOfUTC);
+      if posOfBR <> 0 then
+        timeStri := pageWithTime[posOfBR + 4 .. pred(posOfUTC)];
+        writeln(timeStri);
+      end if;
+    end if;
+  end func;

Task/Web-scraping/Sidef/web-scraping.sidef

@@ -0,0 +1,4 @@
+var ua = frequire('LWP::Simple');
+var url = 'http://tycho.usno.navy.mil/cgi-bin/timer.pl';
+var match = /<BR>(.+? UTC)/.match(ua.get(url));
+say match[0] if match;

Task/Web-scraping/Standard-ML/web-scraping-1.ml

@@ -0,0 +1,27 @@
+val getTime =  fn url =>
+  let
+   val fname       = "/tmp/fConv" ^ (String.extract (Time.toString (Posix.ProcEnv.time()),7,NONE) );
+   val shellCommand = " fetch -o - \""^ url ^"\" | sed -ne 's/^.*alt=.Los Angeles:\\(.* (Daylight Saving)\\).*$/\\1/p' " ;
+    val me         = (  Posix.FileSys.mkfifo
+                               (fname,
+			        Posix.FileSys.S.flags [ Posix.FileSys.S.irusr,Posix.FileSys.S.iwusr ]
+			       ) ;
+                       Posix.Process.fork ()
+		     )
+  in
+   if (Option.isSome me) then
+     let
+        val fin =TextIO.openIn fname
+     in
+        ( Posix.Process.sleep (Time.fromReal 0.5) ;
+          TextIO.inputLine fin  before
+	  (TextIO.closeIn fin ; OS.FileSys.remove fname )
+	)
+     end
+   else
+     ( OS.Process.system (  shellCommand ^ " > " ^ fname  ^ " 2>&1 "     ) ;
+      SOME "" before OS.Process.exit OS.Process.success
+     )
+end;
+
+print ( valOf (getTime "http://www.time.org"));

Task/Web-scraping/Standard-ML/web-scraping-2.ml

@@ -0,0 +1 @@
+11:12 AM (PDT) (Daylight Saving)

Task/Web-scraping/TUSCRIPT/web-scraping.tu

@@ -0,0 +1,3 @@
+$$ MODE TUSCRIPT
+SET time = REQUEST ("http://tycho.usno.navy.mil/cgi-bin/timer.pl")
+SET utc  = FILTER  (time,":*UTC*:",-)

Task/Web-scraping/TXR/web-scraping-1.txr

@@ -0,0 +1,19 @@
+@(next @(open-command "wget -c http://tycho.usno.navy.mil/cgi-bin/timer.pl -O - 2> /dev/null"))
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final"//EN>
+<html>
+<body>
+<TITLE>What time is it?</TITLE>
+<H2> US Naval Observatory Master Clock Time</H2> <H3><PRE>
+@(collect :vars (MO DD HH MM SS (PM "  ") TZ TZNAME))
+<BR>@MO. @DD, @HH:@MM:@SS @(maybe)@{PM /PM/} @(end)@TZ@/\t+/@TZNAME
+@  (until)
+</PRE>@/.*/
+@(end)
+</PRE></H3><P><A HREF="http://www.usno.navy.mil"> US Naval Observatory</A>
+
+</body></html>
+@(output)
+@  (repeat)
+@MO-@DD @HH:@MM:@SS @PM @TZ
+@  (end)
+@(end)

Task/Web-scraping/TXR/web-scraping-2.txr

@@ -0,0 +1,6 @@
+@(next @(open-command "wget -c http://tycho.usno.navy.mil/cgi-bin/timer.pl -O - 2> /dev/null"))
+@(skip)
+<BR>@time@\ UTC@(skip)
+@(output)
+@time
+@(end)

Task/Web-scraping/Tcl/web-scraping-1.tcl

@@ -0,0 +1,6 @@
+package require http
+
+set request [http::geturl "http://tycho.usno.navy.mil/cgi-bin/timer.pl"]
+if {[regexp -line {<BR>(.* UTC)} [http::data $request] --> utc]} {
+    puts $utc
+}

Task/Web-scraping/Tcl/web-scraping-2.tcl

@@ -0,0 +1,2 @@
+set data [exec curl -s http://tycho.usno.navy.mil/cgi-bin/timer.pl]
+puts [lrange [lsearch -glob -inline [split $data <BR>] *UTC*] 0 3]

Task/Web-scraping/ToffeeScript/web-scraping.toffee

@@ -0,0 +1,3 @@
+e, page = require('request').get! 'http://tycho.usno.navy.mil/cgi-bin/timer.pl'
+l = line for line in page.body.split('\n') when line.indexOf('UTC')>0
+console.log l.substr(4,l.length-20)

Task/Web-scraping/UNIX-Shell/web-scraping-1.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+curl -s http://tycho.usno.navy.mil/cgi-bin/timer.pl |
+   sed -ne 's/^<BR>\(.* UTC\).*$/\1/p'

Task/Web-scraping/UNIX-Shell/web-scraping-2.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/tcsh -f
+set page = `wget -q -O- "http://tycho.usno.navy.mil/cgi-bin/timer.pl"`
+echo `awk -v s="${page[22]}" 'BEGIN{print substr(s,5,length(s))}'` ${page[23]} ${page[24]}

Task/Web-scraping/Unicon/web-scraping.unicon

@@ -0,0 +1,7 @@
+procedure main()
+m := open(url := "http://tycho.usno.navy.mil/cgi-bin/timer.pl","m") | stop("Unable to open ",url)
+every (p := "") ||:= |read(m)                                                    # read the page into a single string
+close(m)
+
+map(p) ? ( tab(find("<br>")), ="<br>", write("UTC time=",p[&pos:find(" utc")]))  # scrape and show
+end

Task/Web-scraping/Ursala/web-scraping.ursala

@@ -0,0 +1,10 @@
+#import std
+#import cli
+
+#executable ('parameterized','')
+
+whatime =
+
+<.file$[contents: --<''>]>+ -+
+   @hm skip/*4+ ~=(9%cOi&)-~l*+ *~ ~&K3/'UTC',
+   (ask bash)/0+ -[wget -O - http://tycho.usno.navy.mil/cgi-bin/timer.pl]-!+-