Data commit

Task/Yahoo-search-interface/Tcl/yahoo-search-interface-1.tcl

@@ -0,0 +1,38 @@
+package require http
+
+proc fix s {
+    string map {<b>...</b> "" <b> "" </b> "" <wbr> "" "<wbr />" ""} \
+            [regsub "</a></h3></div>.*" $s ""]
+}
+proc YahooSearch {term {page 1}} {
+    # Build the (ugly) scraper URL
+    append re {<a class="yschttl spt" href=".+?" >(.+?)</a></h3>}
+    append re {</div><div class="abstr">(.+?)}
+    append re {</div><span class=url>(.+?)</span>}
+
+    # Perform the query; note that this handles special characters
+    # in the query term correctly
+    set q [http::formatQuery p $term b [expr {$page*10-9}]]
+    set token [http::geturl http://search.yahoo.com/search?$q]
+    set data [http::data $token]
+    http::cleanup $token
+
+    # Assemble the results into a nice list
+    set results {}
+    foreach {- title content url} [regexp -all -inline $re $data] {
+        lappend results [fix $title] [fix $content] [fix $url]
+    }
+
+    # set up the call for the next page
+    interp alias {} Nextpage {} YahooSearch $term [incr page]
+
+    return $results
+}
+
+# Usage:  get the first two pages of results
+foreach {title content url} [YahooSearch "test"] {
+    puts $title
+}
+foreach {title content url} [Nextpage] {
+     puts $title
+}

Task/Yahoo-search-interface/Tcl/yahoo-search-interface-2.tcl

@@ -0,0 +1,34 @@
+package require Tcl 8.6
+
+oo::class create WebSearcher {
+    variable page term results
+    constructor searchTerm {
+        set page 0
+        set term $searchTerm
+        my nextPage
+    }
+    # This next method *is* a very Tcl-ish way of doing iteration.
+    method for {titleVar contentsVar urlVar body} {
+        upvar 1 $titleVar t $contentsVar c $urlVar v
+        foreach {t c v} $results {
+            uplevel 1 $body
+        }
+    }
+    # Reuse the previous code for simplicity rather than writing it anew
+    # Of course, if we were serious about this, we'd put the code here properly
+    method nextPage {} {
+        set results [YahooSearch $term [incr page]]
+        return
+    }
+}
+
+# How to use. Note the 'foreach' method use below; new "keywords" as methods!
+set ytest [WebSearcher new "test"]
+$ytest for title - url {
+    puts "\"$title\" : $url"
+}
+$ytest nextPage
+$ytest for title - url {
+    puts "\"$title\" : $url"
+}
+$ytest delete ;# standard method that deletes the object

Task/Yahoo-search-interface/Tcl/yahoo-search-interface-3.tcl

@@ -0,0 +1,23 @@
+package require Tcl 8.6
+
+proc yahoo! term {
+    coroutine yahoo![incr ::yahoo] apply {term {
+        yield [info coroutine]
+        while 1 {
+            set results [YahooSearch $term [incr step]]
+            if {[llength $results] == 0} {
+                return -code break
+            }
+            foreach {t c u} $results {
+                yield [dict create title $t content $c url $u]
+            }
+        }
+    }} $term
+}
+
+# test by getting first fifty titles...
+set it [yahoo! "test"]
+for {set i 50} {$i>0} {incr i -1} {
+    puts [dict get [$it] title]
+    after 300  ;# Slow the code down... :-)
+}

Task/Yahoo-search-interface/Tcl/yahoo-search-interface-4.tcl

@@ -0,0 +1,114 @@
+package require Tcl 8.6
+package require http
+package require htmlparse
+package require textutil::adjust
+
+oo::class create yahoosearch {
+
+    method search {s} {
+        my variable searchterm page baseurl
+        set searchterm $s
+        set page 1
+        set baseurl {http://ca.search.yahoo.com/search}
+    }
+
+    method getresults {} {
+        my variable state results current_data
+        set results [list]
+        set current_data [dict create]
+        set state looking_for_results
+        htmlparse::parse -cmd [list [self] html_parser_callback] [my gethtml]
+    }
+
+    method nextpage {} {
+        my variable page
+        incr page 10
+        my getresults
+    }
+
+    method nextresult {} {
+        my variable results page
+        if { ! [info exists results]} {
+            my getresults
+        } elseif {[llength $results] == 0} {
+            my nextpage
+        }
+        set results [lassign $results result]
+        return $result
+    }
+
+    method gethtml {} {
+        my variable searchterm page baseurl
+        set url [format {%s?%s} $baseurl [::http::formatQuery p $searchterm b $page]]
+        set response [http::geturl $url]
+        set html [http::data $response]
+        http::cleanup $response
+        return $html
+    }
+
+    method html_parser_callback {tag slash param textBehindTheTag} {
+        my variable state results current_data
+        switch -exact -- $state {
+            looking_for_results {
+                if {$tag eq "div" && [string first {id="main"} $param] != -1} {
+                    set state ready
+                }
+            }
+            ready {
+                if {($tag eq "div" && [string first {class="res} $param] != -1) ||
+                    ($tag eq "html" && $slash eq "/")
+                } { #" -- unbalanced quote disturbs syntax highlighting
+                    if {[dict size $current_data] > 0} {lappend results $current_data}
+                    set current_data [dict create]
+                    set state getting_url
+                }
+            }
+            getting_url {
+                if {$tag eq "a" && [string match "*yschttl spt*" $param]} {
+                    if {[regexp {href="(.+?)"} $param - url]} {
+                        dict set current_data url $url
+                    } else {
+                        dict set current_data url "no href in tag params: '$param'"
+                    }
+                    dict set current_data title $textBehindTheTag
+                    set state getting_title
+                }
+            }
+            getting_title {
+                if {$tag eq "a" && $slash eq "/"} {
+                    set state looking_for_abstract
+                } else {
+                    dict append current_data title $textBehindTheTag
+                }
+            }
+            looking_for_abstract {
+                if {$tag eq "span" && [string first {class="url} $param] != -1} {
+                    set state ready
+                } elseif {$tag eq "div" && [string first {class="abstr} $param] != -1} {
+                    dict set current_data abstract $textBehindTheTag
+                    set state getting_abstract
+                }
+            }
+            getting_abstract {
+                if {$tag eq "div" && $slash eq "/"} {
+                    set state ready
+                } else {
+                    dict append current_data abstract $textBehindTheTag
+                }
+            }
+        }
+    }
+}
+
+yahoosearch create searcher
+searcher search "search text here"
+
+for {set x 1} {$x <= 15} {incr x} {
+    set result [searcher nextresult]
+    dict with result {
+        puts $title
+        puts $url
+        puts [textutil::adjust::indent [textutil::adjust::adjust $abstract] "  "]
+        puts ""
+    }
+}