90 lines
2.4 KiB
R
90 lines
2.4 KiB
R
YahooSearch <- function(query, page=1, .opts=list(), ignoreMarkUpErrors=TRUE)
|
|
{
|
|
if(!require(RCurl) || !require(XML))
|
|
{
|
|
stop("Could not load required packages")
|
|
}
|
|
|
|
# Replace " " with "%20", etc
|
|
query <- curlEscape(query)
|
|
|
|
# Retrieve page
|
|
b <- 10*(page-1)+1
|
|
theurl <- paste("http://uk.search.yahoo.com/search?p=",
|
|
query, "&b=", b, sep="")
|
|
webpage <- getURL(theurl, .opts=.opts)
|
|
|
|
# Save search for nextpage function
|
|
.Search <- list(query=query, page=page, .opts=.opts,
|
|
ignoreMarkUpErrors=ignoreMarkUpErrors)
|
|
assign(".Search", .Search, envir=globalenv())
|
|
|
|
# Parse HTML; retrieve results block
|
|
webpage <- readLines(tc <- textConnection(webpage)); close(tc)
|
|
if(ignoreMarkUpErrors)
|
|
{
|
|
pagetree <- htmlTreeParse(webpage, error=function(...){})
|
|
} else
|
|
{
|
|
pagetree <- htmlTreeParse(webpage)
|
|
}
|
|
|
|
|
|
findbyattr <- function(x, id, type="id")
|
|
{
|
|
ids <- sapply(x, function(x) x$attributes[type])
|
|
x[ids==id]
|
|
}
|
|
|
|
body <- pagetree$children$html$children$body
|
|
bd <- findbyattr(body$children$div$children, "bd")
|
|
left <- findbyattr(bd$div$children$div$children, "left")
|
|
web <- findbyattr(left$div$children$div$children, "web")
|
|
resol <- web$div$children$ol
|
|
|
|
#Get url, title, content from results
|
|
gettextfromnode <- function(x)
|
|
{
|
|
un <- unlist(x$children)
|
|
paste(un[grep("value", names(un))], collapse=" ")
|
|
}
|
|
|
|
n <- length(resol)
|
|
results <- list()
|
|
length(results) <- n
|
|
for(i in 1:n)
|
|
{
|
|
mainlink <- resol[[i]]$children$div$children[1]$div$children$h3$children$a
|
|
url <- mainlink$attributes["href"]
|
|
title <- gettextfromnode(mainlink)
|
|
|
|
contenttext <- findbyattr(resol[[i]]$children$div$children[2], "abstr", type="class")
|
|
if(length(contenttext)==0)
|
|
{
|
|
contenttext <- findbyattr(resol[[i]]$children$div$children[2]$div$children$div$children,
|
|
"sm-abs", type="class")
|
|
}
|
|
|
|
content <- gettextfromnode(contenttext$div)
|
|
results[[i]] <- list(url=url, title=title, content=content)
|
|
}
|
|
names(results) <- as.character(seq(b, b+n-1))
|
|
results
|
|
}
|
|
|
|
nextpage <- function()
|
|
{
|
|
if(exists(".Search", envir=globalenv()))
|
|
{
|
|
.Search <- get(".Search", envir=globalenv())
|
|
.Search$page <- .Search$page + 1L
|
|
do.call(YahooSearch, .Search)
|
|
} else
|
|
{
|
|
message("No search has been performed yet")
|
|
}
|
|
}
|
|
|
|
#Usage
|
|
YahooSearch("rosetta code")
|
|
nextpage()
|