2016 Update
This commit is contained in:
Tina Müller
parent
948b86eafa
commit
dcf5d15da3
7965 changed files with 139854 additions and 31002 deletions
|
|
@ -1,3 +1,10 @@
|
|||
An [[wp:Inverted_index|Inverted Index]] is a data structure used to create full text search.
|
||||
|
||||
Given a set of text files, implement a program to create an inverted index. Also create a user interface to do a search using that inverted index which returns a list of files that contain the query term / terms. The search index can be in memory.
|
||||
|
||||
;Task:
|
||||
Given a set of text files, implement a program to create an inverted index.
|
||||
|
||||
Also create a user interface to do a search using that inverted index which returns a list of files that contain the query term / terms.
|
||||
|
||||
The search index can be in memory.
|
||||
<br><br>
|
||||
|
|
|
|||
46
Task/Inverted-index/PHP/inverted-index.php
Normal file
46
Task/Inverted-index/PHP/inverted-index.php
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
<?php
|
||||
|
||||
function buildInvertedIndex($filenames)
|
||||
{
|
||||
$invertedIndex = [];
|
||||
|
||||
foreach($filenames as $filename)
|
||||
{
|
||||
$data = file_get_contents($filename);
|
||||
|
||||
if($data === false) die('Unable to read file: ' . $filename);
|
||||
|
||||
preg_match_all('/(\w+)/', $data, $matches, PREG_SET_ORDER);
|
||||
|
||||
foreach($matches as $match)
|
||||
{
|
||||
$word = strtolower($match[0]);
|
||||
|
||||
if(!array_key_exists($word, $invertedIndex)) $invertedIndex[$word] = [];
|
||||
if(!in_array($filename, $invertedIndex[$word], true)) $invertedIndex[$word][] = $filename;
|
||||
}
|
||||
}
|
||||
|
||||
return $invertedIndex;
|
||||
}
|
||||
|
||||
function lookupWord($invertedIndex, $word)
|
||||
{
|
||||
return array_key_exists($word, $invertedIndex) ? $invertedIndex[$word] : false;
|
||||
}
|
||||
|
||||
$invertedIndex = buildInvertedIndex2(['file1.txt', 'file2.txt', 'file3.txt']);
|
||||
|
||||
foreach(['cat', 'is', 'banana', 'it'] as $word)
|
||||
{
|
||||
$matches = lookupWord($invertedIndex, $word);
|
||||
|
||||
if($matches !== false)
|
||||
{
|
||||
echo "Found the word \"$word\" in the following files: " . implode(', ', $matches) . "\n";
|
||||
}
|
||||
else
|
||||
{
|
||||
echo "Unable to find the word \"$word\" in the index\n";
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
sub MAIN (*@files) {
|
||||
(my %norm).push: do for @files -> $file {
|
||||
$file X=> slurp($file).lc.words;
|
||||
my %norm;
|
||||
do for @files -> $file {
|
||||
%norm.push: $file X=> slurp($file).lc.words;
|
||||
}
|
||||
(my %inv).push: %norm.invert.unique;
|
||||
|
||||
|
|
|
|||
43
Task/Inverted-index/PowerShell/inverted-index-1.psh
Normal file
43
Task/Inverted-index/PowerShell/inverted-index-1.psh
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
function Index-File ( [string[]]$FileList )
|
||||
{
|
||||
# Create index hashtable, as needed
|
||||
If ( -not $Script:WordIndex ) { $Script:WordIndex = @{} }
|
||||
|
||||
# For each file to be indexed...
|
||||
ForEach ( $File in $FileList )
|
||||
{
|
||||
# Find any previously existing entries for this file
|
||||
$ExistingEntries = $Script:WordIndex.Keys | Where { $Script:WordIndex[$_] -contains $File }
|
||||
|
||||
# For any previously existing entries
|
||||
# Delete them (prior to reindexing the file)
|
||||
ForEach ( $Key in $ExistingEntries )
|
||||
{
|
||||
$Script:WordIndex[$Key] = @( $Script:WordIndex[$Key] | Where { $_ -ne $File } )
|
||||
}
|
||||
|
||||
# Get the contents of the file, split on non-alphanumeric characters, and remove duplicates
|
||||
$Words = ( Get-Content $File ) -split '[^a-zA-Z\d]' | Sort -Unique
|
||||
|
||||
# For each word in the file...
|
||||
ForEach ( $Word in $Words )
|
||||
{
|
||||
# If the entry for the word already exists...
|
||||
If ( $Script:WordIndex[$Word] )
|
||||
{
|
||||
# Add the file name to the entry
|
||||
$Script:WordIndex[$Word] += $File
|
||||
}
|
||||
Else
|
||||
{
|
||||
# Create a new entry
|
||||
$Script:WordIndex[$Word] = @( $File )
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Find-Word ( [string]$Word )
|
||||
{
|
||||
return $WordIndex[$Word]
|
||||
}
|
||||
15
Task/Inverted-index/PowerShell/inverted-index-2.psh
Normal file
15
Task/Inverted-index/PowerShell/inverted-index-2.psh
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
# Populate test files
|
||||
@'
|
||||
Files full of
|
||||
various words.
|
||||
'@ | Out-File -FilePath C:\Test\File1.txt
|
||||
|
||||
@'
|
||||
Create an index
|
||||
of words.
|
||||
'@ | Out-File -FilePath C:\Test\File2.txt
|
||||
|
||||
@'
|
||||
Use the index
|
||||
to find the files.
|
||||
'@ | Out-File -FilePath C:\Test\File3.txt
|
||||
2
Task/Inverted-index/PowerShell/inverted-index-3.psh
Normal file
2
Task/Inverted-index/PowerShell/inverted-index-3.psh
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
# Index files
|
||||
Index-File C:\Test\File1.txt, C:\Test\File2.txt, C:\Test\File3.txt
|
||||
2
Task/Inverted-index/PowerShell/inverted-index-4.psh
Normal file
2
Task/Inverted-index/PowerShell/inverted-index-4.psh
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
# Query index
|
||||
Find-Word files
|
||||
|
|
@ -1,58 +1,58 @@
|
|||
/*REXX program illustrates building a simple inverted index & word find.*/
|
||||
@.='' /*dictionary of words (so far).*/
|
||||
!='' /*a list of found words (so far).*/
|
||||
call invertI 0, 'BURMA0.TXT' /*read the file: BURMA0.TXT ...*/
|
||||
call invertI 1, 'BURMA1.TXT' /* " " ~ BURMA1.TXT ...*/
|
||||
call invertI 2, 'BURMA2.TXT' /* " " ~ BURMA2.TXT ...*/
|
||||
call invertI 3, 'BURMA3.TXT' /* " " ~ BURMA3.TXT ...*/
|
||||
call invertI 4, 'BURMA4.TXT' /* " " ~ BURMA4.TXT ...*/
|
||||
call invertI 5, 'BURMA5.TXT' /* " " ~ BURMA5.TXT ...*/
|
||||
call invertI 6, 'BURMA6.TXT' /* " " ~ BURMA6.TXT ...*/
|
||||
call invertI 7, 'BURMA7.TXT' /* " " ~ BURMA7.TXT ...*/
|
||||
call invertI 8, 'BURMA8.TXT' /* " " ~ BURMA8.TXT ...*/
|
||||
call invertI 9, 'BURMA9.TXT' /* " " ~ BURMA9.TXT ...*/
|
||||
call findAword 'does' /*find a word. */
|
||||
call findAword '60' /*find another word. */
|
||||
call findAword "don't" /*and find another word. */
|
||||
call findAword "burma-shave" /*and find yet another word. */
|
||||
exit /*stick a fork in it, we're done.*/
|
||||
/*──────────────────────────────────FINDAWORD subroutine────────────────*/
|
||||
findAword: procedure expose @.; arg x /*get an uppercase version of X. */
|
||||
parse arg ox /*get original (as-is) value of X*/
|
||||
_=@.x; oxo='───'ox"───"
|
||||
if _=='' then do
|
||||
say 'word' oxo "not found."
|
||||
return 0
|
||||
end
|
||||
_@=_ /*save _, pass it back to invoker*/
|
||||
say 'word' oxo "found in:"
|
||||
do until _==''; parse var _ f w _
|
||||
say ' file='f ' word='w
|
||||
end /*until ··· */
|
||||
return _@
|
||||
/*─────────────────────────────────────INVERTI subroutine───────────────*/
|
||||
invertI: procedure expose @. !; parse arg #,fn /*file#, filename*/
|
||||
call lineout fn /*close the file, just in case. */
|
||||
w=0 /*number of words found (so far).*/
|
||||
do while lines(fn)\==0 /* [↓] process the entire file.*/
|
||||
_=space(linein(fn)) /*read a line, elide extra blanks*/
|
||||
if _=='' then iterate /*if blank record, then ignore it*/
|
||||
say 'file' #", record:" _ /*echo a record (to be verbose).*/
|
||||
/*REXX program illustrates building a simple inverted index and a method of word find.*/
|
||||
@.= /*a dictionary of words (so far). */
|
||||
!= /*a list of found words (so far). */
|
||||
call invertI 0, 'BURMA0.TXT' /*read the file: BURMA0.TXT ··· */
|
||||
call invertI 1, 'BURMA1.TXT' /* " " " BURMA1.TXT ··· */
|
||||
call invertI 2, 'BURMA2.TXT' /* " " " BURMA2.TXT ··· */
|
||||
call invertI 3, 'BURMA3.TXT' /* " " " BURMA3.TXT ··· */
|
||||
call invertI 4, 'BURMA4.TXT' /* " " " BURMA4.TXT ··· */
|
||||
call invertI 5, 'BURMA5.TXT' /* " " " BURMA5.TXT ··· */
|
||||
call invertI 6, 'BURMA6.TXT' /* " " " BURMA6.TXT ··· */
|
||||
call invertI 7, 'BURMA7.TXT' /* " " " BURMA7.TXT ··· */
|
||||
call invertI 8, 'BURMA8.TXT' /* " " " BURMA8.TXT ··· */
|
||||
call invertI 9, 'BURMA9.TXT' /* " " " BURMA9.TXT ··· */
|
||||
call findAword "huz" /*find a word. */
|
||||
call findAword "60" /*find another word. */
|
||||
call findAword "don't" /*and find another word. */
|
||||
call findAword "burma-shave" /*and find yet another word. */
|
||||
exit /*stick a fork in it, we're all done. */
|
||||
/*──────────────────────────────────────────────────────────────────────────────────────*/
|
||||
findAword: procedure expose @.; arg x /*get an uppercase version of the X arg*/
|
||||
parse arg ox /*get original (as-is) value of X arg.*/
|
||||
_=@.x; oxo='───'ox"───"
|
||||
if _=='' then do
|
||||
say 'word' oxo "not found."
|
||||
return 0
|
||||
end
|
||||
_@=_ /*save _ text, pass it back to invoker.*/
|
||||
say 'word' oxo "found in:"
|
||||
do until _==''; parse var _ f w _
|
||||
say ' file='f " word="w
|
||||
end /*until ··· */
|
||||
return _@
|
||||
/*──────────────────────────────────────────────────────────────────────────────────────*/
|
||||
invertI: procedure expose @. !; parse arg #,fn /*the file number and the filename. */
|
||||
call lineout fn /*close the file, ··· just in case. */
|
||||
w=0 /*the number of words found (so far). */
|
||||
do while lines(fn)\==0 /* [↓] process the entire file. */
|
||||
_=space( linein(fn) ) /*read a line, elide superfluous blanks*/
|
||||
if _=='' then iterate /*if a blank record, then ignore it. */
|
||||
say 'file' #", record:" _ /*display the record ──► terminal. */
|
||||
|
||||
do until _=='' /*pick off words until done. */
|
||||
parse upper var _ ? _ /*pick off a word (uppercased). */
|
||||
?=stripper(?) /*strip any trailing punctuation.*/
|
||||
if ?='' then iterate /*is the word now blank (null) ? */
|
||||
w=w+1 /*bump the word counter (index). */
|
||||
@.?=@.? # w /*append the new word to a list. */
|
||||
if wordpos(?,!)==0 then !=! ? /*add to the list of words found.*/
|
||||
end /*until ··· */
|
||||
end /*while ··· */
|
||||
say; call lineout fn /*close the file, just to be neat*/
|
||||
return w /*return the index of the word. */
|
||||
/*─────────────────────────────────────STRIPPER subroutine──────────────*/
|
||||
stripper: procedure; parse arg q /*remove punctuation at word-end.*/
|
||||
@punctuation='.,:;?¿!¡∙·'; do j=1 for length(@punctuation)
|
||||
q=strip(q,'T',substr(@punctuation,j,1))
|
||||
end /*j*/
|
||||
return q
|
||||
do until _=='' /*pick off words from record until done*/
|
||||
parse upper var _ ? _ /*pick off a word (it's in uppercase).*/
|
||||
?=stripper(?) /*strip any trailing punctuation. */
|
||||
if ?='' then iterate /*is the word now all blank (or null)? */
|
||||
w=w+1 /*bump the word counter (index). */
|
||||
@.?=@.? # w /*append the new word to a list. */
|
||||
if wordpos(?,!)==0 then !=! ? /*add it to the list of words found. */
|
||||
end /*until ··· */
|
||||
end /*while ··· */
|
||||
say; call lineout fn /*close the file, just to be neat&safe.*/
|
||||
return w /*return the index of word in record. */
|
||||
/*──────────────────────────────────────────────────────────────────────────────────────*/
|
||||
stripper: procedure; parse arg q /*remove punctuation at the end of word*/
|
||||
@punctuation= '.,:;?¿!¡∙·'; do j=1 for length(@punctuation)
|
||||
q=strip(q, 'T', substr(@punctuation, j, 1) )
|
||||
end /*j*/
|
||||
return q
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue