Data update
This commit is contained in:
Ingy döt Net
parent
5150844a7d
commit
4bb20c9b71
7735 changed files with 38060 additions and 199180 deletions
|
|
@ -1,114 +0,0 @@
|
|||
with Ada.Text_IO, Generic_Inverted_Index, Ada.Strings.Hash, Parse_Lines;
|
||||
use Ada.Text_IO;
|
||||
|
||||
procedure Inverted_Index is
|
||||
|
||||
type Process_Word is access procedure (Word: String);
|
||||
|
||||
package Inv_Idx is new Generic_Inverted_Index
|
||||
(Source_Type => String,
|
||||
Item_Type => String,
|
||||
Hash => Ada.Strings.Hash);
|
||||
|
||||
use Inv_Idx;
|
||||
|
||||
procedure Output(Sources: Source_Vecs.Vector) is
|
||||
Any_Output: Boolean := False;
|
||||
|
||||
procedure Print_Source(S: String) is
|
||||
begin
|
||||
if not Any_Output then -- this is the first source found
|
||||
Put("Found in the following files: ");
|
||||
Any_Output := True;
|
||||
else -- there has been at least one source before
|
||||
Put(", ");
|
||||
end if;
|
||||
Put(S);
|
||||
end Print_Source;
|
||||
|
||||
procedure Print is new Inv_Idx.Iterate(Print_Source);
|
||||
|
||||
begin
|
||||
Print(Sources);
|
||||
if not Any_Output then
|
||||
Put("I did not find this in any of the given files!");
|
||||
end if;
|
||||
New_Line(2);
|
||||
end Output;
|
||||
|
||||
procedure Read_From_File(Table: in out Storage_Type;
|
||||
Filename: String) is
|
||||
F: File_Type;
|
||||
|
||||
procedure Enter_Word(S: String) is
|
||||
begin
|
||||
Table.Store(Source => Filename, Item => S);
|
||||
end Enter_Word;
|
||||
|
||||
procedure Store_Words is new
|
||||
Parse_Lines.Iterate_Words(Parse_Lines.Word_Pattern, Enter_Word);
|
||||
|
||||
begin
|
||||
Open(File => F, Mode => In_File, Name => Filename);
|
||||
while not End_Of_File(F) loop
|
||||
Store_Words(Get_Line(F));
|
||||
end loop;
|
||||
Close(F);
|
||||
exception
|
||||
when others =>
|
||||
Put_Line("Something wrong with File '" & Filename & "'");
|
||||
Put_Line("I'll ignore this!");
|
||||
end Read_From_File;
|
||||
|
||||
procedure Read_Files(Tab: out Storage_Type; Line: in String) is
|
||||
|
||||
procedure Read_File(S: String) is
|
||||
begin
|
||||
Read_From_File(Tab, S);
|
||||
end Read_File;
|
||||
|
||||
procedure Read_All is new
|
||||
Parse_Lines.Iterate_Words(Parse_Lines.Filename_Pattern, Read_File);
|
||||
|
||||
begin
|
||||
Read_All(Line);
|
||||
end Read_Files;
|
||||
|
||||
S: Storage_Type;
|
||||
Done: Boolean := False;
|
||||
|
||||
begin
|
||||
Put_Line("Enter Filenames:");
|
||||
Read_Files(S, Get_Line);
|
||||
New_Line;
|
||||
|
||||
while not Done loop
|
||||
Put_Line("Enter one or more words to search for; <return> to finish:");
|
||||
declare
|
||||
Words: String := Get_Line;
|
||||
First: Boolean := True;
|
||||
Vec: Source_Vecs.Vector := Source_Vecs.Empty_Vector;
|
||||
|
||||
procedure Compute_Vector(Item: String) is
|
||||
begin
|
||||
if First then
|
||||
Vec := S.Find(Item);
|
||||
First := False;
|
||||
else
|
||||
Vec := Vec and S.Find(Item);
|
||||
end if;
|
||||
end Compute_Vector;
|
||||
|
||||
procedure Compute is new
|
||||
Parse_Lines.Iterate_Words(Parse_Lines.Word_Pattern, Compute_Vector);
|
||||
|
||||
begin
|
||||
if Words = "" then
|
||||
Done := True;
|
||||
else
|
||||
Compute(Words);
|
||||
Output(Vec);
|
||||
end if;
|
||||
end;
|
||||
end loop;
|
||||
end Inverted_Index;
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
with Ada.Containers.Indefinite_Vectors;
|
||||
private with Ada.Containers.Indefinite_Hashed_Maps;
|
||||
|
||||
generic
|
||||
type Source_Type (<>) is private;
|
||||
type Item_Type (<>) is private;
|
||||
with function Hash(Item: Item_Type) return Ada.Containers.Hash_Type is <>;
|
||||
package Generic_Inverted_Index is
|
||||
|
||||
type Storage_Type is tagged private;
|
||||
|
||||
package Source_Vecs is new Ada.Containers.Indefinite_Vectors
|
||||
(Index_Type => Positive,
|
||||
Element_Type => Source_Type);
|
||||
|
||||
procedure Store(Storage: in out Storage_Type;
|
||||
Source: Source_Type;
|
||||
Item: Item_Type);
|
||||
-- stores Source in a table, indexed by Item
|
||||
-- if there is already an Item/Source entry, the Table isn_t changed
|
||||
|
||||
function Find(Storage: Storage_Type; Item: Item_Type)
|
||||
return Source_Vecs.Vector;
|
||||
-- Generates a vector of all Sources for the given Item
|
||||
|
||||
function "and"(Left, Right: Source_Vecs.Vector) return Source_Vecs.Vector;
|
||||
-- returns a vector of all sources, which are both in Left and in Right
|
||||
|
||||
function "or"(Left, Right: Source_Vecs.Vector) return Source_Vecs.Vector;
|
||||
-- returns a vector of all sources, which are in Left, Right, or both
|
||||
|
||||
function Empty(Vec: Source_Vecs.Vector) return Boolean;
|
||||
-- returns true if Vec is empty
|
||||
|
||||
function First_Source(The_Sources: Source_Vecs.Vector) return Source_Type;
|
||||
-- returns the first enty in The_Sources; pre: The_Sourses is not empty
|
||||
|
||||
procedure Delete_First_Source(The_Sources: in out Source_Vecs.Vector;
|
||||
Count: Ada.Containers.Count_Type := 1);
|
||||
-- Removes the first Count entries; pre: The_Sourses has that many entries
|
||||
|
||||
type Process_Source is not null access procedure (Source: Source_Type);
|
||||
|
||||
generic
|
||||
with procedure Do_Something(Source: Source_Type);
|
||||
procedure Iterate(The_Sources: Source_Vecs.Vector);
|
||||
-- calls Do_Something(Source) for all sources in The_Sources;
|
||||
|
||||
private
|
||||
|
||||
function Same_Vector(U,V: Source_Vecs.Vector) return Boolean;
|
||||
|
||||
package Maps is new Ada.Containers.Indefinite_Hashed_Maps
|
||||
-- for each item (=key) we store a vector with sources
|
||||
(Key_Type => Item_Type,
|
||||
Element_Type => Source_Vecs.Vector,
|
||||
Hash => Hash,
|
||||
Equivalent_Keys => "=",
|
||||
"=" => Same_Vector);
|
||||
|
||||
type Storage_Type is new Maps.Map with null record;
|
||||
|
||||
end Generic_Inverted_Index;
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
package body Generic_Inverted_Index is
|
||||
|
||||
use Source_Vecs;
|
||||
use type Maps.Cursor;
|
||||
|
||||
|
||||
procedure Store(Storage: in out Storage_Type;
|
||||
Source: Source_Type;
|
||||
Item: Item_Type) is
|
||||
begin
|
||||
if (Storage.Find(Item) = Maps.No_Element) then
|
||||
Storage.Insert(Key => Item,
|
||||
New_Item => Empty_Vector & Source);
|
||||
else
|
||||
declare
|
||||
The_Vector: Vector := Storage.Element(Item);
|
||||
begin
|
||||
if The_Vector.Last_Element /= Source then
|
||||
Storage.Replace
|
||||
(Key => Item,
|
||||
New_Item => Storage.Element(Item) & Source);
|
||||
end if;
|
||||
end;
|
||||
end if;
|
||||
end Store;
|
||||
|
||||
function Find(Storage: Storage_Type; Item: Item_Type)
|
||||
return Vector is
|
||||
begin
|
||||
return Storage.Element(Item);
|
||||
exception
|
||||
when Constraint_Error => return Empty_Vector; -- found nothing
|
||||
end Find;
|
||||
|
||||
function Is_In(S: Source_Type; V: Vector) return Boolean is
|
||||
VV: Vector := V;
|
||||
begin
|
||||
if Empty(V) then
|
||||
return False;
|
||||
elsif First_Source(V) = S then
|
||||
return True;
|
||||
else
|
||||
Delete_First_Source(VV);
|
||||
return Is_In(S, VV);
|
||||
end if;
|
||||
end Is_In;
|
||||
|
||||
function "and"(Left, Right: Vector) return Vector is
|
||||
V: Vector := Empty_Vector;
|
||||
begin
|
||||
for I in First_Index(Left) .. Last_Index(Left) loop
|
||||
if Is_In(Element(Left, I), Right) then
|
||||
V := V & Element(Left, I);
|
||||
end if;
|
||||
end loop;
|
||||
return V;
|
||||
end "and";
|
||||
|
||||
function "or"(Left, Right: Vector) return Vector is
|
||||
V: Vector := Left; -- all sources in Left
|
||||
begin -- ... add all sources in Right, which are not already in Left
|
||||
for I in First_Index(Right) .. Last_Index(Right) loop
|
||||
if not Is_In(Element(Right, I), Left) then
|
||||
V := V & Element(Right, I);
|
||||
end if;
|
||||
end loop;
|
||||
return V;
|
||||
end "or";
|
||||
|
||||
function Empty(Vec: Vector) return Boolean
|
||||
renames Is_Empty;
|
||||
|
||||
function First_Source(The_Sources: Vector)
|
||||
return Source_Type renames First_Element;
|
||||
|
||||
procedure Delete_First_Source(The_Sources: in out Vector;
|
||||
Count: Ada.Containers.Count_Type := 1)
|
||||
renames Delete_First;
|
||||
|
||||
procedure Iterate(The_Sources: Vector) is
|
||||
V: Vector := The_Sources;
|
||||
begin
|
||||
while not Empty(V) loop
|
||||
Do_Something(First_Source(V));
|
||||
Delete_First_Source(V);
|
||||
end loop;
|
||||
end Iterate;
|
||||
|
||||
function Same_Vector(U,V: Vector) return Boolean is
|
||||
begin
|
||||
raise Program_Error with "there is no need to call this function";
|
||||
return False; -- this avoices a compiler warning
|
||||
end Same_Vector;
|
||||
|
||||
end Generic_Inverted_Index;
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
with Gnat.Regpat;
|
||||
|
||||
package Parse_Lines is
|
||||
|
||||
Word_Pattern: constant String := "([a-zA-Z]+)";
|
||||
Filename_Pattern: constant String := "([a-zA-Z0-9_.,;:]+)";
|
||||
|
||||
procedure Search_For_Pattern(Pattern: Gnat.Regpat.Pattern_Matcher;
|
||||
Search_In: String;
|
||||
First, Last: out Positive;
|
||||
Found: out Boolean);
|
||||
|
||||
function Compile(Raw: String) return Gnat.Regpat.Pattern_Matcher;
|
||||
|
||||
generic
|
||||
Pattern: String;
|
||||
with procedure Do_Something(Word: String);
|
||||
procedure Iterate_Words(S: String);
|
||||
|
||||
end Parse_Lines;
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
with Gnat.Regpat;
|
||||
|
||||
package body Parse_Lines is
|
||||
|
||||
procedure Search_For_Pattern(Pattern: Gnat.Regpat.Pattern_Matcher;
|
||||
Search_In: String;
|
||||
First, Last: out Positive;
|
||||
Found: out Boolean) is
|
||||
use Gnat.Regpat;
|
||||
Result: Match_Array (0 .. 1);
|
||||
begin
|
||||
Match(Pattern, Search_In, Result);
|
||||
Found := Result(1) /= No_Match;
|
||||
if Found then
|
||||
First := Result(1).First;
|
||||
Last := Result(1).Last;
|
||||
end if;
|
||||
end Search_For_Pattern;
|
||||
|
||||
function Compile(Raw: String) return Gnat.Regpat.Pattern_Matcher is
|
||||
begin
|
||||
return Gnat.Regpat.Compile(Raw);
|
||||
end Compile;
|
||||
|
||||
procedure Iterate_Words(S: String) is
|
||||
Current_First: Positive := S'First;
|
||||
First, Last: Positive;
|
||||
Found: Boolean;
|
||||
use Parse_Lines;
|
||||
Compiled_P: Gnat.Regpat.Pattern_Matcher := Compile(Pattern);
|
||||
begin
|
||||
loop
|
||||
Search_For_Pattern(Compiled_P,
|
||||
S(Current_First .. S'Last),
|
||||
First, Last, Found);
|
||||
exit when not Found;
|
||||
Do_Something(S(First .. Last));
|
||||
Current_First := Last+1;
|
||||
end loop;
|
||||
end Iterate_Words;
|
||||
|
||||
end Parse_Lines;
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
with Ada.Containers.Indefinite_Vectors;
|
||||
private with Ada.Containers.Indefinite_Hashed_Maps;
|
||||
|
||||
generic
|
||||
type Source_Type (<>) is private;
|
||||
type Item_Type (<>) is private;
|
||||
with function Hash(Item: Item_Type) return Ada.Containers.Hash_Type is <>;
|
||||
package Generic_Inverted_Index is
|
||||
|
||||
type Storage_Type is tagged private;
|
||||
|
||||
package Source_Vecs is new Ada.Containers.Indefinite_Vectors
|
||||
(Index_Type => Positive,
|
||||
Element_Type => Source_Type);
|
||||
|
||||
procedure Store(Storage: in out Storage_Type;
|
||||
Source: Source_Type;
|
||||
Item: Item_Type);
|
||||
-- stores Source in a table, indexed by Item
|
||||
-- if there is already an Item/Source entry, the Table isn_t changed
|
||||
|
||||
function Find(Storage: Storage_Type; Item: Item_Type)
|
||||
return Source_Vecs.Vector;
|
||||
-- Generates a vector of all Sources for the given Item
|
||||
|
||||
function "and"(Left, Right: Source_Vecs.Vector) return Source_Vecs.Vector;
|
||||
-- returns a vector of all sources, which are both in Left and in Right
|
||||
|
||||
function "or"(Left, Right: Source_Vecs.Vector) return Source_Vecs.Vector;
|
||||
-- returns a vector of all sources, which are in Left, Right, or both
|
||||
|
||||
function Empty(Vec: Source_Vecs.Vector) return Boolean;
|
||||
-- returns true if Vec is empty
|
||||
|
||||
type Process_Source is not null access procedure (Source: Source_Type);
|
||||
|
||||
generic
|
||||
with procedure Do_Something(Source: Source_Type);
|
||||
procedure Iterate(The_Sources: Source_Vecs.Vector);
|
||||
-- calls Do_Something(Source) for all sources in The_Sources;
|
||||
|
||||
private
|
||||
|
||||
function Same_Vector(U,V: Source_Vecs.Vector) return Boolean;
|
||||
|
||||
package Maps is new Ada.Containers.Indefinite_Hashed_Maps
|
||||
-- for each item (=key) we store a vector with sources
|
||||
(Key_Type => Item_Type,
|
||||
Element_Type => Source_Vecs.Vector,
|
||||
Hash => Hash,
|
||||
Equivalent_Keys => "=",
|
||||
"=" => Same_Vector);
|
||||
|
||||
type Storage_Type is new Maps.Map with null record;
|
||||
|
||||
end Generic_Inverted_Index;
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
package body Generic_Inverted_Index is
|
||||
-- uses some of the new Ada 2012 syntax
|
||||
use Source_Vecs;
|
||||
|
||||
procedure Store(Storage: in out Storage_Type;
|
||||
Source: Source_Type;
|
||||
Item: Item_Type) is
|
||||
use type Maps.Cursor;
|
||||
begin
|
||||
if (Storage.Find(Item) = Maps.No_Element) then
|
||||
Storage.Insert(Key => Item,
|
||||
New_Item => Empty_Vector & Source);
|
||||
else
|
||||
declare
|
||||
The_Vector: Vector := Storage.Element(Item);
|
||||
begin
|
||||
if The_Vector.Last_Element /= Source then
|
||||
Storage.Replace
|
||||
(Key => Item,
|
||||
New_Item => Storage.Element(Item) & Source);
|
||||
end if;
|
||||
end;
|
||||
end if;
|
||||
end Store;
|
||||
|
||||
function Find(Storage: Storage_Type; Item: Item_Type)
|
||||
return Vector is
|
||||
begin
|
||||
return Storage.Element(Item);
|
||||
exception
|
||||
when Constraint_Error => return Empty_Vector; -- found nothing
|
||||
end Find;
|
||||
|
||||
function Is_In(S: Source_Type; V: Vector) return Boolean is
|
||||
begin
|
||||
for Some_Element of V loop
|
||||
if Some_Element = S then
|
||||
return True;
|
||||
end if;
|
||||
end loop;
|
||||
return False;
|
||||
end Is_In;
|
||||
|
||||
function "and"(Left, Right: Vector) return Vector is
|
||||
V: Vector := Empty_Vector;
|
||||
begin
|
||||
for Some_Element of Left loop
|
||||
if Is_In(Some_Element, Right) then
|
||||
V := V & Some_Element;
|
||||
end if;
|
||||
end loop;
|
||||
return V;
|
||||
end "and";
|
||||
|
||||
function "or"(Left, Right: Vector) return Vector is
|
||||
V: Vector := Left; -- all sources in Left
|
||||
begin
|
||||
for Some_Element of Right loop
|
||||
if not Is_In(Some_Element, Left) then
|
||||
V := V & Some_Element;
|
||||
end if;
|
||||
end loop;
|
||||
return V;
|
||||
end "or";
|
||||
|
||||
function Empty(Vec: Vector) return Boolean
|
||||
renames Is_Empty;
|
||||
|
||||
procedure Iterate(The_Sources: Vector) is
|
||||
begin
|
||||
for Some_Element of The_Sources loop
|
||||
Do_Something(Some_Element);
|
||||
end loop;
|
||||
end Iterate;
|
||||
|
||||
function Same_Vector(U,V: Vector) return Boolean is
|
||||
begin
|
||||
raise Program_Error with "there is no need to call this function";
|
||||
return False; -- this avoices a compiler warning
|
||||
end Same_Vector;
|
||||
|
||||
end Generic_Inverted_Index;
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
function Index-File ( [string[]]$FileList )
|
||||
{
|
||||
# Create index hashtable, as needed
|
||||
If ( -not $Script:WordIndex ) { $Script:WordIndex = @{} }
|
||||
|
||||
# For each file to be indexed...
|
||||
ForEach ( $File in $FileList )
|
||||
{
|
||||
# Find any previously existing entries for this file
|
||||
$ExistingEntries = $Script:WordIndex.Keys | Where { $Script:WordIndex[$_] -contains $File }
|
||||
|
||||
# For any previously existing entries
|
||||
# Delete them (prior to reindexing the file)
|
||||
ForEach ( $Key in $ExistingEntries )
|
||||
{
|
||||
$Script:WordIndex[$Key] = @( $Script:WordIndex[$Key] | Where { $_ -ne $File } )
|
||||
}
|
||||
|
||||
# Get the contents of the file, split on non-alphanumeric characters, and remove duplicates
|
||||
$Words = ( Get-Content $File ) -split '[^a-zA-Z\d]' | Sort -Unique
|
||||
|
||||
# For each word in the file...
|
||||
ForEach ( $Word in $Words )
|
||||
{
|
||||
# If the entry for the word already exists...
|
||||
If ( $Script:WordIndex[$Word] )
|
||||
{
|
||||
# Add the file name to the entry
|
||||
$Script:WordIndex[$Word] += $File
|
||||
}
|
||||
Else
|
||||
{
|
||||
# Create a new entry
|
||||
$Script:WordIndex[$Word] = @( $File )
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Find-Word ( [string]$Word )
|
||||
{
|
||||
return $WordIndex[$Word]
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
# Populate test files
|
||||
@'
|
||||
Files full of
|
||||
various words.
|
||||
'@ | Out-File -FilePath C:\Test\File1.txt
|
||||
|
||||
@'
|
||||
Create an index
|
||||
of words.
|
||||
'@ | Out-File -FilePath C:\Test\File2.txt
|
||||
|
||||
@'
|
||||
Use the index
|
||||
to find the files.
|
||||
'@ | Out-File -FilePath C:\Test\File3.txt
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
# Index files
|
||||
Index-File C:\Test\File1.txt, C:\Test\File2.txt, C:\Test\File3.txt
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
# Query index
|
||||
Find-Word files
|
||||
Loading…
Add table
Add a link
Reference in a new issue