A-M baby

Task/Inverted-index/0DESCRIPTION

@@ -0,0 +1,3 @@
+An [[wp:Inverted_index|Inverted Index]] is a data structure used to create full text search.
+
+Given a set of text files, implement a program to create an inverted index.  Also create a user interface to do a search using that inverted index which returns a list of files that contain the query term / terms.  The search index  can be in memory.

Task/Inverted-index/1META.yaml

@@ -0,0 +1,4 @@
+---
+category:
+- Search
+note: Classic CS problems and programs

Task/Inverted-index/Ada/inverted-index-1.ada

@@ -0,0 +1,114 @@
+with Ada.Text_IO, Generic_Inverted_Index, Ada.Strings.Hash, Parse_Lines;
+use Ada.Text_IO;
+
+procedure Inverted_Index is
+
+   type Process_Word is access procedure (Word: String);
+
+   package Inv_Idx is new Generic_Inverted_Index
+     (Source_Type => String,
+      Item_Type   => String,
+      Hash        => Ada.Strings.Hash);
+
+   use Inv_Idx;
+
+   procedure Output(Sources: Source_Vecs.Vector) is
+      Any_Output: Boolean := False;
+
+      procedure Print_Source(S: String) is
+      begin
+         if not Any_Output then -- this is the first source found
+            Put("Found in the following files: ");
+            Any_Output := True;
+         else -- there has been at least one source before
+            Put(", ");
+         end if;
+         Put(S);
+      end Print_Source;
+
+      procedure Print is new Inv_Idx.Iterate(Print_Source);
+
+   begin
+      Print(Sources);
+      if not Any_Output then
+         Put("I did not find this in any of the given files!");
+      end if;
+      New_Line(2);
+   end Output;
+
+   procedure Read_From_File(Table: in out Storage_Type;
+                            Filename: String) is
+      F: File_Type;
+
+      procedure Enter_Word(S: String) is
+      begin
+         Table.Store(Source => Filename,  Item => S);
+      end Enter_Word;
+
+      procedure Store_Words is new
+        Parse_Lines.Iterate_Words(Parse_Lines.Word_Pattern, Enter_Word);
+
+   begin
+      Open(File => F, Mode => In_File, Name => Filename);
+       while not End_Of_File(F) loop
+          Store_Words(Get_Line(F));
+      end loop;
+      Close(F);
+   exception
+      when others =>
+         Put_Line("Something wrong with File '" & Filename & "'");
+         Put_Line("I'll ignore this!");
+   end Read_From_File;
+
+   procedure Read_Files(Tab: out Storage_Type; Line: in String) is
+
+      procedure Read_File(S: String) is
+      begin
+         Read_From_File(Tab, S);
+      end Read_File;
+
+      procedure Read_All is new
+        Parse_Lines.Iterate_Words(Parse_Lines.Filename_Pattern, Read_File);
+
+   begin
+      Read_All(Line);
+   end Read_Files;
+
+   S: Storage_Type;
+   Done: Boolean := False;
+
+begin
+   Put_Line("Enter Filenames:");
+   Read_Files(S, Get_Line);
+   New_Line;
+
+   while not Done loop
+      Put_Line("Enter one or more words to search for; <return> to finish:");
+      declare
+         Words: String := Get_Line;
+         First: Boolean := True;
+         Vec: Source_Vecs.Vector := Source_Vecs.Empty_Vector;
+
+         procedure Compute_Vector(Item: String) is
+         begin
+            if First then
+               Vec := S.Find(Item);
+               First := False;
+            else
+               Vec := Vec and S.Find(Item);
+            end if;
+         end Compute_Vector;
+
+         procedure Compute is new
+           Parse_Lines.Iterate_Words(Parse_Lines.Word_Pattern, Compute_Vector);
+
+      begin
+         if Words = "" then
+            Done := True;
+         else
+            Compute(Words);
+            Output(Vec);
+         end if;
+      end;
+   end loop;
+end Inverted_Index;

Task/Inverted-index/Ada/inverted-index-2.ada

@@ -0,0 +1,63 @@
+with Ada.Containers.Indefinite_Vectors;
+private with Ada.Containers.Indefinite_Hashed_Maps;
+
+generic
+   type Source_Type (<>) is private;
+   type Item_Type (<>) is private;
+   with function Hash(Item: Item_Type) return Ada.Containers.Hash_Type is <>;
+package Generic_Inverted_Index is
+
+   type Storage_Type is tagged private;
+
+   package Source_Vecs is new Ada.Containers.Indefinite_Vectors
+     (Index_Type   => Positive,
+      Element_Type => Source_Type);
+
+   procedure Store(Storage: in out Storage_Type;
+                   Source: Source_Type;
+                   Item: Item_Type);
+   -- stores Source in a table, indexed by Item
+   -- if there is already an Item/Source entry, the Table isn_t changed
+
+   function Find(Storage: Storage_Type; Item: Item_Type)
+                return Source_Vecs.Vector;
+   -- Generates a vector of all Sources for the given Item
+
+   function "and"(Left, Right: Source_Vecs.Vector) return Source_Vecs.Vector;
+   -- returns a vector of all sources, which are both in Left and in Right
+
+   function "or"(Left, Right: Source_Vecs.Vector) return Source_Vecs.Vector;
+   -- returns a vector of all sources, which are in Left, Right, or both
+
+   function Empty(Vec: Source_Vecs.Vector) return Boolean;
+   -- returns true if Vec is empty
+
+   function First_Source(The_Sources: Source_Vecs.Vector) return Source_Type;
+   -- returns the first enty in The_Sources; pre: The_Sourses is not empty
+
+   procedure Delete_First_Source(The_Sources: in out Source_Vecs.Vector;
+                                 Count: Ada.Containers.Count_Type := 1);
+   -- Removes the first Count entries; pre: The_Sourses has that many entries
+
+   type Process_Source is not null access procedure (Source: Source_Type);
+
+   generic
+      with procedure Do_Something(Source: Source_Type);
+   procedure Iterate(The_Sources: Source_Vecs.Vector);
+   -- calls Do_Something(Source) for all sources in The_Sources;
+
+private
+
+   function Same_Vector(U,V: Source_Vecs.Vector) return Boolean;
+
+   package Maps is new Ada.Containers.Indefinite_Hashed_Maps
+     -- for each item (=key) we store a vector with sources
+     (Key_Type         => Item_Type,
+      Element_Type     => Source_Vecs.Vector,
+      Hash             => Hash,
+      Equivalent_Keys  => "=",
+      "="              => Same_Vector);
+
+   type Storage_Type is new Maps.Map with null record;
+
+end Generic_Inverted_Index;

Task/Inverted-index/Ada/inverted-index-3.ada

@@ -0,0 +1,95 @@
+package body Generic_Inverted_Index is
+
+   use Source_Vecs;
+   use type Maps.Cursor;
+
+
+   procedure Store(Storage: in out Storage_Type;
+                   Source: Source_Type;
+                   Item: Item_Type) is
+   begin
+      if (Storage.Find(Item) = Maps.No_Element) then
+         Storage.Insert(Key => Item,
+                        New_Item => Empty_Vector & Source);
+      else
+         declare
+            The_Vector: Vector := Storage.Element(Item);
+         begin
+            if The_Vector.Last_Element /= Source then
+               Storage.Replace
+                 (Key      => Item,
+                  New_Item => Storage.Element(Item) & Source);
+            end if;
+         end;
+      end if;
+   end Store;
+
+   function Find(Storage: Storage_Type; Item: Item_Type)
+                return Vector is
+   begin
+      return Storage.Element(Item);
+   exception
+      when Constraint_Error => return Empty_Vector; -- found nothing
+   end Find;
+
+   function Is_In(S: Source_Type; V: Vector) return Boolean is
+      VV: Vector := V;
+   begin
+      if Empty(V) then
+         return False;
+      elsif First_Source(V) = S then
+         return True;
+      else
+         Delete_First_Source(VV);
+         return Is_In(S, VV);
+      end if;
+   end Is_In;
+
+   function "and"(Left, Right: Vector) return Vector is
+       V: Vector := Empty_Vector;
+   begin
+      for I in First_Index(Left) .. Last_Index(Left) loop
+         if Is_In(Element(Left, I), Right) then
+            V := V & Element(Left, I);
+         end if;
+      end loop;
+      return V;
+   end "and";
+
+   function "or"(Left, Right: Vector) return Vector is
+       V: Vector := Left; -- all sources in Left
+   begin -- ... add all sources in Right, which are not already in Left
+      for I in First_Index(Right) .. Last_Index(Right) loop
+         if not Is_In(Element(Right, I), Left) then
+            V := V & Element(Right, I);
+         end if;
+      end loop;
+      return V;
+   end "or";
+
+   function Empty(Vec: Vector) return Boolean
+     renames Is_Empty;
+
+   function First_Source(The_Sources: Vector)
+                        return Source_Type renames First_Element;
+
+   procedure Delete_First_Source(The_Sources: in out Vector;
+                                 Count: Ada.Containers.Count_Type := 1)
+     renames Delete_First;
+
+   procedure Iterate(The_Sources: Vector) is
+      V: Vector := The_Sources;
+   begin
+      while not Empty(V) loop
+         Do_Something(First_Source(V));
+         Delete_First_Source(V);
+      end loop;
+   end Iterate;
+
+   function Same_Vector(U,V: Vector) return Boolean is
+   begin
+      raise Program_Error with "there is no need to call this function";
+      return False; -- this avoices a compiler warning
+   end Same_Vector;
+
+end Generic_Inverted_Index;

Task/Inverted-index/Ada/inverted-index-4.ada

@@ -0,0 +1,20 @@
+with Gnat.Regpat;
+
+package Parse_Lines is
+
+   Word_Pattern: constant String := "([a-zA-Z]+)";
+   Filename_Pattern: constant String := "([a-zA-Z0-9_.,;:]+)";
+
+   procedure Search_For_Pattern(Pattern: Gnat.Regpat.Pattern_Matcher;
+                                Search_In: String;
+                                First, Last: out Positive;
+                                Found: out Boolean);
+
+   function Compile(Raw: String) return Gnat.Regpat.Pattern_Matcher;
+
+   generic
+      Pattern: String;
+      with procedure Do_Something(Word: String);
+   procedure Iterate_Words(S: String);
+
+end Parse_Lines;

Task/Inverted-index/Ada/inverted-index-5.ada

@@ -0,0 +1,42 @@
+with Gnat.Regpat;
+
+package body Parse_Lines is
+
+   procedure Search_For_Pattern(Pattern: Gnat.Regpat.Pattern_Matcher;
+                                Search_In: String;
+                                First, Last: out Positive;
+                                Found: out Boolean) is
+      use Gnat.Regpat;
+      Result: Match_Array (0 .. 1);
+   begin
+      Match(Pattern, Search_In, Result);
+      Found := Result(1) /= No_Match;
+      if Found then
+         First := Result(1).First;
+         Last := Result(1).Last;
+      end if;
+   end Search_For_Pattern;
+
+   function Compile(Raw: String) return Gnat.Regpat.Pattern_Matcher is
+   begin
+      return Gnat.Regpat.Compile(Raw);
+   end Compile;
+
+      procedure Iterate_Words(S: String) is
+      Current_First: Positive := S'First;
+      First, Last:   Positive;
+      Found:         Boolean;
+      use Parse_Lines;
+      Compiled_P: Gnat.Regpat.Pattern_Matcher := Compile(Pattern);
+   begin
+      loop
+         Search_For_Pattern(Compiled_P,
+                            S(Current_First .. S'Last),
+                            First, Last, Found);
+         exit when not Found;
+         Do_Something(S(First .. Last));
+         Current_First := Last+1;
+      end loop;
+   end Iterate_Words;
+
+end Parse_Lines;

Task/Inverted-index/Ada/inverted-index-6.ada

@@ -0,0 +1,56 @@
+with Ada.Containers.Indefinite_Vectors;
+private with Ada.Containers.Indefinite_Hashed_Maps;
+
+generic
+   type Source_Type (<>) is private;
+   type Item_Type (<>) is private;
+   with function Hash(Item: Item_Type) return Ada.Containers.Hash_Type is <>;
+package Generic_Inverted_Index is
+
+   type Storage_Type is tagged private;
+
+   package Source_Vecs is new Ada.Containers.Indefinite_Vectors
+     (Index_Type   => Positive,
+      Element_Type => Source_Type);
+
+   procedure Store(Storage: in out Storage_Type;
+                   Source: Source_Type;
+                   Item: Item_Type);
+   -- stores Source in a table, indexed by Item
+   -- if there is already an Item/Source entry, the Table isn_t changed
+
+   function Find(Storage: Storage_Type; Item: Item_Type)
+                return Source_Vecs.Vector;
+   -- Generates a vector of all Sources for the given Item
+
+   function "and"(Left, Right: Source_Vecs.Vector) return Source_Vecs.Vector;
+   -- returns a vector of all sources, which are both in Left and in Right
+
+   function "or"(Left, Right: Source_Vecs.Vector) return Source_Vecs.Vector;
+   -- returns a vector of all sources, which are in Left, Right, or both
+
+   function Empty(Vec: Source_Vecs.Vector) return Boolean;
+   -- returns true if Vec is empty
+
+   type Process_Source is not null access procedure (Source: Source_Type);
+
+   generic
+      with procedure Do_Something(Source: Source_Type);
+   procedure Iterate(The_Sources: Source_Vecs.Vector);
+   -- calls Do_Something(Source) for all sources in The_Sources;
+
+private
+
+   function Same_Vector(U,V: Source_Vecs.Vector) return Boolean;
+
+   package Maps is new Ada.Containers.Indefinite_Hashed_Maps
+     -- for each item (=key) we store a vector with sources
+     (Key_Type         => Item_Type,
+      Element_Type     => Source_Vecs.Vector,
+      Hash             => Hash,
+      Equivalent_Keys  => "=",
+      "="              => Same_Vector);
+
+   type Storage_Type is new Maps.Map with null record;
+
+end Generic_Inverted_Index;

Task/Inverted-index/Ada/inverted-index-7.ada

@@ -0,0 +1,82 @@
+package body Generic_Inverted_Index is
+             -- uses some of the new Ada 2012 syntax
+   use Source_Vecs;
+
+   procedure Store(Storage: in out Storage_Type;
+                   Source: Source_Type;
+                   Item: Item_Type) is
+      use type Maps.Cursor;
+   begin
+      if (Storage.Find(Item) = Maps.No_Element) then
+         Storage.Insert(Key => Item,
+                        New_Item => Empty_Vector & Source);
+      else
+         declare
+            The_Vector: Vector := Storage.Element(Item);
+         begin
+            if The_Vector.Last_Element /= Source then
+               Storage.Replace
+                 (Key      => Item,
+                  New_Item => Storage.Element(Item) & Source);
+            end if;
+         end;
+      end if;
+   end Store;
+
+   function Find(Storage: Storage_Type; Item: Item_Type)
+                return Vector is
+   begin
+      return Storage.Element(Item);
+   exception
+      when Constraint_Error => return Empty_Vector; -- found nothing
+   end Find;
+
+   function Is_In(S: Source_Type; V: Vector) return Boolean is
+   begin
+      for Some_Element of V loop
+         if Some_Element = S then
+            return True;
+         end if;
+      end loop;
+      return False;
+   end Is_In;
+
+   function "and"(Left, Right: Vector) return Vector is
+       V: Vector := Empty_Vector;
+   begin
+      for Some_Element of Left loop
+         if Is_In(Some_Element, Right) then
+            V := V & Some_Element;
+         end if;
+      end loop;
+      return V;
+   end "and";
+
+   function "or"(Left, Right: Vector) return Vector is
+       V: Vector := Left; -- all sources in Left
+   begin
+      for Some_Element of Right loop
+         if not Is_In(Some_Element, Left) then
+            V := V & Some_Element;
+         end if;
+      end loop;
+      return V;
+   end "or";
+
+   function Empty(Vec: Vector) return Boolean
+     renames Is_Empty;
+
+   procedure Iterate(The_Sources: Vector) is
+   begin
+      for Some_Element in The_Sources loop
+         Do_Something(Element(Some_Element));
+      end loop;
+   end Iterate;
+
+   function Same_Vector(U,V: Vector) return Boolean is
+   begin
+      raise Program_Error with "there is no need to call this function";
+      return False; -- this avoices a compiler warning
+   end Same_Vector;
+
+end Generic_Inverted_Index;

Task/Inverted-index/AutoHotkey/inverted-index.ahk

@@ -0,0 +1,65 @@
+; http://www.autohotkey.com/forum/viewtopic.php?t=41479
+inputbox, files, files, file pattern such as c:\files\*.txt
+
+word2docs := object() ; autohotkey_L is needed.
+
+stime := A_tickcount
+Loop, %files%, 0,1
+{
+   tooltip,%A_index%  / 500
+
+   wordList := WordsIn(A_LoopFileFullPath)
+   InvertedIndex(wordList, A_loopFileFullpath)
+}
+
+tooltip
+msgbox, % "total time " (A_tickcount-stime)/1000
+
+gosub, search
+return
+
+search:
+Loop
+{
+   InputBox, keyword , input single keyword only
+   msgbox, % foundDocs := findword(keyword)
+}
+return
+
+WordsIn(docpath)
+{
+   FileRead, content, %docpath%
+  spos = 1
+   Loop
+   {
+     if !(spos := Regexmatch(content, "[a-zA-Z]{2,}",match, spos))
+       break
+     spos += strlen(match)
+     this_wordList .= match "`n"
+   }
+
+  Sort, this_wordList, U
+  return this_wordList
+}
+
+InvertedIndex(byref words, docpath)
+{
+   global word2docs
+
+  loop, parse, words, `n,`r
+  {
+    if A_loopField =
+      continue
+    word2docs[A_loopField] := word2docs[A_loopField] docpath "`n"
+  }
+}
+
+findWord(word2find)
+{
+  global word2docs
+
+  if (word2docs[word2find] = "")
+     return ""
+  else
+    return word2docs[word2find]
+}

Task/Inverted-index/BBC-BASIC/inverted-index.bbc

@@ -0,0 +1,72 @@
+      DIM FileList$(4)
+      FileList$() = "BBCKEY0.TXT", "BBCKEY1.TXT", "BBCKEY2.TXT", \
+      \             "BBCKEY3.TXT", "BBCKEY4.TXT"
+
+      DictSize% = 30000
+      DIM Index{(DictSize%-1) word$, link%}
+
+      REM Build inverted index:
+      FOR file% = DIM(FileList$(),1) TO 0 STEP -1
+        filename$ = FileList$(file%)
+        F% = OPENIN(filename$)
+        IF F% = 0 ERROR 100, "Failed to open file"
+
+        WHILE NOT EOF#F%
+          REPEAT C%=BGET#F% : UNTIL C%>64 OR EOF#F% : word$ = CHR$(C%)
+          REPEAT C%=BGET#F% : word$ += CHR$(C%) : UNTIL C%<65
+          word$ = FNlower(LEFT$(word$))
+
+          hash% = FNhash(word$)
+          WHILE Index{(hash%)}.word$<>"" AND Index{(hash%)}.word$<>word$
+            hash% = (hash% + 1) MOD DictSize% : REM Collision
+          ENDWHILE
+          Index{(hash%)}.word$ = word$
+          link% = Index{(hash%)}.link%
+          IF link%=0 OR link%!4<>file% THEN
+            DIM heap% 7 : heap%!4 = file%
+            !heap% = link%
+            Index{(hash%)}.link% = heap% : REM Linked list
+          ENDIF
+        ENDWHILE
+
+        CLOSE #F%
+      NEXT file%
+
+      REM Now query the index:
+      PRINT FNquery("random")
+      PRINT FNquery("envelope")
+      PRINT FNquery("zebra")
+      PRINT FNquery("the")
+      END
+
+      DEF FNquery(A$)
+      LOCAL hash%, link%, temp%
+      A$ = FNlower(A$)
+      hash% = FNhash(A$)
+      temp% = hash%
+      WHILE Index{(hash%)}.word$ <> A$
+        hash% = (hash% + 1) MOD DictSize%
+        IF hash% = temp% THEN = """" + A$ + """ not found"
+      ENDWHILE
+      link% = Index{(hash%)}.link%
+      A$ = """" + A$ + """ found in "
+      WHILE link%
+        A$ += FileList$(link%!4) + ", "
+        link% = !link%
+      ENDWHILE
+      = LEFT$(LEFT$(A$))
+
+      DEF FNhash(A$)
+      LOCAL hash%
+      IF LEN(A$) < 4 A$ += STRING$(4-LEN(A$),CHR$0)
+      hash% = !!^A$
+      IF LEN(A$) > 4 hash% EOR= !(!^A$ + LEN(A$) - 4)
+      = hash% MOD DictSize%
+
+      DEF FNlower(A$)
+      LOCAL A%,C%
+      FOR A% = 1 TO LEN(A$)
+        C% = ASCMID$(A$,A%)
+        IF C% >= 65 IF C% <= 90 MID$(A$,A%,1) = CHR$(C%+32)
+      NEXT
+      = A$

Task/Inverted-index/C/inverted-index-1.c

@@ -0,0 +1,151 @@
+#include <stdio.h>
+#include <stdlib.h>
+
+char chr_legal[] = "abcdefghijklmnopqrstuvwxyz0123456789_-./";
+int  chr_idx[256] = {0};
+char idx_chr[256] = {0};
+
+#define FNAME 0
+typedef struct trie_t *trie, trie_t;
+struct trie_t {
+	trie next[sizeof(chr_legal)]; /* next letter; slot 0 is for file name */
+	int eow;
+};
+
+trie trie_new() { return calloc(sizeof(trie_t), 1); }
+
+#define find_word(r, w) trie_trav(r, w, 1)
+/* tree traversal: returns node if end of word and matches string, optionally
+ * create node if doesn't exist
+ */
+trie trie_trav(trie root, const char * str, int no_create)
+{
+	int c;
+	while (root) {
+		if ((c = str[0]) == '\0') {
+			if (!root->eow && no_create) return 0;
+			break;
+		}
+		if (! (c = chr_idx[c]) ) {
+			str++;
+			continue;
+		}
+
+		if (!root->next[c]) {
+			if (no_create) return 0;
+			root->next[c] = trie_new();
+		}
+		root = root->next[c];
+		str++;
+	}
+	return root;
+}
+
+/*  complete traversal of whole tree, calling callback at each end of word node.
+ *  similar method can be used to free nodes, had we wanted to do that.
+ */
+int trie_all(trie root, char path[], int depth, int (*callback)(char *))
+{
+	int i;
+	if (root->eow && !callback(path)) return 0;
+
+	for (i = 1; i < sizeof(chr_legal); i++) {
+		if (!root->next[i]) continue;
+
+		path[depth] = idx_chr[i];
+		path[depth + 1] = '\0';
+		if (!trie_all(root->next[i], path, depth + 1, callback))
+			return 0;
+	}
+	return 1;
+}
+
+void add_index(trie root, const char *word, const char *fname)
+{
+	trie x = trie_trav(root, word, 0);
+	x->eow = 1;
+
+	if (!x->next[FNAME])
+		x->next[FNAME] = trie_new();
+	x = trie_trav(x->next[FNAME], fname, 0);
+	x->eow = 1;
+}
+
+int print_path(char *path)
+{
+	printf(" %s", path);
+	return 1;
+}
+
+/*  pretend we parsed text files and got lower cased words: dealing     *
+ *  with text file is a whole other animal and would make code too long */
+const char *files[] = { "f1.txt", "source/f2.txt", "other_file" };
+const char *text[][5] ={{ "it", "is", "what", "it", "is" },
+		        { "what", "is", "it", 0 },
+		        { "it", "is", "a", "banana", 0 }};
+
+trie init_tables()
+{
+	int i, j;
+	trie root = trie_new();
+	for (i = 0; i < sizeof(chr_legal); i++) {
+		chr_idx[(int)chr_legal[i]] = i + 1;
+		idx_chr[i + 1] = chr_legal[i];
+	}
+
+/* Enable USE_ADVANCED_FILE_HANDLING to use advanced file handling.
+ * You need to have files named like above files[], with words in them
+ * like in text[][].  Case doesn't matter (told you it's advanced).
+ */
+#define USE_ADVANCED_FILE_HANDLING 0
+#if USE_ADVANCED_FILE_HANDLING
+	void read_file(const char * fname) {
+		char cmd[1024];
+		char word[1024];
+		sprintf(cmd, "perl -p -e 'while(/(\\w+)/g) {print lc($1),\"\\n\"}' %s", fname);
+		FILE *in = popen(cmd, "r");
+		while (!feof(in)) {
+			fscanf(in, "%1000s", word);
+			add_index(root, word, fname);
+		}
+		pclose(in);
+	};
+
+	read_file("f1.txt");
+	read_file("source/f2.txt");
+	read_file("other_file");
+#else
+	for (i = 0; i < 3; i++) {
+		for (j = 0; j < 5; j++) {
+			if (!text[i][j]) break;
+			add_index(root, text[i][j], files[i]);
+		}
+	}
+#endif /*USE_ADVANCED_FILE_HANDLING*/
+
+	return root;
+}
+
+void search_index(trie root, const char *word)
+{
+	char path[1024];
+	printf("Search for \"%s\": ", word);
+	trie found = find_word(root, word);
+
+	if (!found) printf("not found\n");
+	else {
+		trie_all(found->next[FNAME], path, 0, print_path);
+		printf("\n");
+	}
+}
+
+int main()
+{
+	trie root = init_tables();
+
+	search_index(root, "what");
+	search_index(root, "is");
+	search_index(root, "banana");
+	search_index(root, "boo");
+	return 0;
+}

Task/Inverted-index/C/inverted-index-2.c

@@ -0,0 +1,4 @@
+Search for "what":  f1.txt source/f2.txt
+Search for "is":  f1.txt other_file source/f2.txt
+Search for "banana":  other_file
+Search for "boo": not found

Task/Inverted-index/CoffeeScript/inverted-index-1.coffee

@@ -0,0 +1,35 @@
+fs = require 'fs'
+
+make_index = (fns) ->
+  # words are indexed by filename and 1-based line numbers
+  index = {}
+  for fn in fns
+    for line, line_num in fs.readFileSync(fn).toString().split '\n'
+      words = get_words line
+      for word in words
+        word = mangle(word)
+        index[word] ||= []
+        index[word].push [fn, line_num+1]
+  index
+
+grep = (index, word) ->
+  console.log "locations for '#{word}':"
+  locations = index[mangle(word)] || []
+  for location in locations
+    [fn, line_num] = location
+    console.log "#{fn}:#{line_num}"
+  console.log "\n"
+
+get_words = (line) ->
+  words = line.replace(/\W/g, ' ').split ' '
+  (word for word in words when word != '')
+
+mangle = (word) ->
+  # avoid conflicts with words like "constructor"
+  '_' + word
+
+do ->
+  fns = (fn for fn in fs.readdirSync('.') when fn.match /\.coffee/)
+  index = make_index(fns)
+  grep index, 'make_index'
+  grep index, 'sort'

Task/Inverted-index/CoffeeScript/inverted-index-2.coffee

@@ -0,0 +1,15 @@
+> coffee inverted_index.coffee
+locations for 'make_index':
+inverted_index.coffee:3
+inverted_index.coffee:33
+inverted_index.coffee:34
+
+
+locations for 'sort':
+anagrams.coffee:8
+derangements.coffee:14
+heap.coffee:34
+heap.coffee:43
+huffman.coffee:81
+inverted_index.coffee:35
+knuth_sample.coffee:12

Task/Inverted-index/Common-Lisp/inverted-index-1.lisp

@@ -0,0 +1,34 @@
+(defpackage rosettacode.inverted-index
+  (:use cl))
+(in-package rosettacode.inverted-index)
+
+;; Return a list of tokens in the string LINE.  This is rather
+;; complicated as CL has no good standard function to do it.
+(defun tokenize (line)
+  (let ((start 0) (len (length line)))
+    (loop for s = (position-if #'alphanumericp line :start start)
+          while s
+          for e = (position-if-not #'alphanumericp line :start (1+ s))
+          collect (subseq line s e)
+          while (and e (< e len))
+          do (setq start e))))
+
+(defun index-file (index filename)
+  (with-open-file (in filename)
+    (loop for line = (read-line in nil nil)
+          while line
+          do (dolist (token (tokenize line))
+               (pushnew filename (gethash token index '()))))))
+
+(defun build-index (filenames)
+  (let ((index (make-hash-table :test #'equal)))
+    (dolist (f filenames)
+      (index-file index f))
+    index))
+
+;; Find the files for QUERY.  We use the same tokenizer for the query
+;; as for files.
+(defun lookup (index query)
+  (remove-duplicates (loop for token in (tokenize query)
+                           append (gethash token index))
+                     :test #'equal))

Task/Inverted-index/Common-Lisp/inverted-index-2.lisp

@@ -0,0 +1,4 @@
+(defparameter *index* (build-index '("file1.txt" "file2.txt" "file3.txt")))
+(defparameter *query* "foo bar")
+(defparameter *result* (lookup *index* *query*))
+(format t "Result for query ~s: ~{~a~^, ~}~%" *query* *result*)

Task/Inverted-index/D/inverted-index.d

@@ -0,0 +1,33 @@
+import std.stdio, std.algorithm, std.string, std.file, std.regex;
+
+void main() {
+    string[][string] index;
+
+    void parseFile(in string fn) {
+        if (!exists(fn) || !isFile(fn))
+            throw new Exception("File not found");
+
+        foreach (word; readText(fn).splitter(regex(r"\W"))) {
+            word = word.toLower();
+            if (!index.get(word, null).canFind(fn))
+                index[word] ~= fn;
+        }
+    }
+
+    immutable fileNames = ["inv1.txt", "inv2.txt", "inv3.txt"];
+    foreach (fName; fileNames)
+        parseFile(fName);
+
+    while (true) {
+        writef("\nEnter a word to search for: (q to quit): ");
+        immutable w = readln().strip().toLower();
+        if (w == "q") {
+            writeln("quitting.");
+            break;
+        }
+        if (w in index)
+            writefln("'%s' found in%( %).", w, index[w]);
+        else
+            writefln("'%s' not found.", w);
+    }
+}

Task/Inverted-index/Factor/inverted-index-1.factor

@@ -0,0 +1,16 @@
+USING: assocs fry io.encodings.utf8 io.files kernel sequences
+sets splitting vectors ;
+IN: rosettacode.inverted-index
+
+: file-words ( file -- assoc )
+    utf8 file-contents " ,;:!?.()[]{}\n\r" split harvest ;
+: add-to-file-list ( files file -- files )
+    over [ swap [ adjoin ] keep ] [ nip 1vector ] if ;
+: add-to-index ( words index file -- )
+    '[ _ [ _ add-to-file-list ] change-at ] each ;
+: (index-files) ( files index -- )
+   [ [ [ file-words ] keep ] dip swap add-to-index ] curry each ;
+: index-files ( files -- index )
+    H{ } clone [ (index-files) ] keep ;
+: query ( terms index -- files )
+    [ at ] curry map [ ] [ intersect ] map-reduce ;

Task/Inverted-index/Factor/inverted-index-2.factor

@@ -0,0 +1,6 @@
+( scratchpad ) { "f1" "f2" "f3" } index-files
+
+--- Data stack:
+H{ { "a" ~vector~ } { "is" ~vector~ } { "what" ~vector~ } { ...
+( scratchpad ) { "what" "is" "it" } swap query .
+V{ "f1" "f2" }

Task/Inverted-index/Go/inverted-index.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+    "bufio"
+    "bytes"
+    "errors"
+    "fmt"
+    "io"
+    "os"
+)
+
+// inverted index representation
+var index map[string][]int // ints index into indexed
+var indexed []doc
+
+type doc struct {
+    file  string
+    title string
+}
+
+func main() {
+    // initialize representation
+    index = make(map[string][]int)
+
+    // build index
+    if err := indexDir("docs"); err != nil {
+        fmt.Println(err)
+        return
+    }
+
+    // run user interface
+    ui()
+}
+
+func indexDir(dir string) error {
+    df, err := os.Open(dir)
+    if err != nil {
+        return err
+    }
+    fis, err := df.Readdir(-1)
+    if err != nil {
+        return err
+    }
+    if len(fis) == 0 {
+        return errors.New(fmt.Sprintf("no files in %s", dir))
+    }
+    indexed := 0
+    for _, fi := range fis {
+        if !fi.IsDir() {
+            if indexFile(dir + "/" + fi.Name()) {
+                indexed++
+            }
+        }
+    }
+    return nil
+}
+
+func indexFile(fn string) bool {
+    f, err := os.Open(fn)
+    if err != nil {
+        fmt.Println(err)
+        return false // only false return
+    }
+
+    // register new file
+    x := len(indexed)
+    indexed = append(indexed, doc{fn, fn})
+    pdoc := &indexed[x]
+
+    // scan lines
+    r := bufio.NewReader(f)
+    lines := 0
+    for {
+        b, isPrefix, err := r.ReadLine()
+        switch {
+        case err == io.EOF:
+            return true
+        case err != nil:
+            fmt.Println(err)
+            return true
+        case isPrefix:
+            fmt.Printf("%s: unexpected long line\n", fn)
+            return true
+        case lines < 20 && bytes.HasPrefix(b, []byte("Title:")):
+            // in a real program you would write code
+            // to skip the Gutenberg document header
+            // and not index it.
+            pdoc.title = string(b[7:])
+        }
+        // index line of text in b
+        // again, in a real program you would write a much
+        // nicer word splitter.
+    wordLoop:
+        for _, bword := range bytes.Fields(b) {
+            bword := bytes.Trim(bword, ".,-~?!\"'`;:()<>[]{}\\|/=_+*&^%$#@")
+            if len(bword) > 0 {
+                word := string(bword)
+                dl := index[word]
+                for _, d := range dl {
+                    if d == x {
+                        continue wordLoop
+                    }
+                }
+                index[word] = append(dl, x)
+            }
+        }
+    }
+    return true
+}
+
+func ui() {
+    fmt.Println(len(index), "words indexed in", len(indexed), "files")
+    fmt.Println("enter single words to search for")
+    fmt.Println("enter a blank line when done")
+    var word string
+    for {
+        fmt.Print("search word: ")
+        wc, _ := fmt.Scanln(&word)
+        if wc == 0 {
+            return
+        }
+        switch dl := index[word]; len(dl) {
+        case 0:
+            fmt.Println("no match")
+        case 1:
+            fmt.Println("one match:")
+            fmt.Println("   ", indexed[dl[0]].file, indexed[dl[0]].title)
+        default:
+            fmt.Println(len(dl), "matches:")
+            for _, d := range dl {
+                fmt.Println("   ", indexed[d].file, indexed[d].title)
+            }
+        }
+    }
+}

Task/Inverted-index/Haskell/inverted-index.hs

@@ -0,0 +1,32 @@
+import Control.Monad
+import Data.Char (isAlpha, toLower)
+import qualified Data.Map as M
+import qualified Data.IntSet as S
+import System.Environment (getArgs)
+
+main =  do
+    (files, _ : q) <- liftM (break (== "--")) getArgs
+    buildII files >>= mapM_ putStrLn . queryII q
+
+data IIndex = IIndex
+    [FilePath]              -- Files in the index
+    (M.Map String S.IntSet) -- Maps word to indices of the list
+  deriving Show
+
+buildII :: [FilePath] -> IO IIndex
+buildII files =
+    liftM (IIndex files . foldl f M.empty . zip [0..]) $
+    mapM readFile files
+  where f m (i, s) =
+            foldl g m $ map (lowercase . filter isAlpha) $ words s
+          where g m word = M.insertWith S.union word (S.singleton i) m
+
+queryII :: [String] -> IIndex -> [FilePath]
+queryII q (IIndex files m) =
+    map (files !!) $ S.toList $ intersections $
+    map (\word -> M.findWithDefault S.empty (lowercase word) m) q
+
+intersections [] = S.empty
+intersections xs = foldl1 S.intersection xs
+
+lowercase = map toLower

Task/Inverted-index/Icon/inverted-index-1.icon

@@ -0,0 +1,55 @@
+procedure main()
+
+  texts := table()     # substitute for read and parse files
+  texts["T0.txt"] := ["it", "is", "what", "it", "is"]
+  texts["T1.txt"] := ["what", "is", "it"]
+  texts["T2.txt"] := ["it", "is", "a", "banana"]
+
+  every textname := key(texts) do  # build index for each 'text'
+     SII := InvertedIndex(SII,textname,texts[textname])
+
+  TermSearchUI(SII)  # search UI
+
+end
+
+procedure InvertedIndex(ii,k,words)  #: accumulate a simple inverted index
+
+/ii := table(set())    # create lookup table and null set
+every w := !words do {
+   if *ii[w] = 0 then ii[w] := set()  # new word, new set
+   insert(ii[w],k)
+   }
+
+return ii
+end
+
+procedure TermSearchUI(ii)    #: search UI, all words must match
+
+repeat {
+   writes("Enter search terms (^z to quit) : ")
+   terms := map(trim(read() | break))
+
+   x := []
+   terms ? while not pos(0) do {
+      tab(many(' \t'))
+      put(x,tab(upto('\ \t')|0))
+      }
+
+   show("Searching for : ",x)
+   show("Found in : ",s := TermSearch(ii,x)) | show("Not found : ",x)
+   }
+write("End of search")
+return
+end
+
+procedure TermSearch(ii,x)  #: return set of matches or fail
+every s := !x do
+   ( /u := ii[s] ) | (u **:= ii[s])
+if *u > 0 then return u
+end
+
+procedure show(s,x) # display helper
+every writes(s|!x) do writes(" ")
+write()
+return
+end

Task/Inverted-index/Icon/inverted-index-2.icon

@@ -0,0 +1,19 @@
+record InvertedIndexRec(simple,full)
+
+procedure FullInvertedIndex(ii,k,words)  #: accumulate a full inverted index
+
+/ii := InvertedIndexRec( table(set()), table() ) # create lookup table and null set
+
+wc := 0
+every (w := !words, wc +:= 1) do {
+   if *ii.simple[w] = 0 then {
+       ii.simple[w] := set()  # new word, new set
+       ii.full[w] := table()  # also new table
+       }
+   insert(ii.simple[w],k)
+   /ii.full[w,k] := set()
+   insert(ii.full[w,k],wc)
+   }
+
+return ii
+end

Task/Inverted-index/J/inverted-index-1.j

@@ -0,0 +1,24 @@
+require'files regex strings'
+
+rxutf8 0  NB. support latin1 searches for this example, instead of utf8
+files=:words=:buckets=:''
+wordre=: rxcomp '[\w'']+'
+parse=: ,@:rxfrom~ wordre&rxmatches
+
+invert=: verb define
+  files=: files,todo=. ~.y-.files
+  >invert1 each todo
+)
+
+invert1=: verb define
+  file=. files i.<y
+  words=: ~.words,contents=. ~.parse tolower fread jpath y
+  ind=. words i. contents
+  buckets=: buckets,(1+words -&# buckets)#a:
+  #buckets=: (file,~each ind{buckets) ind}buckets
+)
+
+search=: verb define
+  hits=. buckets{~words i.~.parse tolower y
+  files {~ >([-.-.)each/hits
+)

Task/Inverted-index/J/inverted-index-2.j

@@ -0,0 +1,9 @@
+   invert '~help/primer/cut.htm';'~help/primer/end.htm';'~help/primer/gui.htm'
+   >search 'finally learning'
+~help/primer/end.htm
+~help/primer/gui.htm
+   >search 'argument'
+~help/primer/cut.htm
+~help/primer/gui.htm
+   >search 'around'
+~help/primer/gui.htm

Task/Inverted-index/Java/inverted-index-1.java

@@ -0,0 +1,103 @@
+package org.rosettacode;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class InvertedIndex {
+
+	List<String> stopwords = Arrays.asList("a", "able", "about",
+			"across", "after", "all", "almost", "also", "am", "among", "an",
+			"and", "any", "are", "as", "at", "be", "because", "been", "but",
+			"by", "can", "cannot", "could", "dear", "did", "do", "does",
+			"either", "else", "ever", "every", "for", "from", "get", "got",
+			"had", "has", "have", "he", "her", "hers", "him", "his", "how",
+			"however", "i", "if", "in", "into", "is", "it", "its", "just",
+			"least", "let", "like", "likely", "may", "me", "might", "most",
+			"must", "my", "neither", "no", "nor", "not", "of", "off", "often",
+			"on", "only", "or", "other", "our", "own", "rather", "said", "say",
+			"says", "she", "should", "since", "so", "some", "than", "that",
+			"the", "their", "them", "then", "there", "these", "they", "this",
+			"tis", "to", "too", "twas", "us", "wants", "was", "we", "were",
+			"what", "when", "where", "which", "while", "who", "whom", "why",
+			"will", "with", "would", "yet", "you", "your");
+
+	Map<String, List<Tuple>> index = new HashMap<String, List<Tuple>>();
+	List<String> files = new ArrayList<String>();
+
+	public void indexFile(File file) throws IOException {
+		int fileno = files.indexOf(file.getPath());
+		if (fileno == -1) {
+			files.add(file.getPath());
+			fileno = files.size() - 1;
+		}
+
+		int pos = 0;
+		BufferedReader reader = new BufferedReader(new FileReader(file));
+		for (String line = reader.readLine(); line != null; line = reader
+				.readLine()) {
+			for (String _word : line.split("\\W+")) {
+				String word = _word.toLowerCase();
+				pos++;
+				if (stopwords.contains(word))
+					continue;
+				List<Tuple> idx = index.get(word);
+				if (idx == null) {
+					idx = new LinkedList<Tuple>();
+					index.put(word, idx);
+				}
+				idx.add(new Tuple(fileno, pos));
+			}
+		}
+		System.out.println("indexed " + file.getPath() + " " + pos + " words");
+	}
+
+	public void search(List<String> words) {
+		for (String _word : words) {
+			Set<String> answer = new HashSet<String>();
+			String word = _word.toLowerCase();
+			List<Tuple> idx = index.get(word);
+			if (idx != null) {
+				for (Tuple t : idx) {
+					answer.add(files.get(t.fileno));
+				}
+			}
+			System.out.print(word);
+			for (String f : answer) {
+				System.out.print(" " + f);
+			}
+			System.out.println("");
+		}
+	}
+
+	public static void main(String[] args) {
+		try {
+			InvertedIndex idx = new InvertedIndex();
+			for (int i = 1; i < args.length; i++) {
+				idx.indexFile(new File(args[i]));
+			}
+			idx.search(Arrays.asList(args[0].split(",")));
+		} catch (Exception e) {
+			e.printStackTrace();
+		}
+	}
+
+	private class Tuple {
+		private int fileno;
+		private int position;
+
+		public Tuple(int fileno, int position) {
+			this.fileno = fileno;
+			this.position = position;
+		}
+	}
+}

Task/Inverted-index/Java/inverted-index-2.java

@@ -0,0 +1,12 @@
+java -cp bin org.rosettacode.InvertedIndex "huntsman,merit,dog,the,gutenberg,lovecraft,olympian" pg30637.txt pg7025.txt pg82.txt pg9090.txt
+indexed pg30637.txt 106473 words
+indexed pg7025.txt 205714 words
+indexed pg82.txt 205060 words
+indexed pg9090.txt 68962 words
+huntsman pg82.txt pg7025.txt
+merit pg9090.txt pg30637.txt pg82.txt pg7025.txt
+dog pg30637.txt pg82.txt pg7025.txt
+the
+gutenberg pg9090.txt pg30637.txt pg82.txt pg7025.txt
+lovecraft pg30637.txt
+olympian pg30637.txt

Task/Inverted-index/Perl/inverted-index.pl

@@ -0,0 +1,54 @@
+use Set::Object 'set';
+
+# given an array of files, returns the index
+sub createindex
+{
+    my @files = @_;
+
+    my %iindex;
+
+    foreach my $file (@files)
+    {
+	open(F, "<", $file) or die "Can't read file $file: $!";
+	while(<F>) {
+            s/\A\W+//;
+	    foreach my $w (map {lc} grep {length() >= 3} split /\W+/)
+	    {
+		if ( exists($iindex{$w}) )
+		{
+		    $iindex{$w}->insert($file);
+		} else {
+		    $iindex{$w} = set($file);
+		}
+	    }
+	}
+	close(F);
+    }
+    return %iindex;
+}
+
+# given an index, search for words
+sub search_words_with_index
+{
+    my %idx = %{shift()};
+    my @words = @_;
+    my $res = set();
+
+    foreach my $w (map {lc} @words)
+    {
+	$w =~ s/\W+//g;            # strip non-words chars
+        length $w < 3 and next;
+        exists $idx{$w} or return set();
+        $res = $res->is_null
+          ? set(@{$idx{$w}})
+          : $res * $idx{$w};       # set intersection
+    }
+    return @$res;
+}
+
+# TESTING
+# USAGE: invidx.pl the,list,of,words file1 file2 .. fileN
+my @searchwords = split /,/, shift;
+# first arg is a comma-separated list of words to search for
+print "$_\n"
+    foreach search_words_with_index({createindex(@ARGV)}, @searchwords);

Task/Inverted-index/PicoLisp/inverted-index.l

@@ -0,0 +1,15 @@
+(off *MyIndex)
+
+(use Word
+   (for File '("file1" "file2" "file3")
+      (in File
+         (while (skip)
+            (if (idx '*MyIndex (setq Word (till " ^I^J^M" T)) T)
+               (push1 (car @) File)
+               (set Word (cons File)) ) ) ) ) )
+
+(de searchFor @
+   (apply sect
+      (extract
+         '((Word) (val (car (idx '*MyIndex Word))))
+         (rest) ) ) )

Task/Inverted-index/Python/inverted-index-1.py

@@ -0,0 +1,41 @@
+'''
+This implements: http://en.wikipedia.org/wiki/Inverted_index of 28/07/10
+'''
+
+from pprint import pprint as pp
+from glob import glob
+try: reduce
+except: from functools import reduce
+try:    raw_input
+except: raw_input = input
+
+
+def parsetexts(fileglob='InvertedIndex/T*.txt'):
+    texts, words = {}, set()
+    for txtfile in glob(fileglob):
+        with open(txtfile, 'r') as f:
+            txt = f.read().split()
+            words |= set(txt)
+            texts[txtfile.split('\\')[-1]] = txt
+    return texts, words
+
+def termsearch(terms): # Searches simple inverted index
+    return reduce(set.intersection,
+                  (invindex[term] for term in terms),
+                  set(texts.keys()))
+
+texts, words = parsetexts()
+print('\nTexts')
+pp(texts)
+print('\nWords')
+pp(sorted(words))
+
+invindex = {word:set(txt
+                        for txt, wrds in texts.items() if word in wrds)
+            for word in words}
+print('\nInverted Index')
+pp({k:sorted(v) for k,v in invindex.items()})
+
+terms = ["what", "is", "it"]
+print('\nTerm Search for: ' + repr(terms))
+pp(sorted(termsearch(terms)))

Task/Inverted-index/Python/inverted-index-2.py

@@ -0,0 +1,52 @@
+from collections import Counter
+
+
+def termsearch(terms): # Searches full inverted index
+    if not set(terms).issubset(words):
+        return set()
+    return reduce(set.intersection,
+                  (set(x[0] for x in txtindx)
+                   for term, txtindx in finvindex.items()
+                   if term in terms),
+                  set(texts.keys()) )
+
+def phrasesearch(phrase):
+    wordsinphrase = phrase.strip().strip('"').split()
+    if not set(wordsinphrase).issubset(words):
+        return set()
+    #firstword, *otherwords = wordsinphrase # Only Python 3
+    firstword, otherwords = wordsinphrase[0], wordsinphrase[1:]
+    found = []
+    for txt in termsearch(wordsinphrase):
+        # Possible text files
+        for firstindx in (indx for t,indx in finvindex[firstword]
+                          if t == txt):
+            # Over all positions of the first word of the phrase in this txt
+            if all( (txt, firstindx+1 + otherindx) in finvindex[otherword]
+                    for otherindx, otherword in enumerate(otherwords) ):
+                found.append(txt)
+    return found
+
+
+finvindex = {word:set((txt, wrdindx)
+                      for txt, wrds in texts.items()
+                      for wrdindx in (i for i,w in enumerate(wrds) if word==w)
+                      if word in wrds)
+             for word in words}
+print('\nFull Inverted Index')
+pp({k:sorted(v) for k,v in finvindex.items()})
+
+print('\nTerm Search on full inverted index for: ' + repr(terms))
+pp(sorted(termsearch(terms)))
+
+phrase = '"what is it"'
+print('\nPhrase Search for: ' + phrase)
+print(phrasesearch(phrase))
+
+# Show multiple match capability
+phrase = '"it is"'
+print('\nPhrase Search for: ' + phrase)
+ans = phrasesearch(phrase)
+print(ans)
+ans = Counter(ans)
+print('  The phrase is found most commonly in text: ' + repr(ans.most_common(1)[0][0]))

Task/Inverted-index/REXX/inverted-index.rexx

@@ -0,0 +1,64 @@
+/*REXX program illustrates building a simple inverted index & word find.*/
+@.=''                                  /*dictionary of words   (so far).*/
+!=''                                   /*a list of found words (so far).*/
+
+call invertI 0, 'BURMA0.TXT'           /*read file 0 ...                */
+call invertI 1, 'BURMA1.TXT'           /*  "   "   1 ...                */
+call invertI 2, 'BURMA2.TXT'           /*  "   "   2 ...                */
+call invertI 3, 'BURMA3.TXT'           /*  "   "   3 ...                */
+call invertI 4, 'BURMA4.TXT'           /*  "   "   4 ...                */
+call invertI 5, 'BURMA5.TXT'           /*  "   "   5 ...                */
+call invertI 6, 'BURMA6.TXT'           /*  "   "   6 ...                */
+call invertI 7, 'BURMA7.TXT'           /*  "   "   7 ...                */
+call invertI 8, 'BURMA8.TXT'           /*  "   "   8 ...                */
+call invertI 9, 'BURMA9.TXT'           /*  "   "   9 ...                */
+
+call findAword 'does'                  /*find a word.                   */
+call findAword '60'                    /*find another word.             */
+call findAword "don't"                 /*and find another word.         */
+call findAword "burma-shave"           /*and find yet another word.     */
+exit                                   /*stick a fork in it, we're done.*/
+/*──────────────────────────────────FINDAWORD subroutine────────────────*/
+findAword:  procedure expose @.        /*get A word, and uppercase it.  */
+parse arg ox;    arg x                 /*OX= word;  X= uppercase version*/
+_=@.x
+oxo='───'ox"───"
+if _==''  then do
+               say 'word' oxo "not found."
+               return 0
+               end
+_@=_                                   /*save _, pass it back to invoker*/
+say 'word' oxo "found in:"
+                             do until _=='';       parse var _ f w _;  say
+                             say '       file='f ' word='w
+                             end   /*until ... */
+return _@
+/*─────────────────────────────────────INVERTI subroutine───────────────*/
+invertI:  procedure expose @. !;  parse arg #,fn       /*file#, filename*/
+call lineout fn                        /*close the file, just in case.  */
+w=0                                    /*number of words so far.        */
+
+  do  while lines(fn)\==0              /*process the entire file (below)*/
+  _=space(linein(fn))                  /*read 1 line, elide extra blanks*/
+  if _==''  then iterate               /*if blank record, then ignore it*/
+  say 'file' #",record="_              /*echo a record, just to be verbose.*/
+
+    do  until _==''                    /*pick off words until done.     */
+    parse upper var _ xxx _            /*pick off a word (uppercased).  */
+    xxx=stripper(xxx)                  /*strip any ending punctuation.  */
+    if xxx='' then iterate             /*is the word now blank (null) ? */
+    w=w+1                              /*bump the word counter.         */
+    @.xxx=@.xxx # w
+    if wordpos(xxx,!)==0 then !=! xxx  /*add to THE list of words found.*/
+    end   /*until ... */
+  end     /*while lines(fn)¬==0*/
+
+say;   call lineout fn                 /*close the file, just to be neat*/
+return w                               /*return the index of the word.  */
+/*─────────────────────────────────────STRIPPER subroutine──────────────*/
+stripper:  procedure;  parse arg q     /*remove punctuation at word-end.*/
+@punctuation='.,:;?¿!¡'                /*serveral punctuation marks.    */
+                           do j=1  for length(@punctuation)
+                           q=strip(q,'T',substr(@punctuation,j,1))
+                           end   /*j*/
+return q

Task/Inverted-index/Ruby/inverted-index-1.rb

@@ -0,0 +1,29 @@
+if File.exist? "index.dat"
+  @data = Marshal.load open("index.dat")
+else
+  @data = {}
+end
+
+# Let's give the string class the ability to tokenize itsself into lowercase
+# words with no punctuation.
+class String
+  def index_sanitize
+    self.split.collect do |token|
+      token.downcase.gsub(/\W/, '')
+    end
+  end
+end
+
+# Just implementing a simple inverted index here.
+ARGV.each do |filename|
+  open filename do |file|
+    file.read.index_sanitize.each do |word|
+      @data[word] ||= []
+      @data[word] << filename unless @data[word].include? filename
+    end
+  end
+end
+
+open("index.dat", "w") do |index|
+  index.write Marshal.dump(@data)
+end

Task/Inverted-index/Ruby/inverted-index-2.rb

@@ -0,0 +1,22 @@
+if File.exist? "index.dat"
+  @data = Marshal.load open("index.dat")
+else
+  raise "The index data file could not be located."
+end
+
+class String
+  def index_sanitize
+    self.split.collect do |token|
+      token.downcase.gsub(/\W/, '')
+    end
+  end
+end
+
+# Take anything passed in on the command line in any form and break it
+# down the same way we did when making the index.
+ARGV.join(' ').index_sanitize.each do |word|
+  @result ||= @data[word]
+  @result &= @data[word]
+end
+
+p @result

Task/Inverted-index/Tcl/inverted-index-1.tcl

@@ -0,0 +1,89 @@
+package require Tcl 8.5
+proc wordsInString str {
+    # We define "words" to be "maximal sequences of 'word' characters".
+    # The other possible definition is to use 'non-space' characters.
+    regexp -all -inline {\w+} $str
+}
+
+# Adds a document to the index. The index is a map from words to a map
+# from filenames to lists of word locations.
+proc addDocumentToIndex {filename} {
+    global index
+    set f [open $filename]
+    set data [read $f]
+    close $f
+
+    set i 0
+    array set localidx {}
+    foreach word [wordsInString $data] {
+	lappend localidx($word) $i
+	incr i
+    }
+
+    # Transcribe into global index
+    foreach {word places} [array get localidx] {
+	dict set index($word) $filename $places
+    }
+}
+
+# How to use the index to find files containing a word
+proc findFilesForWord {word} {
+    global index
+    if {[info exists index($word)]} {
+	return [dict keys $index($word)]
+    }
+}
+# How to use the index to find files containing all words from a list.
+# Note that this does not use the locations within the file.
+proc findFilesWithAllWords {words} {
+    set files [findFilesForWord [lindex $words 0]]
+    foreach w [lrange $words 1 end] {
+	set wf [findFilesForWord $w]
+	set newfiles {}
+	foreach f $files {
+	    if {$f in $wf} {lappend newfiles $f}
+	}
+	set files $newfiles
+    }
+    return $files
+}
+
+# How to use the index to find a sequence of words in a file.
+proc findFilesWithWordSequence {words} {
+    global index
+    set files {}
+    foreach w $words {
+	if {![info exist index($w)]} {
+	    return
+	}
+    }
+    dict for {file places} $index([lindex $words 0]) {
+	if {$file in $files} continue
+	foreach start $places {
+	    set gotStart 1
+	    foreach w [lrange $words 1 end] {
+		incr start
+		set gotNext 0
+		foreach {f ps} $index($w) {
+		    if {$f ne $file} continue
+		    foreach p $ps {
+			if {$p == $start} {
+			    set gotNext 1
+			    break
+			}
+		    }
+		    if {$gotNext} break
+		}
+		if {!$gotNext} {
+		    set gotStart 0
+		    break
+		}
+	    }
+	    if {$gotStart} {
+		lappend files $file
+		break
+	    }
+	}
+    }
+    return $files
+}

Task/Inverted-index/Tcl/inverted-index-2.tcl

@@ -0,0 +1,35 @@
+package require Tk
+pack [labelframe .files -text Files] -side left -fill y
+pack [listbox .files.list -listvariable files]
+pack [button .files.add -command AddFile -text "Add File to Index"]
+pack [labelframe .found -text Found] -side right -fill y
+pack [listbox .found.list -listvariable found] -fill x
+pack [entry .found.entry -textvariable terms] -fill x
+pack [button .found.findAll -command FindAll \
+	-text "Find File with All"] -side left
+pack [button .found.findSeq -command FindSeq \
+	-text "Find File with Sequence"] -side right
+
+# The actions invoked by various GUI buttons
+proc AddFile {} {
+    global files
+    set f [tk_getOpenFile]
+    if {$f ne ""} {
+	addDocumentToIndex $f
+	lappend files $f
+    }
+}
+proc FindAll {} {
+    global found terms
+    set words [wordsInString $terms]
+    set fs [findFilesWithAllWords $words]
+    lappend found "Searching for files with all $terms" {*}$fs \
+	"---------------------"
+}
+proc FindSeq {} {
+    global found terms
+    set words [wordsInString $terms]
+    set fs [findFilesWithWordSequence $words]
+    lappend found "Searching for files with \"$terms\"" {*}$fs \
+	"---------------------"
+}