Data commit

Task/Compiler-lexical-analyzer/J/compiler-lexical-analyzer-1.j

@@ -0,0 +1,121 @@
+symbols=:256#0
+ch=: {{1 0+x[symbols=: x (a.i.y)} symbols}}
+'T0 token'  =: 0 ch '%+-!(){};,<>=!|&'
+'L0 letter' =: 1 ch '_',,u:65 97+/i.26
+'D0 digit'  =: 2 ch u:48+i.10
+'S0 space'  =: 3 ch ' ',LF
+'C0 commen' =: 4 ch '/'
+'C1 comment'=: 5 ch '*'
+'q0 quote'  =: 6 ch ''''
+'Q0 dquote' =: 7 ch '"'
+
+width=: 1+>./symbols
+default=: ,:(1+i.width),every 2
+states=:((1+i.width),every 1),width#default
+extend=: {{
+  if.y>#states do.states=: y{.states,y#default
+  end.states
+}}
+pad=: {{if. 0=#y do.y=.#states end.y}}
+function=: {{ NB. x: before, m: op, n: symbol, y: after
+  y[states=: (y,m) (<x,n)} extend 1+x>.y=.pad y
+}}
+{{for_op.y do.(op)=: op_index function end.0}};:'nop init start'
+all=: {{y=.pad y
+  for_symbol.i.width do.
+    x symbol nop y
+  end.y
+}}
+any=: {{y=.pad y
+  for_symbol.i.width do.
+    x symbol start y
+  end.y
+}}
+
+NB. identifiers and keywords
+     L0  letter nop L0
+     L0   digit nop L0
+
+NB. numbers
+     D0   digit nop D0
+     D0  letter nop D0
+
+NB. white space
+     S0   space nop S0
+
+NB. comments
+C1=: C0 comment nop ''
+C2=: C1         all ''
+     C2         all C2
+C3=: C2  commen nop ''
+C4=: C3 comment nop ''
+
+NB. quoted characters
+q1=: q0         any ''
+
+NB. strings
+Q1=: Q0         all ''
+     Q1         all Q1
+Q2=: Q1  dquote nop ''
+     Q0  dquote nop Q2
+
+tokenize=:{{
+  tok=. (0;states;symbols);:y
+  for_fix.cut'<= >= == != && ||'do.
+    M=.;:;fix
+    for_k.|.I.M E.tok do.
+      tok=.(fix,<'') (0 1+k)} tok
+    end.
+  end.tok-.a:
+}}
+
+(tknames=:;: {{)n
+ Op_multiply Op_divide Op_mod Op_add Op_subtract Op_less Op_lessequal
+ Op_greater Op_greaterequal Op_equal Op_notequal Op_not Op_and Op_or
+ Op_assign LeftParen RightParen Keyword_if LeftBrace Keyword_else
+ RightBrace Keyword_while Semicolon Keyword_print Comma Keyword_putc
+}}-.LF)=: tkref=: tokenize '*/%+-<<=>>===!=!&&||=()if{else}while;print,putc'
+NB. the reference tokens here were arranged to avoid whitespace tokens
+NB. also, we reserve multiple token instances where a literal string
+NB. appears in different syntactic productions. Here, we only use the initial
+NB. instances -- the others will be used in the syntax analyzer which
+NB. uses the same tkref and tknames,
+
+shift=: |.!.0
+numvals=: {{
+  ndx=. I.(0<#@>y)**/@> y e.L:0 '0123456789'
+  ({{".y,'x'}}each ndx{y) ndx} y
+}}
+chrvals=: {{
+  q=. y=<,''''
+  s=. y=<,'\'
+  j=. I.(-.s)*(1&shift * _1&shift)q
+  k=. I.(y e.;:'\n')*(1 shift q)*(_2 shift q)*_1 shift s
+  jvals=. a.i.L:0 j{y      NB. not escaped
+  kvals=. (k{s){<"0 a.i.LF,'\' NB. escaped
+  (,a:,jvals,:a:) (,_1 0 1+/j)} (,a:,a:,kvals,:a:) (,_2 _1 0 1+/k)} y
+}}
+
+validstring=: ((1<#)*('"'={.)*('"'={:)*('\'=])-:'\n'&E.(+._1&shift)@+.'\\'&E.) every
+
+validid=: ((<,'\')~:_1&|.) * (e.&tkref) < (e.&(u:I.symbols=letter)@{. * */@(e.&(u:I.symbols e.letter,digit))@}.) every
+
+lex=: {{
+  lineref=.I.y=LF
+  tokens=.(tokenize y),<,'_'
+  offsets=.0,}:#@;\tokens
+  lines=. lineref I.offsets
+  columns=. offsets-lines{0,lineref
+  keep=. -.({.@> tokens)e.u:I.space=symbols
+  names=. (<'End_of_input') _1} (tkref i.tokens) {(_3}.tknames),4#<'Error'
+  unknown=. names=<'Error'
+  values=. a: _1} unknown#inv  numvals chrvals unknown#tokens
+  names=. (<'Integer') (I.(values~:a:)*tokens~:values)} names
+  names=. (<'String') (I.validstring tokens)} names
+  names=. (<'Identifier') (I.validid tokens)} names
+  names=. (<'End_of_input') _1} names
+  comments=. '*/'&-:@(_2&{.)@> tokens
+  whitespace=. (values=tokens) * e.&(' ',LF)@{.@> tokens
+  keep=. (tokens~:<,'''')*-.comments+.whitespace+.unknown*a:=values
+  keep&#each ((1+lines),.columns);<names,.values
+}}

Task/Compiler-lexical-analyzer/J/compiler-lexical-analyzer-2.j

@@ -0,0 +1,66 @@
+flex=: {{
+  'A B'=.y
+  'names values'=.|:":each B
+  (":A),.' ',.names,.' ',.values
+}}@lex
+
+testcase3=: {{)n
+/*
+  All lexical tokens - not syntactically correct, but that will
+  have to wait until syntax analysis
+ */
+/* Print   */  print    /* Sub     */  -
+/* Putc    */  putc     /* Lss     */  <
+/* If      */  if       /* Gtr     */  >
+/* Else    */  else     /* Leq     */  <=
+/* While   */  while    /* Geq     */  >=
+/* Lbrace  */  {        /* Eq      */  ==
+/* Rbrace  */  }        /* Neq     */  !=
+/* Lparen  */  (        /* And     */  &&
+/* Rparen  */  )        /* Or      */  ||
+/* Uminus  */  -        /* Semi    */  ;
+/* Not     */  !        /* Comma   */  ,
+/* Mul     */  *        /* Assign  */  =
+/* Div     */  /        /* Integer */  42
+/* Mod     */  %        /* String  */  "String literal"
+/* Add     */  +        /* Ident   */  variable_name
+/* character literal */  '\n'
+/* character literal */  '\\'
+/* character literal */  ' '
+}}
+
+   flex testcase3
+ 5 16 Keyword_print
+ 5 40 Op_subtract
+ 6 16 Keyword_putc
+ 6 40 Op_less
+ 7 16 Keyword_if
+ 7 40 Op_greater
+ 8 16 Keyword_else
+ 8 40 Op_lessequal
+ 9 16 Keyword_while
+ 9 40 Op_greaterequal
+10 16 LeftBrace
+10 40 Op_equal
+11 16 RightBrace
+11 40 Op_notequal
+12 16 LeftParen
+12 40 Op_and
+13 16 RightParen
+13 40 Op_or
+14 16 Op_subtract
+14 40 Semicolon
+15 16 Op_not
+15 40 Comma
+16 16 Op_multiply
+16 40 Op_assign
+17 16 Op_divide
+17 40 Integer         42
+18 16 Op_mod
+18 40 String          "String literal"
+19 16 Op_add
+19 40 Identifier      variable_name
+20 28 Identifier      10
+21 28 Integer         92
+22 27 Integer         32
+23  1 End_of_input