121 lines
3.5 KiB
Text
121 lines
3.5 KiB
Text
symbols=:256#0
|
|
ch=: {{1 0+x[symbols=: x (a.i.y)} symbols}}
|
|
'T0 token' =: 0 ch '%+-!(){};,<>=!|&'
|
|
'L0 letter' =: 1 ch '_',,u:65 97+/i.26
|
|
'D0 digit' =: 2 ch u:48+i.10
|
|
'S0 space' =: 3 ch ' ',LF
|
|
'C0 commen' =: 4 ch '/'
|
|
'C1 comment'=: 5 ch '*'
|
|
'q0 quote' =: 6 ch ''''
|
|
'Q0 dquote' =: 7 ch '"'
|
|
|
|
width=: 1+>./symbols
|
|
default=: ,:(1+i.width),every 2
|
|
states=:((1+i.width),every 1),width#default
|
|
extend=: {{
|
|
if.y>#states do.states=: y{.states,y#default
|
|
end.states
|
|
}}
|
|
pad=: {{if. 0=#y do.y=.#states end.y}}
|
|
function=: {{ NB. x: before, m: op, n: symbol, y: after
|
|
y[states=: (y,m) (<x,n)} extend 1+x>.y=.pad y
|
|
}}
|
|
{{for_op.y do.(op)=: op_index function end.0}};:'nop init start'
|
|
all=: {{y=.pad y
|
|
for_symbol.i.width do.
|
|
x symbol nop y
|
|
end.y
|
|
}}
|
|
any=: {{y=.pad y
|
|
for_symbol.i.width do.
|
|
x symbol start y
|
|
end.y
|
|
}}
|
|
|
|
NB. identifiers and keywords
|
|
L0 letter nop L0
|
|
L0 digit nop L0
|
|
|
|
NB. numbers
|
|
D0 digit nop D0
|
|
D0 letter nop D0
|
|
|
|
NB. white space
|
|
S0 space nop S0
|
|
|
|
NB. comments
|
|
C1=: C0 comment nop ''
|
|
C2=: C1 all ''
|
|
C2 all C2
|
|
C3=: C2 commen nop ''
|
|
C4=: C3 comment nop ''
|
|
|
|
NB. quoted characters
|
|
q1=: q0 any ''
|
|
|
|
NB. strings
|
|
Q1=: Q0 all ''
|
|
Q1 all Q1
|
|
Q2=: Q1 dquote nop ''
|
|
Q0 dquote nop Q2
|
|
|
|
tokenize=:{{
|
|
tok=. (0;states;symbols);:y
|
|
for_fix.cut'<= >= == != && ||'do.
|
|
M=.;:;fix
|
|
for_k.|.I.M E.tok do.
|
|
tok=.(fix,<'') (0 1+k)} tok
|
|
end.
|
|
end.tok-.a:
|
|
}}
|
|
|
|
(tknames=:;: {{)n
|
|
Op_multiply Op_divide Op_mod Op_add Op_subtract Op_less Op_lessequal
|
|
Op_greater Op_greaterequal Op_equal Op_notequal Op_not Op_and Op_or
|
|
Op_assign LeftParen RightParen Keyword_if LeftBrace Keyword_else
|
|
RightBrace Keyword_while Semicolon Keyword_print Comma Keyword_putc
|
|
}}-.LF)=: tkref=: tokenize '*/%+-<<=>>===!=!&&||=()if{else}while;print,putc'
|
|
NB. the reference tokens here were arranged to avoid whitespace tokens
|
|
NB. also, we reserve multiple token instances where a literal string
|
|
NB. appears in different syntactic productions. Here, we only use the initial
|
|
NB. instances -- the others will be used in the syntax analyzer which
|
|
NB. uses the same tkref and tknames,
|
|
|
|
shift=: |.!.0
|
|
numvals=: {{
|
|
ndx=. I.(0<#@>y)**/@> y e.L:0 '0123456789'
|
|
({{".y,'x'}}each ndx{y) ndx} y
|
|
}}
|
|
chrvals=: {{
|
|
q=. y=<,''''
|
|
s=. y=<,'\'
|
|
j=. I.(-.s)*(1&shift * _1&shift)q
|
|
k=. I.(y e.;:'\n')*(1 shift q)*(_2 shift q)*_1 shift s
|
|
jvals=. a.i.L:0 j{y NB. not escaped
|
|
kvals=. (k{s){<"0 a.i.LF,'\' NB. escaped
|
|
(,a:,jvals,:a:) (,_1 0 1+/j)} (,a:,a:,kvals,:a:) (,_2 _1 0 1+/k)} y
|
|
}}
|
|
|
|
validstring=: ((1<#)*('"'={.)*('"'={:)*('\'=])-:'\n'&E.(+._1&shift)@+.'\\'&E.) every
|
|
|
|
validid=: ((<,'\')~:_1&|.) * (e.&tkref) < (e.&(u:I.symbols=letter)@{. * */@(e.&(u:I.symbols e.letter,digit))@}.) every
|
|
|
|
lex=: {{
|
|
lineref=.I.y=LF
|
|
tokens=.(tokenize y),<,'_'
|
|
offsets=.0,}:#@;\tokens
|
|
lines=. lineref I.offsets
|
|
columns=. offsets-lines{0,lineref
|
|
keep=. -.({.@> tokens)e.u:I.space=symbols
|
|
names=. (<'End_of_input') _1} (tkref i.tokens) {(_3}.tknames),4#<'Error'
|
|
unknown=. names=<'Error'
|
|
values=. a: _1} unknown#inv numvals chrvals unknown#tokens
|
|
names=. (<'Integer') (I.(values~:a:)*tokens~:values)} names
|
|
names=. (<'String') (I.validstring tokens)} names
|
|
names=. (<'Identifier') (I.validid tokens)} names
|
|
names=. (<'End_of_input') _1} names
|
|
comments=. '*/'&-:@(_2&{.)@> tokens
|
|
whitespace=. (values=tokens) * e.&(' ',LF)@{.@> tokens
|
|
keep=. (tokens~:<,'''')*-.comments+.whitespace+.unknown*a:=values
|
|
keep&#each ((1+lines),.columns);<names,.values
|
|
}}
|