Data commit
This commit is contained in:
Ingy döt Net
parent
7387c8f97b
commit
cb5bb5e222
199093 changed files with 3378972 additions and 0 deletions
|
|
@ -0,0 +1,191 @@
|
|||
import re, strformat, strutils
|
||||
|
||||
type
|
||||
TokenKind = enum
|
||||
tkUnknown = "UNKNOWN_TOKEN",
|
||||
tkMul = "Op_multiply",
|
||||
tkDiv = "Op_divide",
|
||||
tkMod = "Op_mod",
|
||||
tkAdd = "Op_add",
|
||||
tkSub = "Op_subtract",
|
||||
tkNeg = "Op_negate",
|
||||
tkLt = "Op_less",
|
||||
tkLte = "Op_lessequal",
|
||||
tkGt = "Op_greater",
|
||||
tkGte = "Op_greaterequal",
|
||||
tkEq = "Op_equal",
|
||||
tkNeq = "Op_notequal",
|
||||
tkNot = "Op_not",
|
||||
tkAsgn = "Op_assign",
|
||||
tkAnd = "Op_and",
|
||||
tkOr = "Op_or",
|
||||
tkLpar = "LeftParen",
|
||||
tkRpar = "RightParen",
|
||||
tkLbra = "LeftBrace",
|
||||
tkRbra = "RightBrace",
|
||||
tkSmc = "Semicolon",
|
||||
tkCom = "Comma",
|
||||
tkIf = "Keyword_if",
|
||||
tkElse = "Keyword_else",
|
||||
tkWhile = "Keyword_while",
|
||||
tkPrint = "Keyword_print",
|
||||
tkPutc = "Keyword_putc",
|
||||
tkId = "Identifier",
|
||||
tkInt = "Integer",
|
||||
tkChar = "Integer",
|
||||
tkStr = "String",
|
||||
tkEof = "End_of_input"
|
||||
|
||||
Token = object
|
||||
kind: TokenKind
|
||||
value: string
|
||||
|
||||
TokenAnn = object
|
||||
## Annotated token with messages for compiler
|
||||
token: Token
|
||||
line, column: int
|
||||
|
||||
proc getSymbols(table: openArray[(char, TokenKind)]): seq[char] =
|
||||
result = newSeq[char]()
|
||||
for ch, tokenKind in items(table):
|
||||
result.add ch
|
||||
|
||||
const
|
||||
tkSymbols = { # single-char tokens
|
||||
'*': tkMul,
|
||||
'%': tkMod,
|
||||
'+': tkAdd,
|
||||
'-': tkSub,
|
||||
'(': tkLpar,
|
||||
')': tkRpar,
|
||||
'{': tkLbra,
|
||||
'}': tkRbra,
|
||||
';': tkSmc,
|
||||
',': tkCom,
|
||||
'/': tkDiv, # the comment case /* ... */ is handled in `stripUnimportant`
|
||||
}
|
||||
symbols = getSymbols(tkSymbols)
|
||||
|
||||
proc findTokenKind(table: openArray[(char, TokenKind)]; needle: char):
|
||||
TokenKind =
|
||||
for ch, tokenKind in items(table):
|
||||
if ch == needle: return tokenKind
|
||||
tkUnknown
|
||||
|
||||
proc stripComment(text: var string, lineNo, colNo: var int) =
|
||||
var matches: array[1, string]
|
||||
|
||||
if match(text, re"\A(/\*[\s\S]*?\*/)", matches):
|
||||
text = text[matches[0].len..^1]
|
||||
for s in matches[0]:
|
||||
if s == '\n':
|
||||
inc lineNo
|
||||
colNo = 1
|
||||
else:
|
||||
inc colNo
|
||||
|
||||
proc stripUnimportant(text: var string; lineNo, colNo: var int) =
|
||||
while true:
|
||||
if text.len == 0: return
|
||||
elif text[0] == '\n':
|
||||
inc lineNo
|
||||
colNo = 1
|
||||
text = text[1..^1]
|
||||
elif text[0] == ' ':
|
||||
inc colNo
|
||||
text = text[1..^1]
|
||||
elif text.len >= 2 and text[0] == '/' and text[1] == '*':
|
||||
stripComment(text, lineNo, colNo)
|
||||
else: return
|
||||
|
||||
proc lookAhead(ch1, ch2: char, tk1, tk2: TokenKind): (TokenKind, int) =
|
||||
if ch1 == ch2: (tk1, 2)
|
||||
else: (tk2, 1)
|
||||
|
||||
proc consumeToken(text: var string; tkl: var int): Token =
|
||||
## Return token removing it from the `text` and write its length to
|
||||
## `tkl`. If the token can not be defined, return `tkUnknown` as a
|
||||
## token, shrink text by 1 and write 1 to its length.
|
||||
|
||||
var
|
||||
matches: array[1, string]
|
||||
tKind: TokenKind
|
||||
val: string
|
||||
|
||||
if text.len == 0:
|
||||
(tKind, tkl) = (tkEof, 0)
|
||||
|
||||
# Simple characters
|
||||
elif text[0] in symbols: (tKind, tkl) = (tkSymbols.findTokenKind(text[0]), 1)
|
||||
elif text[0] == '<': (tKind, tkl) = lookAhead(text[1], '=', tkLte, tkLt)
|
||||
elif text[0] == '>': (tKind, tkl) = lookAhead(text[1], '=', tkGte, tkGt)
|
||||
elif text[0] == '=': (tKind, tkl) = lookAhead(text[1], '=', tkEq, tkAsgn)
|
||||
elif text[0] == '!': (tKind, tkl) = lookAhead(text[1], '=', tkNeq, tkNot)
|
||||
elif text[0] == '&': (tKind, tkl) = lookAhead(text[1], '&', tkAnd, tkUnknown)
|
||||
elif text[0] == '|': (tKind, tkl) = lookAhead(text[1], '|', tkOr, tkUnknown)
|
||||
|
||||
# Keywords
|
||||
elif match(text, re"\Aif\b"): (tKind, tkl) = (tkIf, 2)
|
||||
elif match(text, re"\Aelse\b"): (tKind, tkl) = (tkElse, 4)
|
||||
elif match(text, re"\Awhile\b"): (tKind, tkl) = (tkWhile, 5)
|
||||
elif match(text, re"\Aprint\b"): (tKind, tkl) = (tkPrint, 5)
|
||||
elif match(text, re"\Aputc\b"): (tKind, tkl) = (tkPutc, 4)
|
||||
|
||||
# Literals and identifiers
|
||||
elif match(text, re"\A([0-9]+)", matches):
|
||||
(tKind, tkl) = (tkInt, matches[0].len)
|
||||
val = matches[0]
|
||||
elif match(text, re"\A([_a-zA-Z][_a-zA-Z0-9]*)", matches):
|
||||
(tKind, tkl) = (tkId, matches[0].len)
|
||||
val = matches[0]
|
||||
elif match(text, re"\A('(?:[^'\n]|\\\\|\\n)')", matches):
|
||||
(tKind, tkl) = (tkChar, matches[0].len)
|
||||
val = case matches[0]
|
||||
of r"' '": $ord(' ')
|
||||
of r"'\n'": $ord('\n')
|
||||
of r"'\\'": $ord('\\')
|
||||
else: $ord(matches[0][1]) # "'a'"[1] == 'a'
|
||||
elif match(text, re"\A(""[^""\n]*"")", matches):
|
||||
(tKind, tkl) = (tkStr, matches[0].len)
|
||||
val = matches[0]
|
||||
else: (tKind, tkl) = (tkUnknown, 1)
|
||||
|
||||
text = text[tkl..^1]
|
||||
Token(kind: tKind, value: val)
|
||||
|
||||
proc tokenize*(text: string): seq[TokenAnn] =
|
||||
result = newSeq[TokenAnn]()
|
||||
var
|
||||
lineNo, colNo: int = 1
|
||||
text = text
|
||||
token: Token
|
||||
tokenLength: int
|
||||
|
||||
while text.len > 0:
|
||||
stripUnimportant(text, lineNo, colNo)
|
||||
token = consumeToken(text, tokenLength)
|
||||
result.add TokenAnn(token: token, line: lineNo, column: colNo)
|
||||
inc colNo, tokenLength
|
||||
|
||||
proc output*(s: seq[TokenAnn]): string =
|
||||
var
|
||||
tokenKind: TokenKind
|
||||
value: string
|
||||
line, column: int
|
||||
|
||||
for tokenAnn in items(s):
|
||||
line = tokenAnn.line
|
||||
column = tokenAnn.column
|
||||
tokenKind = tokenAnn.token.kind
|
||||
value = tokenAnn.token.value
|
||||
result.add(
|
||||
fmt"{line:>5}{column:>7} {tokenKind:<15}{value}"
|
||||
.strip(leading = false) & "\n")
|
||||
|
||||
when isMainModule:
|
||||
import os
|
||||
|
||||
let input = if paramCount() > 0: readFile paramStr(1)
|
||||
else: readAll stdin
|
||||
|
||||
echo input.tokenize.output
|
||||
|
|
@ -0,0 +1,310 @@
|
|||
import lexbase, streams
|
||||
from strutils import Whitespace
|
||||
|
||||
type
|
||||
TokenKind = enum
|
||||
tkInvalid = "Invalid",
|
||||
tkOpMultiply = "Op_multiply",
|
||||
tkOpDivide = "Op_divide",
|
||||
tkOpMod = "Op_mod",
|
||||
tkOpAdd = "Op_add",
|
||||
tkOpSubtract = "Op_subtract",
|
||||
tkOpLess = "Op_less",
|
||||
tkOpLessEqual = "Op_lessequal",
|
||||
tkOpGreater = "Op_greater",
|
||||
tkOpGreaterEqual = "Op_greaterequal",
|
||||
tkOpEqual = "Op_equal",
|
||||
tkOpNotEqual = "Op_notequal",
|
||||
tkOpNot = "Op_not",
|
||||
tkOpAssign = "Op_assign",
|
||||
tkOpAnd = "Op_and",
|
||||
tkOpOr = "Op_or",
|
||||
tkLeftParen = "LeftParen",
|
||||
tkRightParen = "RightParen",
|
||||
tkLeftBrace = "LeftBrace",
|
||||
tkRightBrace = "RightBrace",
|
||||
tkSemicolon = "Semicolon",
|
||||
tkComma = "Comma",
|
||||
tkKeywordIf = "Keyword_if",
|
||||
tkKeywordElse = "Keyword_else",
|
||||
tkKeywordWhile = "Keyword_while",
|
||||
tkKeywordPrint = "Keyword_print",
|
||||
tkKeywordPutc = "Keyword_putc",
|
||||
tkIdentifier = "Identifier",
|
||||
tkInteger = "Integer",
|
||||
tkString = "String",
|
||||
tkEndOfInput = "End_of_input"
|
||||
|
||||
Lexer = object of BaseLexer
|
||||
kind: TokenKind
|
||||
token, error: string
|
||||
startPos: int
|
||||
|
||||
template setError(l: var Lexer; err: string): untyped =
|
||||
l.kind = tkInvalid
|
||||
if l.error.len == 0:
|
||||
l.error = err
|
||||
|
||||
proc hasError(l: Lexer): bool {.inline.} =
|
||||
l.error.len > 0
|
||||
|
||||
proc open(l: var Lexer; input: Stream) {.inline.} =
|
||||
lexbase.open(l, input)
|
||||
l.startPos = 0
|
||||
l.kind = tkInvalid
|
||||
l.token = ""
|
||||
l.error = ""
|
||||
|
||||
proc handleNewLine(l: var Lexer) =
|
||||
case l.buf[l.bufpos]
|
||||
of '\c': l.bufpos = l.handleCR l.bufpos
|
||||
of '\n': l.bufpos = l.handleLF l.bufpos
|
||||
else: discard
|
||||
|
||||
proc skip(l: var Lexer) =
|
||||
while true:
|
||||
case l.buf[l.bufpos]
|
||||
of Whitespace:
|
||||
if l.buf[l.bufpos] notin NewLines:
|
||||
inc l.bufpos
|
||||
else:
|
||||
handleNewLine l
|
||||
of '/':
|
||||
if l.buf[l.bufpos + 1] == '*':
|
||||
inc l.bufpos, 2
|
||||
while true:
|
||||
case l.buf[l.bufpos]
|
||||
of '*':
|
||||
if l.buf[l.bufpos + 1] == '/':
|
||||
inc l.bufpos, 2
|
||||
break
|
||||
else: inc l.bufpos
|
||||
of NewLines:
|
||||
handleNewLine l
|
||||
of EndOfFile:
|
||||
setError l, "EOF reached in comment"
|
||||
return
|
||||
else:
|
||||
inc l.bufpos
|
||||
else: break
|
||||
else: break
|
||||
|
||||
proc handleSpecial(l: var Lexer): char =
|
||||
assert l.buf[l.bufpos] == '\\'
|
||||
inc l.bufpos
|
||||
case l.buf[l.bufpos]
|
||||
of 'n':
|
||||
l.token.add "\\n"
|
||||
result = '\n'
|
||||
inc l.bufpos
|
||||
of '\\':
|
||||
l.token.add "\\\\"
|
||||
result = '\\'
|
||||
inc l.bufpos
|
||||
else:
|
||||
setError l, "Unknown escape sequence: '\\" & l.buf[l.bufpos] & "'"
|
||||
result = '\0'
|
||||
|
||||
proc handleChar(l: var Lexer) =
|
||||
assert l.buf[l.bufpos] == '\''
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
l.kind = tkInvalid
|
||||
inc l.bufpos
|
||||
if l.buf[l.bufpos] == '\\':
|
||||
l.token = $ord(handleSpecial l)
|
||||
if hasError l: return
|
||||
elif l.buf[l.bufpos] == '\'':
|
||||
setError l, "Empty character constant"
|
||||
return
|
||||
else:
|
||||
l.token = $ord(l.buf[l.bufpos])
|
||||
inc l.bufpos
|
||||
if l.buf[l.bufpos] == '\'':
|
||||
l.kind = tkInteger
|
||||
inc l.bufpos
|
||||
else:
|
||||
setError l, "Multi-character constant"
|
||||
|
||||
proc handleString(l: var Lexer) =
|
||||
assert l.buf[l.bufpos] == '"'
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
l.token = "\""
|
||||
inc l.bufpos
|
||||
while true:
|
||||
case l.buf[l.bufpos]
|
||||
of '\\':
|
||||
discard handleSpecial l
|
||||
if hasError l: return
|
||||
of '"':
|
||||
l.kind = tkString
|
||||
add l.token, '"'
|
||||
inc l.bufpos
|
||||
break
|
||||
of NewLines:
|
||||
setError l, "EOL reached before end-of-string"
|
||||
return
|
||||
of EndOfFile:
|
||||
setError l, "EOF reached before end-of-string"
|
||||
return
|
||||
else:
|
||||
add l.token, l.buf[l.bufpos]
|
||||
inc l.bufpos
|
||||
|
||||
proc handleNumber(l: var Lexer) =
|
||||
assert l.buf[l.bufpos] in {'0'..'9'}
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
l.token = "0"
|
||||
while l.buf[l.bufpos] == '0': inc l.bufpos
|
||||
while true:
|
||||
case l.buf[l.bufpos]
|
||||
of '0'..'9':
|
||||
if l.token == "0":
|
||||
setLen l.token, 0
|
||||
add l.token, l.buf[l.bufpos]
|
||||
inc l.bufpos
|
||||
of 'a'..'z', 'A'..'Z', '_':
|
||||
setError l, "Invalid number"
|
||||
return
|
||||
else:
|
||||
l.kind = tkInteger
|
||||
break
|
||||
|
||||
proc handleIdent(l: var Lexer) =
|
||||
assert l.buf[l.bufpos] in {'a'..'z'}
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
setLen l.token, 0
|
||||
while true:
|
||||
if l.buf[l.bufpos] in {'a'..'z', 'A'..'Z', '0'..'9', '_'}:
|
||||
add l.token, l.buf[l.bufpos]
|
||||
inc l.bufpos
|
||||
else:
|
||||
break
|
||||
l.kind = case l.token
|
||||
of "if": tkKeywordIf
|
||||
of "else": tkKeywordElse
|
||||
of "while": tkKeywordWhile
|
||||
of "print": tkKeywordPrint
|
||||
of "putc": tkKeywordPutc
|
||||
else: tkIdentifier
|
||||
|
||||
proc getToken(l: var Lexer): TokenKind =
|
||||
l.kind = tkInvalid
|
||||
setLen l.token, 0
|
||||
skip l
|
||||
|
||||
case l.buf[l.bufpos]
|
||||
of '*':
|
||||
l.kind = tkOpMultiply
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of '/':
|
||||
l.kind = tkOpDivide
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of '%':
|
||||
l.kind = tkOpMod
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of '+':
|
||||
l.kind = tkOpAdd
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of '-':
|
||||
l.kind = tkOpSubtract
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of '<':
|
||||
l.kind = tkOpLess
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
if l.buf[l.bufpos] == '=':
|
||||
l.kind = tkOpLessEqual
|
||||
inc l.bufpos
|
||||
of '>':
|
||||
l.kind = tkOpGreater
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
if l.buf[l.bufpos] == '=':
|
||||
l.kind = tkOpGreaterEqual
|
||||
inc l.bufpos
|
||||
of '=':
|
||||
l.kind = tkOpAssign
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
if l.buf[l.bufpos] == '=':
|
||||
l.kind = tkOpEqual
|
||||
inc l.bufpos
|
||||
of '!':
|
||||
l.kind = tkOpNot
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
if l.buf[l.bufpos] == '=':
|
||||
l.kind = tkOpNotEqual
|
||||
inc l.bufpos
|
||||
of '&':
|
||||
if l.buf[l.bufpos + 1] == '&':
|
||||
l.kind = tkOpAnd
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos, 2
|
||||
else:
|
||||
setError l, "Unrecognized character"
|
||||
of '|':
|
||||
if l.buf[l.bufpos + 1] == '|':
|
||||
l.kind = tkOpOr
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos, 2
|
||||
else:
|
||||
setError l, "Unrecognized character"
|
||||
of '(':
|
||||
l.kind = tkLeftParen
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of ')':
|
||||
l.kind = tkRightParen
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of '{':
|
||||
l.kind = tkLeftBrace
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of '}':
|
||||
l.kind = tkRightBrace
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of ';':
|
||||
l.kind = tkSemicolon
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of ',':
|
||||
l.kind = tkComma
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
inc l.bufpos
|
||||
of '\'': handleChar l
|
||||
of '"': handleString l
|
||||
of '0'..'9': handleNumber l
|
||||
of 'a'..'z', 'A'..'Z': handleIdent l
|
||||
of EndOfFile:
|
||||
l.startPos = l.getColNumber l.bufpos
|
||||
l.kind = tkEndOfInput
|
||||
else:
|
||||
setError l, "Unrecognized character"
|
||||
result = l.kind
|
||||
|
||||
when isMainModule:
|
||||
import os, strformat
|
||||
proc main() =
|
||||
var l: Lexer
|
||||
if paramCount() < 1:
|
||||
open l, newFileStream stdin
|
||||
else:
|
||||
open l, newFileStream paramStr(1)
|
||||
while l.getToken notin {tkInvalid}:
|
||||
stdout.write &"{l.lineNumber:5} {l.startPos + 1:5} {l.kind:<14}"
|
||||
if l.kind in {tkIdentifier, tkInteger, tkString}:
|
||||
stdout.write &" {l.token}"
|
||||
stdout.write '\n'
|
||||
if l.kind == tkEndOfInput:
|
||||
break
|
||||
if hasError l:
|
||||
echo &"({l.lineNumber},{l.getColNumber l.bufpos + 1}) {l.error}"
|
||||
main()
|
||||
|
|
@ -0,0 +1,221 @@
|
|||
import strutils
|
||||
|
||||
type
|
||||
|
||||
TokenKind* = enum
|
||||
tokMult = "Op_multiply", tokDiv = "Op_divide", tokMod = "Op_mod",
|
||||
tokAdd = "Op_add", tokSub = "Op_subtract", tokLess = "Op_less",
|
||||
tokLessEq = "Op_lessequal", tokGreater = "Op_greater",
|
||||
tokGreaterEq = "Op_greaterequal", tokEq = "Op_equal",
|
||||
tokNotEq = "Op_notequal", tokNot = "Op_not", tokAssign = "Op_assign",
|
||||
tokAnd = "Op_and", tokOr = "Op_or"
|
||||
tokLPar = "LeftParen", tokRPar = "RightParen"
|
||||
tokLBrace = "LeftBrace", tokRBrace = "RightBrace"
|
||||
tokSemi = "Semicolon", tokComma = "Comma"
|
||||
tokIf = "Keyword_if", tokElse = "Keyword_else", tokWhile = "Keyword_while",
|
||||
tokPrint = "Keyword_print", tokPutc = "Keyword_putc"
|
||||
tokIdent = "Identifier", tokInt = "Integer", tokChar = "Integer",
|
||||
tokString = "String"
|
||||
tokEnd = "End_of_input"
|
||||
|
||||
Token* = object
|
||||
ln*, col*: int
|
||||
case kind*: TokenKind
|
||||
of tokIdent: ident*: string
|
||||
of tokInt: intVal*: int
|
||||
of tokChar: charVal*: char
|
||||
of tokString: stringVal*: string
|
||||
else: discard
|
||||
|
||||
Lexer* = object
|
||||
input: string
|
||||
pos: int
|
||||
ln, col: int
|
||||
|
||||
LexicalError* = object of CatchableError
|
||||
ln*, col*: int
|
||||
|
||||
proc error(lexer: var Lexer, message: string) =
|
||||
var err = newException(LexicalError, message)
|
||||
err.ln = lexer.ln
|
||||
err.col = lexer.col
|
||||
|
||||
template current: char =
|
||||
if lexer.pos < lexer.input.len: lexer.input[lexer.pos]
|
||||
else: '\x00'
|
||||
template get(n: int): string =
|
||||
if lexer.pos < lexer.input.len:
|
||||
lexer.input[min(lexer.pos, lexer.input.len)..
|
||||
min(lexer.pos + n - 1, lexer.input.len)]
|
||||
else: ""
|
||||
|
||||
template next() =
|
||||
inc(lexer.pos); inc(lexer.col)
|
||||
if current() == '\n':
|
||||
inc(lexer.ln)
|
||||
lexer.col = 0
|
||||
elif current() == '\r':
|
||||
lexer.col = 0
|
||||
|
||||
proc skip(lexer: var Lexer) =
|
||||
while true:
|
||||
if current() in Whitespace:
|
||||
while current() in Whitespace:
|
||||
next()
|
||||
continue
|
||||
elif get(2) == "/*":
|
||||
next(); next()
|
||||
while get(2) != "*/":
|
||||
if current() == '\x00':
|
||||
lexer.error("Unterminated comment")
|
||||
next()
|
||||
next(); next()
|
||||
continue
|
||||
else: discard
|
||||
break
|
||||
|
||||
proc charOrEscape(lexer: var Lexer): char =
|
||||
if current() != '\\':
|
||||
result = current()
|
||||
next()
|
||||
else:
|
||||
next()
|
||||
case current()
|
||||
of 'n': result = '\n'
|
||||
of '\\': result = '\\'
|
||||
else: lexer.error("Unknown escape sequence '\\" & current() & "'")
|
||||
next()
|
||||
|
||||
proc next*(lexer: var Lexer): Token =
|
||||
let
|
||||
ln = lexer.ln
|
||||
col = lexer.col
|
||||
|
||||
case current()
|
||||
of '*': result = Token(kind: tokMult); next()
|
||||
of '/': result = Token(kind: tokDiv); next()
|
||||
of '%': result = Token(kind: tokMod); next()
|
||||
of '+': result = Token(kind: tokAdd); next()
|
||||
of '-': result = Token(kind: tokSub); next()
|
||||
of '<':
|
||||
next()
|
||||
if current() == '=': result = Token(kind: tokLessEq)
|
||||
else: result = Token(kind: tokLess)
|
||||
of '>':
|
||||
next()
|
||||
if current() == '=':
|
||||
result = Token(kind: tokGreaterEq)
|
||||
next()
|
||||
else:
|
||||
result = Token(kind: tokGreater)
|
||||
of '=':
|
||||
next()
|
||||
if current() == '=':
|
||||
result = Token(kind: tokEq)
|
||||
next()
|
||||
else:
|
||||
result = Token(kind: tokAssign)
|
||||
of '!':
|
||||
next()
|
||||
if current() == '=':
|
||||
result = Token(kind: tokNotEq)
|
||||
next()
|
||||
else:
|
||||
result = Token(kind: tokNot)
|
||||
of '&':
|
||||
next()
|
||||
if current() == '&':
|
||||
result = Token(kind: tokAnd)
|
||||
next()
|
||||
else:
|
||||
lexer.error("'&&' expected")
|
||||
of '|':
|
||||
next()
|
||||
if current() == '|':
|
||||
result = Token(kind: tokOr)
|
||||
next()
|
||||
else:
|
||||
lexer.error("'||' expected")
|
||||
of '(': result = Token(kind: tokLPar); next()
|
||||
of ')': result = Token(kind: tokRPar); next()
|
||||
of '{': result = Token(kind: tokLBrace); next()
|
||||
of '}': result = Token(kind: tokRBrace); next()
|
||||
of ';': result = Token(kind: tokSemi); next()
|
||||
of ',': result = Token(kind: tokComma); next()
|
||||
of '\'':
|
||||
next()
|
||||
if current() == '\'': lexer.error("Empty character literal")
|
||||
let ch = lexer.charOrEscape()
|
||||
if current() != '\'':
|
||||
lexer.error("Character literal must contain a single character or " &
|
||||
"escape sequence")
|
||||
result = Token(kind: tokChar, charVal: ch)
|
||||
next()
|
||||
of '0'..'9':
|
||||
var number = ""
|
||||
while current() in Digits:
|
||||
number.add(current())
|
||||
next()
|
||||
if current() in IdentStartChars:
|
||||
lexer.error("Integer literal ends in non-digit characters")
|
||||
result = Token(kind: tokInt, intVal: parseInt(number))
|
||||
of '"':
|
||||
next()
|
||||
var str = ""
|
||||
while current() notin {'"', '\x00', '\n'}:
|
||||
str.add(lexer.charOrEscape())
|
||||
if current() == '\x00':
|
||||
lexer.error("Unterminated string literal")
|
||||
elif current() == '\n':
|
||||
lexer.error("Line feed in string literal")
|
||||
else:
|
||||
next()
|
||||
result = Token(kind: tokString, stringVal: str)
|
||||
of IdentStartChars:
|
||||
var ident = $current()
|
||||
next()
|
||||
while current() in IdentChars:
|
||||
ident.add(current())
|
||||
next()
|
||||
case ident
|
||||
of "if": result = Token(kind: tokIf)
|
||||
of "else": result = Token(kind: tokElse)
|
||||
of "while": result = Token(kind: tokWhile)
|
||||
of "print": result = Token(kind: tokPrint)
|
||||
of "putc": result = Token(kind: tokPutc)
|
||||
else: result = Token(kind: tokIdent, ident: ident)
|
||||
of '\x00':
|
||||
result = Token(kind: tokEnd)
|
||||
else:
|
||||
lexer.error("Unexpected character: '" & current() & "'")
|
||||
|
||||
result.ln = ln
|
||||
result.col = col
|
||||
lexer.skip()
|
||||
|
||||
proc peek*(lexer: var Lexer): Token =
|
||||
discard
|
||||
|
||||
proc initLexer*(input: string): Lexer =
|
||||
result = Lexer(input: input, pos: 0, ln: 1, col: 1)
|
||||
result.skip()
|
||||
|
||||
when isMainModule:
|
||||
let code = readAll(stdin)
|
||||
var
|
||||
lexer = initLexer(code)
|
||||
token: Token
|
||||
while true:
|
||||
token = lexer.next()
|
||||
stdout.write(token.ln, ' ', token.col, ' ', token.kind)
|
||||
case token.kind
|
||||
of tokInt: stdout.write(' ', token.intVal)
|
||||
of tokChar: stdout.write(' ', token.charVal.ord)
|
||||
of tokString: stdout.write(" \"", token.stringVal
|
||||
.replace("\\", "\\\\")
|
||||
.replace("\n", "\\n"), '"')
|
||||
of tokIdent: stdout.write(' ', token.ident)
|
||||
else: discard
|
||||
stdout.write('\n')
|
||||
if token.kind == tokEnd:
|
||||
break
|
||||
Loading…
Add table
Add a link
Reference in a new issue