package main import ( "fmt" "regexp" "strconv" "strings" ) type position struct { pos int filepath string line, column int } func (p position) String() string { return fmt.Sprintf("%s:%d:%d", p.filepath, p.line, p.column) } func (p *position) Position() position { return *p } type Range [2]position type token[T comparable] struct { position value T } type Lexeme interface { Position() position Equals(Lexeme) bool } type Id string type Token token[string] type Int token[int] type Identifier token[Id] var ( Lparen = &Token{value: "("} Rparen = &Token{value: ")"} Space = &Token{value: " "} Tab = &Token{value: "\t"} Newline = &Token{value: "\n"} SingleQuote = &Token{value: "'"} Let = &Token{value: "let"} LambdaTok = &Token{value: "lambda"} ) func (t Token) String() string { return fmt.Sprint(t.value) } func (t Int) String() string { return fmt.Sprint(t.value) } func (t Identifier) String() string { return fmt.Sprint(t.value) } var knownTokens = []*Token{ Lparen, Rparen, Space, Tab, Newline, SingleQuote, Let, LambdaTok, } func (t *Token) Equals(a Lexeme) bool { switch x := a.(type) { case *Token: return t.value == x.value default: return false } } func (t *Int) Equals(a Lexeme) bool { return false } func (t *Identifier) Equals(a Lexeme) bool { switch x := a.(type) { case *Identifier: return t.value == x.value default: return false } } var matchNumbers = regexp.MustCompile(`-?\d+`) var matchIdentifier = regexp.MustCompile(`[^\'() ]+`) func lex(source string) (*Input, error) { var ( pos, line, column int tokens []Lexeme ) outer: for len(source) > 0 { p := position{pos: pos, line: line, column: column, filepath: "sourceinput"} for _, try := range knownTokens { if !strings.HasPrefix(source, try.value) { continue } var ( l = len(try.value) t = &Token{position: p, value: try.value} ) source = source[l:] pos += l switch try { case Newline: line += 1 column = 0 case Tab, Space: column += l default: column += l tokens = append(tokens, t) } continue outer } index := matchNumbers.FindStringSubmatchIndex(source) if index != nil && index[0] == 0 { a := source[index[0]:index[1]] conv, err := strconv.Atoi(a) if err == nil { var i = &Int{position: p, value: conv} tokens = append(tokens, i) l := len(a) source = source[l:] column += l pos += l continue outer } } index = matchIdentifier.FindStringIndex(source) if index != nil && index[0] == 0 { a := source[index[0]:index[1]] var id = &Identifier{position: p, value: Id(a)} tokens = append(tokens, id) l := len(a) source = source[l:] column += l pos += l continue outer } return nil, fmt.Errorf("unrecognized token '%c' at %s", source[0], p) } return &Input{input: tokens}, nil }