sexp/lex.go

package main

import (
	"fmt"
	"regexp"
	"slices"
	"strconv"
	"strings"
)

type position struct {
	pos          int
	filepath     string
	line, column int
}

func (p position) String() string {
	return fmt.Sprintf("%s:%d:%d", p.filepath, p.line, p.column)
}

func (p *position) Position() position {
	return *p
}

type (
	Range [2]position

	token[T comparable] struct {
		position
		value T
	}

	Lexeme interface {
		Position() position
		Equals(Lexeme) bool
	}

	Id string

	Token token[string]

	IntToken token[int]

	FloatToken token[float64]

	Identifier token[Id]
)

var (
	Lparen         = &Token{value: "("}
	Rparen         = &Token{value: ")"}
	LsquareBracket = &Token{value: "["}
	RsquareBracket = &Token{value: "]"}
	Space          = &Token{value: " "}
	Tab            = &Token{value: "\t"}
	Newline        = &Token{value: "\n"}
	SingleQuote    = &Token{value: "'"}
	Let            = &Token{value: "let"}
	LambdaTok      = &Token{value: "lambda"}
	If             = &Token{value: "if"}
	TrueTok        = &Token{value: "t"}
	NilTok         = &Token{value: "nil"}
)

func (t Token) String() string {
	return fmt.Sprint(t.value)
}

func (t IntToken) String() string {
	return fmt.Sprint(t.value)
}

func (t Identifier) String() string {
	return fmt.Sprint(t.value)
}

var knownTokens = []*Token{
	Lparen,
	Rparen,
	Space,
	Tab,
	Newline,
	SingleQuote,
	Let,
	LambdaTok,
	If,
	TrueTok,
	NilTok,
}

func (t *Token) Equals(a Lexeme) bool {
	switch x := a.(type) {
	case *Token:
		return t.value == x.value
	default:
		return false
	}
}

func (t *IntToken) Equals(a Lexeme) bool   { return false }
func (t *FloatToken) Equals(a Lexeme) bool { return false }

func (t *Identifier) Equals(a Lexeme) bool {
	switch x := a.(type) {
	case *Identifier:
		return t.value == x.value
	default:
		return false
	}
}

var matchInt = regexp.MustCompile(`-?\d+`)
var matchFloat = regexp.MustCompile(`-?\d+\.\d*`)
var matchIdentifier = regexp.MustCompile(`[^\'() \[\]]+`)

func lex(source string) (*Input, error) {
	var (
		pos, line, column int = 1, 1, 1
		tokens            []Lexeme
	)
outer:
	for len(source) > 0 {
		p := position{pos: pos, line: line, column: column, filepath: "stdin"}
		if source[0] == ';' {
			idx := slices.Index([]byte(source), '\n')
			if idx < 0 {
				break
			} else {
				source = source[idx:]
				continue
			}
		}
		for _, try := range knownTokens {
			if !strings.HasPrefix(source, try.value) {
				continue
			}
			var (
				l = len(try.value)
				t = &Token{position: p, value: try.value}
			)
			source = source[l:]
			pos += l
			switch try {
			case Newline:
				line += 1
				column = 0
			case Tab, Space:
				column += l
			default:
				column += l
				tokens = append(tokens, t)
			}
			continue outer
		}
		index := matchFloat.FindStringSubmatchIndex(source)
		if index != nil && index[0] == 0 {
			a := source[index[0]:index[1]]
			conv, err := strconv.ParseFloat(a, 64)
			if err == nil {
				var i = &FloatToken{position: p, value: conv}
				tokens = append(tokens, i)
				l := len(a)
				source = source[l:]
				column += l
				pos += l
				continue outer
			}
		}
		index = matchInt.FindStringSubmatchIndex(source)
		if index != nil && index[0] == 0 {
			a := source[index[0]:index[1]]
			conv, err := strconv.Atoi(a)
			if err == nil {
				var i = &IntToken{position: p, value: conv}
				tokens = append(tokens, i)
				l := len(a)
				source = source[l:]
				column += l
				pos += l
				continue outer
			}
		}
		index = matchIdentifier.FindStringIndex(source)
		if index != nil && index[0] == 0 {
			a := source[index[0]:index[1]]
			var id = &Identifier{position: p, value: Id(a)}
			tokens = append(tokens, id)
			l := len(a)
			source = source[l:]
			column += l
			pos += l
			continue outer
		}
		return nil, fmt.Errorf("unrecognized token '%c' at %s", source[0], p)
	}
	return &Input{input: tokens}, nil
}