diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index 1fd7b342..f8f38041 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -8,8 +8,6 @@ import ( "unicode/utf8" ) -type Builder = strings.Builder - type Lexer struct { Pos Position Reader *bufio.Reader @@ -83,7 +81,7 @@ func (l *Lexer) Tokenize() *Token { case unicode.IsSpace(r): continue case IsDigit(r): - return l.ReadNumber(pos) + return l.ReadNumber(pos, r) case unicode.IsLetter(r), r == '_': return l.ReadIdentifier(pos, r) case r == 0xfeff: @@ -189,7 +187,7 @@ func (l *Lexer) NewTokenizer(backupLast bool) *Tokenizer { return &Tokenizer{Builder: strings.Builder{}, BackupLast: backupLast, Lexer: l} } -func (t *Tokenizer) Tokenize(yield func(rune, *Builder) bool) { +func (t *Tokenizer) Tokenize(yield func(rune, *strings.Builder) bool) { for { r, _, err := t.Reader.ReadRune() t.Pos.Col++ @@ -230,3 +228,47 @@ func (t *Tokenizer) ResetKeepBuilder(backupLast bool) { t.BackupLast = backupLast t.eof = false } + +// RuneReader is an interface for reading runes from a stream. +type RuneReader interface { + AdvanceRune() (rune, error) + CurrRune() (rune, error) + PeekRune() (rune, error) + Position() Position +} + +func (l *Lexer) AdvanceRune() (rune, error) { + r, _, err := l.Reader.ReadRune() + if err != nil { + return 0, err + } + l.Pos.Col++ + if r == '\n' { + l.ResetPosition() + } + return r, nil +} + +func (l *Lexer) CurrRune() (rune, error) { + b, err := l.Reader.Peek(4) + if err != nil && len(b) == 0 { + return 0, err + } + r, _ := utf8.DecodeRune(b) + return r, nil +} + +func (l *Lexer) PeekRune() (rune, error) { + b, err := l.Reader.Peek(8) + if err != nil && len(b) == 0 { + return 0, err + } + _, size1 := utf8.DecodeRune(b) + if len(b) <= size1 { + return 0, io.EOF + } + r2, _ := utf8.DecodeRune(b[size1:]) + return r2, nil +} + +func (l *Lexer) Position() Position { return l.Pos } diff --git a/internal/lexer/number.go b/internal/lexer/number.go index 8f66d5eb..b59102b0 100644 --- a/internal/lexer/number.go +++ b/internal/lexer/number.go @@ -1,5 +1,9 @@ package lexer +import ( + "strings" +) + type NumberAttrs struct { Format IntFormat Flags NumberFlags @@ -37,120 +41,116 @@ const ( ErrInvalidDecimalPoint // Decimal point can only be used in decimal (base 10) format ) -func (l *Lexer) ReadNumber(pos Position) *Token { +func (l *Lexer) ReadNumber(pos Position, first rune) *Token { + num, params := ReadNumber(l, first) + return NewToken(pos, Numeric, num).withAttrs(attrs{"params": params}) +} + +func ReadNumber(rd RuneReader, first rune) (string, NumberAttrs) { var ( + b strings.Builder format IntFormat flags NumberFlags - errorType NumberErrorCode - errPos int + err *NumberError isExp, isDec bool last rune ) - newError := func(code NumberErrorCode, b *Builder) { - errorType = code - if b != nil { - errPos = b.Len() + newError := func(code NumberErrorCode, errPos int) { + if err == nil { + err = &NumberError{Code: code, Offset: uint32(errPos)} } } - l.Backup() - t := l.NewTokenizer(true) -readNumber: - for r, b := range t.Tokenize { - // 0 prefix - if b.String() == "0" { + + b.WriteRune(first) + last = first + + if first == '0' { + if r, er := rd.CurrRune(); er == nil { switch r { - case 'x': + case 'x', 'X': format = NumberFormatHex - goto writeAndContinue - case 'b': + b.WriteRune(r) + rd.AdvanceRune() + last = r + case 'b', 'B': format = NumberFormatBinary - goto writeAndContinue + b.WriteRune(r) + rd.AdvanceRune() + last = r default: format = NumberFormatDecimal } } + } + +readNumber: + for { + r, er := rd.CurrRune() + if er != nil { + break + } switch r { + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + if format == NumberFormatBinary && r > '1' { + newError(ErrIntIncompatibleDigit, b.Len()) + } case 'e', 'E': // Exponent or hex digit if format == NumberFormatDecimal { if isExp { - newError(ErrIntIncompatibleDigit, b) + newError(ErrIntIncompatibleDigit, b.Len()) break } if last == '_' { - newError(ErrIntMisplacedSeparator, b) - errPos-- + newError(ErrIntMisplacedSeparator, b.Len()-1) } isExp = true flags |= HasExponent | IsFloat break } fallthrough // Hex or invalid digit - case 'a', 'A', 'b', 'B', 'c', 'C', 'd', 'D', 'f', 'F': + case 'a', 'b', 'c', 'd', 'f', 'A', 'B', 'C', 'D', 'F': if format != NumberFormatHex { // Hex letter or e on other format - newError(ErrIntIncompatibleDigit, b) - } - case '+', '-': // After 'e' - if last != 'e' && last != 'E' { - if !IsDigit(last) { - // 12e+- - newError(ErrIntIncompatibleDigit, b) - } - break readNumber + newError(ErrIntIncompatibleDigit, b.Len()) } case '.': - switch { - case isDec: + if isDec || isExp || format != NumberFormatDecimal { break readNumber - case format != NumberFormatDecimal: - newError(ErrIntIncompatibleDigit, b) - case last == '_': - newError(ErrIntMisplacedSeparator, b) - errPos-- } - if n, isEOF := l.BackupPeek(); isEOF || !IsDigit(rune(n)) { + if last == '_' { + newError(ErrIntMisplacedSeparator, b.Len()-1) + } + // Check if next character is a digit + next, err2 := rd.PeekRune() + if err2 != nil || !IsDigit(next) { break readNumber } isDec = true flags |= IsFloat case '_': - // Underscore separators: no consecutive, must be in between digits if last == '_' || (format == NumberFormatDecimal && !IsDigit(last)) { - newError(ErrIntMisplacedSeparator, b) + newError(ErrIntMisplacedSeparator, b.Len()) } flags |= HasSeparator - default: - switch { - case !IsDigit(r): + case '+', '-': + if (last != 'e' && last != 'E') || format != NumberFormatDecimal { break readNumber - case format == NumberFormatDecimal, - format == NumberFormatHex, - format == NumberFormatBinary && r <= '1': - default: - newError(ErrIntIncompatibleDigit, b) } + default: + break readNumber } - writeAndContinue: + b.WriteRune(r) + rd.AdvanceRune() last = r } - num := t.String() - // Last character validation + + num := b.String() if last == '_' { - // Last digit can't be a separator - newError(ErrIntMisplacedSeparator, nil) - errPos = len(num) - 1 + newError(ErrIntMisplacedSeparator, len(num)-1) } else if format != NumberFormatHex && !IsDigit(last) { - // "1e-" .. EOF - newError(ErrIntIncompatibleDigit, nil) - errPos = len(num) - } - var err *NumberError - if errorType != 0 { - err = &NumberError{Code: errorType, Offset: uint32(errPos)} + newError(ErrIntIncompatibleDigit, len(num)) } - return NewToken(pos, Numeric, num).withAttrs(attrs{ - "params": NumberAttrs{Format: format, Flags: flags, Error: err}, - }) + return num, NumberAttrs{Format: format, Flags: flags, Error: err} } diff --git a/internal/lexer/string.go b/internal/lexer/string.go index ba445705..d24f7ae9 100644 --- a/internal/lexer/string.go +++ b/internal/lexer/string.go @@ -1,6 +1,7 @@ package lexer import ( + "strings" "unicode" "unicode/utf8" ) @@ -147,7 +148,7 @@ func (l *Lexer) readStrInterp() StringEscape { err *EscapeError tokens []Token braceCt = 1 - b Builder + b strings.Builder ) b.WriteRune('{') loop: diff --git a/pkg/klon/lexer.go b/pkg/klon/lexer.go index 92e22d91..6b7d5f1a 100644 --- a/pkg/klon/lexer.go +++ b/pkg/klon/lexer.go @@ -49,6 +49,41 @@ func (rd *reader) resetLineIf(r rune) { } } +func (rd *reader) AdvanceRune() (rune, error) { return rd.readRune() } +func (rd *reader) CurrRune() (rune, error) { + if rd.pos >= len(rd.buffer) { + if err := rd.refill(); err != nil { + return 0, err + } + } + r, _ := utf8.DecodeRune(rd.buffer[rd.pos:]) + return r, nil +} +func (rd *reader) PeekRune() (rune, error) { + if rd.pos >= len(rd.buffer) { + if err := rd.refill(); err != nil { + return 0, err + } + } + _, n := utf8.DecodeRune(rd.buffer[rd.pos:]) + nextPos := rd.pos + n + if nextPos >= len(rd.buffer) { + if rd.reader != nil { + if err := rd.refill(); err != nil { + return 0, err + } + _, n = utf8.DecodeRune(rd.buffer[rd.pos:]) + nextPos = rd.pos + n + } + } + if nextPos >= len(rd.buffer) { + return 0, io.EOF + } + r2, _ := utf8.DecodeRune(rd.buffer[nextPos:]) + return r2, nil +} +func (rd *reader) Position() lexer.Position { return rd.offset } + func (rd *reader) peekRune() (rune, int, error) { if rd.needsMore() { if err := rd.tryRefill(); err != nil { @@ -111,10 +146,6 @@ func (rd *reader) readToken() Token { return rd.readNumber(r, start, bufPos) } case '.': - // TODO: Don't allow leading/trailing decimal point for numbers - if curr, _, _ := rd.currRune(); curr >= '0' && curr <= '9' { - return rd.readNumber(r, start, bufPos) - } if (rd.parseFlags & allowDot) == 0 { return Token{Kind: Dot, Pos: start, Src: ".", BufPos: bufPos} } @@ -204,41 +235,45 @@ func (rd *reader) readQuotedString(quote rune, start lexer.Position, bufPos int, } func (rd *reader) readNumber(first rune, start lexer.Position, bufPos int) Token { - var b strings.Builder - isNumber := true - var isDecimal, wasUnderscore bool - value := func() Token { - tok := Token{Kind: Number, Src: b.String(), Pos: start, BufPos: bufPos} - if !isNumber || tok.Src[0] < '0' || tok.Src[0] > '9' { - tok.Kind = String + var ( + prefix string + isNumber = true + ) + if first == '-' || first == '+' { + prefix = string(first) + r, n, err := rd.currRune() + if err != nil || !lexer.IsDigit(r) { + b := &strings.Builder{} + b.WriteRune(first) + return rd.readUnquotedString(b, start, bufPos) } - return tok + rd.advanceBytes(n) + first = r } - // Check first digit or +, -, . - b.WriteRune(first) - for { - r, size, err := rd.currRune() - if err != nil { - return value() - } - switch { - case r == '_' && wasUnderscore, r == '.' && isDecimal: - isNumber = false - case r == '_': - wasUnderscore = true - case r == '.': - isDecimal = true - case unicode.IsSpace(r), rd.isPunct(r): - return value() - case r < '0' || r > '9': - isNumber = false - } - b.WriteRune(r) - rd.advanceBytes(size) - if !isNumber { - return rd.readUnquotedString(&b, start, bufPos) + + literal, params := lexer.ReadNumber(rd, first) + + // Klon transitions to unquoted string for certain numeric patterns + if literal[0] == '.' || literal[len(literal)-1] == '_' { + isNumber = false + } + + r, _, err := rd.currRune() + isDelim := err != nil || unicode.IsSpace(r) || rd.isPunct(r) || r == ',' + + if isNumber && isDelim { + return Token{ + Kind: Number, + Src: prefix + literal, + Pos: start, + BufPos: bufPos, + Attrs: attrs{"params": params}, } } + + b := &strings.Builder{} + b.WriteString(prefix + literal) + return rd.readUnquotedString(b, start, bufPos) } func (rd *reader) isPunct(r rune) bool {