From 9e4684dbedd0efa5ae9387e0b3b5ba5b4d449037 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Thu, 11 Aug 2022 01:57:04 -0500 Subject: [PATCH] Number tokenizing functions use one digit conversion functions --- lexer/lexer.go | 108 ---------------------------------------- lexer/numbers.go | 127 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 108 deletions(-) create mode 100644 lexer/numbers.go diff --git a/lexer/lexer.go b/lexer/lexer.go index 8d909ab..57cbbb7 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -58,114 +58,6 @@ func (lexer *LexingOperation) tokenize () (err error) { return } -// tokenizeSymbolBeginning lexes a token that starts with a number. -func (lexer *LexingOperation) tokenizeNumberBeginning (negative bool) (err error) { - var number uint64 - - if lexer.char == '0' { - lexer.nextRune() - - if lexer.char == 'x' { - lexer.nextRune() - number, err = lexer.tokenizeHexidecimalNumber() - } else if lexer.char == 'b' { - lexer.nextRune() - number, err = lexer.tokenizeBinaryNumber() - } else if lexer.char == '.' { - number, err = lexer.tokenizeDecimalNumber() - } else if lexer.char >= '0' && lexer.char <= '9' { - number, err = lexer.tokenizeOctalNumber() - } else { - return file.NewError ( - lexer.file.Location(), 1, - "unexpected character in number literal", - file.ErrorKindError) - } - } else { - number, err = lexer.tokenizeDecimalNumber() - } - - if err != nil { return } - - token := Token { } - - if negative { - token.kind = TokenKindInt - token.value = int64(number) * -1 - } else { - token.kind = TokenKindUInt - token.value = uint64(number) - } - - lexer.addToken(token) - return -} - -// tokenizeHexidecimalNumber Reads and tokenizes a hexidecimal number. -func (lexer *LexingOperation) tokenizeHexidecimalNumber () (number uint64, err error) { - for { - if lexer.char >= '0' && lexer.char <= '9' { - number *= 16 - number += uint64(lexer.char - '0') - } else if lexer.char >= 'A' && lexer.char <= 'F' { - number *= 16 - number += uint64(lexer.char - 'A' + 9) - } else if lexer.char >= 'a' && lexer.char <= 'f' { - number *= 16 - number += uint64(lexer.char - 'a' + 9) - } else { - break - } - - err = lexer.nextRune() - if err != nil { return } - } - return -} - -// tokenizeBinaryNumber Reads and tokenizes a binary number. -func (lexer *LexingOperation) tokenizeBinaryNumber () (number uint64, err error) { - for { - if lexer.char == '0' { - number *= 2 - } else if lexer.char == '1' { - number *= 2 - number += 1 - } else { - break - } - - err = lexer.nextRune() - if err != nil { return } - } - return -} - -// tokenizeDecimalNumber Reads and tokenizes a decimal number. -func (lexer *LexingOperation) tokenizeDecimalNumber () (number uint64, err error) { - for lexer.char >= '0' && lexer.char <= '9' { - number *= 10 - number += uint64(lexer.char - '0') - - err = lexer.nextRune() - if err != nil { return } - } - - return -} - -// tokenizeOctalNumber Reads and tokenizes an octal number. -func (lexer *LexingOperation) tokenizeOctalNumber () (number uint64, err error) { - for lexer.char >= '0' && lexer.char <= '7' { - number *= 8 - number += uint64(lexer.char - '0') - - err = lexer.nextRune() - if err != nil { return } - } - return -} - func (lexer *LexingOperation) tokenizeAlphaBeginning () (err error) { got := "" diff --git a/lexer/numbers.go b/lexer/numbers.go new file mode 100644 index 0000000..df1634d --- /dev/null +++ b/lexer/numbers.go @@ -0,0 +1,127 @@ +package lexer + +import "github.com/sashakoshka/arf/file" + +// tokenizeSymbolBeginning lexes a token that starts with a number. +func (lexer *LexingOperation) tokenizeNumberBeginning (negative bool) (err error) { + var number uint64 + + if lexer.char == '0' { + lexer.nextRune() + + if lexer.char == 'x' { + lexer.nextRune() + number, err = lexer.tokenizeHexidecimalNumber() + } else if lexer.char == 'b' { + lexer.nextRune() + number, err = lexer.tokenizeBinaryNumber() + } else if lexer.char == '.' { + number, err = lexer.tokenizeDecimalNumber() + } else if lexer.char >= '0' && lexer.char <= '9' { + number, err = lexer.tokenizeOctalNumber() + } else { + return file.NewError ( + lexer.file.Location(), 1, + "unexpected character in number literal", + file.ErrorKindError) + } + } else { + number, err = lexer.tokenizeDecimalNumber() + } + + if err != nil { return } + + token := Token { } + + if negative { + token.kind = TokenKindInt + token.value = int64(number) * -1 + } else { + token.kind = TokenKindUInt + token.value = uint64(number) + } + + lexer.addToken(token) + return +} + +func runeToDigit (char rune, radix uint64) (digit uint64, worked bool) { + worked = true + + if char >= '0' && char <= '9' { + digit = uint64(char - '0') + } else if char >= 'A' && char <= 'F' { + digit = uint64(char - 'A' + 9) + } else if char >= 'a' && char <= 'f' { + digit = uint64(char - 'a' + 9) + } else { + worked = false + } + + if digit >= radix { + worked = false + } + + return +} + +// tokenizeHexidecimalNumber Reads and tokenizes a hexidecimal number. +func (lexer *LexingOperation) tokenizeHexidecimalNumber () (number uint64, err error) { + for { + digit, worked := runeToDigit(lexer.char, 16) + if !worked { break } + + number *= 16 + number += digit + + err = lexer.nextRune() + if err != nil { return } + } + return +} + +// tokenizeBinaryNumber Reads and tokenizes a binary number. +func (lexer *LexingOperation) tokenizeBinaryNumber () (number uint64, err error) { + for { + digit, worked := runeToDigit(lexer.char, 2) + if !worked { break } + + number *= 2 + number += digit + + err = lexer.nextRune() + if err != nil { return } + } + return +} + +// tokenizeDecimalNumber Reads and tokenizes a decimal number. +func (lexer *LexingOperation) tokenizeDecimalNumber () (number uint64, err error) { + for { + digit, worked := runeToDigit(lexer.char, 10) + if !worked { break } + + number *= 10 + number += digit + + err = lexer.nextRune() + if err != nil { return } + } + + return +} + +// tokenizeOctalNumber Reads and tokenizes an octal number. +func (lexer *LexingOperation) tokenizeOctalNumber () (number uint64, err error) { + for { + digit, worked := runeToDigit(lexer.char, 8) + if !worked { break } + + number *= 8 + number += digit + + err = lexer.nextRune() + if err != nil { return } + } + return +}