This repository has been archived on 2022-08-30. You can view files and clone it, but cannot push or open issues or pull requests.
2022-08-07 19:18:59 +00:00
|
|
|
package lexer
|
|
|
|
|
2022-08-08 07:08:50 +00:00
|
|
|
import "github.com/sashakoshka/arf/file"
|
2022-08-07 19:18:59 +00:00
|
|
|
|
|
|
|
// LexingOperation holds information about an ongoing lexing operataion.
|
|
|
|
type LexingOperation struct {
|
|
|
|
file *file.File
|
2022-08-09 05:51:56 +00:00
|
|
|
char rune
|
2022-08-07 19:18:59 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Tokenize converts a file into a slice of tokens (lexemes)
|
|
|
|
func Tokenize (file *file.File) (tokens []Token) {
|
|
|
|
lexer := LexingOperation { }
|
|
|
|
return lexer.tokenize(file)
|
|
|
|
}
|
|
|
|
|
|
|
|
// tokenize converts a file into a slice of tokens (lexemes)
|
|
|
|
func (lexer *LexingOperation) tokenize (file *file.File) (tokens []Token) {
|
|
|
|
return
|
|
|
|
}
|