package lexer import "github.com/sashakoshka/arf/file" // LexingOperation holds information about an ongoing lexing operataion. type LexingOperation struct { file *file.File } // Tokenize converts a file into a slice of tokens (lexemes) func Tokenize (file *file.File) (tokens []Token) { lexer := LexingOperation { } return lexer.tokenize(file) } // tokenize converts a file into a slice of tokens (lexemes) func (lexer *LexingOperation) tokenize (file *file.File) (tokens []Token) { return }