diff --git a/lexer/lexer.go b/lexer/lexer.go new file mode 100644 index 0000000..bdbc429 --- /dev/null +++ b/lexer/lexer.go @@ -0,0 +1,21 @@ +package lexer + +import ( + "github.com/sashakoshka/arf/file" +) + +// LexingOperation holds information about an ongoing lexing operataion. +type LexingOperation struct { + file *file.File +} + +// Tokenize converts a file into a slice of tokens (lexemes) +func Tokenize (file *file.File) (tokens []Token) { + lexer := LexingOperation { } + return lexer.tokenize(file) +} + +// tokenize converts a file into a slice of tokens (lexemes) +func (lexer *LexingOperation) tokenize (file *file.File) (tokens []Token) { + return +}