21 lines
512 B
Go
21 lines
512 B
Go
package lexer
|
|
|
|
import "github.com/sashakoshka/arf/file"
|
|
|
|
// LexingOperation holds information about an ongoing lexing operataion.
|
|
type LexingOperation struct {
|
|
file *file.File
|
|
char rune
|
|
}
|
|
|
|
// Tokenize converts a file into a slice of tokens (lexemes)
|
|
func Tokenize (file *file.File) (tokens []Token) {
|
|
lexer := LexingOperation { }
|
|
return lexer.tokenize(file)
|
|
}
|
|
|
|
// tokenize converts a file into a slice of tokens (lexemes)
|
|
func (lexer *LexingOperation) tokenize (file *file.File) (tokens []Token) {
|
|
return
|
|
}
|