22 lines
513 B
Go
22 lines
513 B
Go
|
package lexer
|
||
|
|
|
||
|
|
import (
|
||
|
|
"github.com/sashakoshka/arf/file"
|
||
|
|
)
|
||
|
|
|
||
|
|
// LexingOperation holds information about an ongoing lexing operataion.
|
||
|
|
type LexingOperation struct {
|
||
|
|
file *file.File
|
||
|
|
}
|
||
|
|
|
||
|
|
// Tokenize converts a file into a slice of tokens (lexemes)
|
||
|
|
func Tokenize (file *file.File) (tokens []Token) {
|
||
|
|
lexer := LexingOperation { }
|
||
|
|
return lexer.tokenize(file)
|
||
|
|
}
|
||
|
|
|
||
|
|
// tokenize converts a file into a slice of tokens (lexemes)
|
||
|
|
func (lexer *LexingOperation) tokenize (file *file.File) (tokens []Token) {
|
||
|
|
return
|
||
|
|
}
|