From 47234e9f367d6b4d2e163d39361ce7ec2e6a52ca Mon Sep 17 00:00:00 2001
From: Sasha Koshka <supergriffin64@gmail.com>
Date: Sun, 7 Aug 2022 15:18:59 -0400
Subject: [PATCH] Added lexer stub

---
 lexer/lexer.go | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)
 create mode 100644 lexer/lexer.go

diff --git a/lexer/lexer.go b/lexer/lexer.go
new file mode 100644
index 0000000..bdbc429
--- /dev/null
+++ b/lexer/lexer.go
@@ -0,0 +1,21 @@
+package lexer
+
+import (
+        "github.com/sashakoshka/arf/file"
+)
+
+// LexingOperation holds information about an ongoing lexing operataion.
+type LexingOperation struct {
+	file *file.File
+}
+
+// Tokenize converts a file into a slice of tokens (lexemes)
+func Tokenize (file *file.File) (tokens []Token) {
+	lexer := LexingOperation { }
+	return lexer.tokenize(file)
+}
+
+// tokenize converts a file into a slice of tokens (lexemes)
+func (lexer *LexingOperation) tokenize (file *file.File) (tokens []Token) {
+	return
+}