Merge pull request 'dont-create-lexer' (#27) from dont-create-lexer into main

Reviewed-on: sashakoshka/fspl#27

analyzer/test-common.go

@@ -4,6 +4,7 @@ import "io"
 import "fmt"
 import "testing"
 import "strings"
+import "git.tebibyte.media/sashakoshka/fspl/lexer"
 import "git.tebibyte.media/sashakoshka/fspl/errors"
 import "git.tebibyte.media/sashakoshka/fspl/parser"
 
@@ -29,7 +30,14 @@ func testReaderErr (
 ) {
 	ast := parser.Tree { }
 	for index, stream := range inputs {
-		err := ast.Parse(fmt.Sprintf("stream%d.fspl", index), stream)
+		lx, err := lexer.LexReader (
+			fmt.Sprintf("stream%d.fspl", index),
+			stream)
+		if err != nil {
+			test.Error("lexer returned error:", err)
+			return
+		}
+		err = ast.Parse(lx)
 		if err != nil {
 			test.Error("parser returned error:", err)
 			return
@@ -66,7 +74,14 @@ func testString (test *testing.T, input string) {
 func testReader (test *testing.T, inputs ...io.Reader) {
 	ast := parser.Tree { }
 	for index, stream := range inputs {
-		err := ast.Parse(fmt.Sprintf("stream%d.fspl", index), stream)
+		lx, err := lexer.LexReader (
+			fmt.Sprintf("stream%d.fspl", index),
+			stream)
+		if err != nil {
+			test.Error("lexer returned error:", err)
+			return
+		}
+		err = ast.Parse(lx)
 		if err != nil {
 			test.Error("parser returned error:", err)
 			return

cmd/fsplc/main.go

@@ -8,6 +8,7 @@ import "strings"
 import "os/exec"
 import "path/filepath"
 import "git.tebibyte.media/sashakoshka/fspl/llvm"
+import "git.tebibyte.media/sashakoshka/fspl/lexer"
 import "git.tebibyte.media/sashakoshka/fspl/parser"
 import "git.tebibyte.media/sashakoshka/fspl/analyzer"
 import ferrors "git.tebibyte.media/sashakoshka/fspl/errors"
@@ -44,7 +45,9 @@ func (this *Compiler) Compile (inputs []string) error {
 
 	var syntaxTree parser.Tree
 	for _, name := range inputs {
-		err := syntaxTree.ParseFile(name)
+		lx, err := lexer.LexFile(name)
+		if err != nil { return err }
+		err = syntaxTree.Parse(lx)
 		if err != nil { return err }
 	}
 

generator/test-common.go

@@ -3,13 +3,19 @@ package generator
 import "io"
 import "testing"
 import "strings"
+import "git.tebibyte.media/sashakoshka/fspl/lexer"
 import "git.tebibyte.media/sashakoshka/fspl/parser"
 import "git.tebibyte.media/sashakoshka/fspl/analyzer"
 import "git.tebibyte.media/sashakoshka/fspl/testcommon"
 
 func testString (test *testing.T, correct string, input string) {
 	ast := parser.Tree { }
-	err := ast.Parse("input.fspl", strings.NewReader(input))
+	lx, err := lexer.LexReader("input.fspl", strings.NewReader(input))
+	if err != nil {
+		test.Error("lexer returned error:", err)
+		return
+	}
+	err = ast.Parse(lx)
 	if err != nil && err != io.EOF{
 		test.Error("parser returned error:", err)
 		return

lexer/lexer.go

@@ -1,5 +1,6 @@
 package lexer
 
+import "os"
 import "io"
 import "fmt"
 import "bufio"
@@ -105,10 +106,9 @@ type Lexer interface {
 	Next () (Token, error)
 }
 
-// NewLexer creates a new default lexer that reads from the given reader. The
+// LexReader creates a new default lexer that reads from the given reader. The
 // filename parameter is used for token locations and error messages.
-func NewLexer (filename string, reader io.Reader) (Lexer, error) {
-	// TODO: replace this function with LexReader and LexFile functions
+func LexReader (filename string, reader io.Reader) (Lexer, error) {
 	lexer := &fsplLexer {
 		filename:    filename,
 		lineScanner: bufio.NewScanner(reader),
@@ -117,6 +117,19 @@ func NewLexer (filename string, reader io.Reader) (Lexer, error) {
 	return lexer, nil
 }
 
+// LexFile creates a new default lexer that reads from the given file.
+func LexFile (filename string) (Lexer, error) {
+	file, err := os.Open(filename)
+	if err != nil { return nil, err }
+	
+	lexer := &fsplLexer {
+		filename:    filename,
+		lineScanner: bufio.NewScanner(file),
+	}
+	lexer.nextRune()
+	return lexer, nil
+}
+
 type fsplLexer struct {
 	filename    string
 	lineScanner *bufio.Scanner

lexer/test-common.go

@@ -63,7 +63,7 @@ func testStringErr (
 	}
 
 	reader := strings.NewReader(input)
-	lx, err := NewLexer("input.fspl", reader)
+	lx, err := LexReader("input.fspl", reader)
 	if err != nil {
 		if errMessage == "" {
 			printUnexpectedErr(err)

parser/parser.go

@@ -43,12 +43,8 @@ type Parser struct {
 	tree  *Tree
 }
 
-// NewParser creates a new parser that parses the given file.
-func NewParser (name string, file io.Reader) (*Parser, error) {
-	// TODO: have the parser take in a lexer instead of creating one.
-	lx, err := lexer.NewLexer(name, file)
-	if err != nil { return nil, err }
-	
+// NewParser creates a new parser that parses the output of the given lexer.
+func NewParser (lx lexer.Lexer) (*Parser, error) {
 	return &Parser {
 		lexer: lx,
 	}, nil

parser/test-common.go

@@ -3,12 +3,18 @@ package parser
 import "io"
 import "testing"
 import "strings"
+import "git.tebibyte.media/sashakoshka/fspl/lexer"
 import "git.tebibyte.media/sashakoshka/fspl/errors"
 import "git.tebibyte.media/sashakoshka/fspl/testcommon"
 
 func testString (test *testing.T, correct string, input string) {
 	ast := Tree { }
-	err := ast.Parse("input.fspl", strings.NewReader(input))
+	lx, err := lexer.LexReader("input.fspl", strings.NewReader(input))
+	if err != nil && err != io.EOF{
+		test.Error("lexer returned error:\n" + errors.Format(err))
+		return
+	}
+	err = ast.Parse(lx)
 	if err != nil && err != io.EOF{
 		test.Error("parser returned error:\n" + errors.Format(err))
 		return

parser/tree.go

@@ -1,8 +1,7 @@
 package parser
 
-import "io"
-import "os"
 import "fmt"
+import "git.tebibyte.media/sashakoshka/fspl/lexer"
 import "git.tebibyte.media/sashakoshka/fspl/entity"
 
 // Tree represents a parsed abstract syntax tree. It has no constructor and its
@@ -22,17 +21,10 @@ func (this *Tree) String () string {
 	return out
 }
 
-// ParseFile parses the contents of the given file into the tree.
-func (this *Tree) ParseFile (name string) error {
-	file, err := os.Open(name)
-	if err != nil { return err }
-	defer file.Close()
-	return this.Parse(name, file)
-}
 
-// Parse parses the contents of the given io.Reader into the tree.
-func (this *Tree) Parse (name string, file io.Reader) error {
-	parser, err := NewParser(name, file)
+// Parse parses the output of the given lexer into the tree.
+func (this *Tree) Parse (lx lexer.Lexer) error {
+	parser, err := NewParser(lx)
 	if err != nil { return err }
 	return parser.ParseInto(this)
 }