lexer/lexer.go

@@ -78,7 +78,13 @@ type fsplLexer struct {
 	eof bool
 }
 
-func (this *fsplLexer) Next () (token lexer.Token, err error) {
+func (this *fsplLexer) Next () (lexer.Token, error) {
+	token, err := this.nextInternal()
+	if err == io.EOF { err = this.errUnexpectedEOF() }
+	return token, err
+}
+
+func (this *fsplLexer) nextInternal () (token lexer.Token, err error) {
 	err = this.skipWhitespace()
 
 	token.Pos = this.pos()
@@ -119,10 +125,6 @@ func (this *fsplLexer) Next () (token lexer.Token, err error) {
 		}
 	}
 
-	errUnexpectedEOF := func () {
-		err = participle.Errorf(this.pos(), "unexpected EOF")
-	}
-
 	switch {
 	// Ident
 	case unicode.IsLower(this.rune):
@@ -145,27 +147,20 @@ func (this *fsplLexer) Next () (token lexer.Token, err error) {
 	case this.rune == '\'':
 		token.Type = String
 		err = this.nextRune()
-		if err == io.EOF {
-			errUnexpectedEOF()
-			return
-		}
 		if err != nil { return }
 
+		println("NEW STRING")
 		for this.rune != '\'' {
+			println(string(this.rune))
 			if this.rune == '\\' {
-				err = this.nextRune()
-				if err == io.EOF {
-					errUnexpectedEOF()
-					return
-				}
+				var result rune
+				result, err = this.escapeSequence()
+				if err != nil { return }
+				token.Value += string(result)
+			} else {
+				appendRune()
 				if err != nil { return }
 			}
-			appendRune()
-			if err == io.EOF {
-				errUnexpectedEOF()
-				return
-			}
-			if err != nil { return }
 		}
 		err = this.nextRune()
 		if err != nil { return }
@@ -243,6 +238,39 @@ func (this *fsplLexer) nextRune () error {
 	return err
 }
 
+func (this *fsplLexer) escapeSequence () (rune, error) {
+	err := this.nextRune()
+	if err != nil { return 0, err }
+
+	if isDigit(this.rune) {
+		var number rune
+		for index := 0; index < 3; index ++ {
+			if !isDigit(this.rune) { break }
+		
+			number *= 8
+			number += this.rune - '0'
+
+			err = this.nextRune()
+			if err != nil { return 0, err }
+		}
+		return number, nil
+	}
+
+	defer this.nextRune()
+	switch this.rune {
+	case '\\', '\n', '\'':
+		return this.rune, nil
+	case 'a': return '\a', nil
+	case 'b': return '\b', nil
+	case 't': return '\t', nil
+	case 'n': return '\n', nil
+	case 'v': return '\v', nil
+	case 'f': return '\f', nil
+	case 'r': return '\r', nil
+	default: return 0, this.errBadEscapeSequence()
+	}
+}
+
 func (this *fsplLexer) skipWhitespace () error {
 	for isWhitespace(this.rune) {
 		err := this.nextRune()
@@ -260,6 +288,14 @@ func (this *fsplLexer) pos () lexer.Position {
 	}
 }
 
+func (this *fsplLexer) errUnexpectedEOF () error {
+	return participle.Errorf(this.pos(), "unexpected EOF")
+}
+
+func (this *fsplLexer) errBadEscapeSequence () error {
+	return participle.Errorf(this.pos(), "bad escape sequence")
+}
+
 func isWhitespace (char rune) bool {
 	switch char {
 	case ' ', '\t', '\r', '\n': return true

lexer/lexer_test.go

@@ -35,7 +35,7 @@ testString(test,
 normalIdent TypeIdent
 [$$abcdEfGhIjKlMnOpQrStUvWxYz]{+}(-)
 ~ ! @ # $ % ^ & *- _ = + \ | ; :** ::, < . > / ?
-!! ++ -- && || 'some \'string'
+!! ++ -- && || 'some \'string' 'nullterm\0' '\110\117\115\105'
 1234 -1234 9876540321 (754.2340)
 `,
 tok(Ident, "normalIdent"),
@@ -85,6 +85,8 @@ tok(Symbol, "&&"),
 tok(Symbol, "||"),
 
 tok(String, "some 'string"),
+tok(String, "nullterm\000"),
+tok(String, "HOME"),
 tok(Int, "1234"),
 tok(Int, "-1234"),
 tok(Int, "9876540321"),