diff --git a/generate/lex.go b/generate/lex.go index 0d9aaf8..a20c3d9 100644 --- a/generate/lex.go +++ b/generate/lex.go @@ -15,6 +15,7 @@ const ( TokenRBrace TokenLBracket TokenRBracket + TokenComment ) var tokenNames = map[parse.TokenKind] string { @@ -26,6 +27,7 @@ var tokenNames = map[parse.TokenKind] string { TokenRBrace: "RBrace", TokenLBracket: "LBracket", TokenRBracket: "RBracket", + TokenComment: "Comment", } func Lex(fileName string, reader io.Reader) (parse.Lexer, error) { @@ -81,6 +83,18 @@ func (this *lexer) nextInternal() (token parse.Token, err error) { } } + unexpected := func() error { + if unicode.IsPrint(this.rune) { + return parse.Errorf ( + this.pos(), "unexpected rune '%c'", + this.rune) + } else { + return parse.Errorf ( + this.pos(), "unexpected rune %U", + this.rune) + } + } + defer func () { newPos := this.pos() newPos.End -- // TODO figure out why tf we have to do this @@ -133,14 +147,21 @@ func (this *lexer) nextInternal() (token parse.Token, err error) { token.Kind = TokenRBracket appendRune() if this.eof { err = nil; return } - case unicode.IsPrint(this.rune): - err = parse.Errorf ( - this.pos(), "unexpected rune '%c'", - this.rune) + // Comment + case this.rune == '/': + token.Kind = TokenComment + appendRune() + if this.eof { return } + if this.rune != '/' { + err = unexpected() + return + } + for this.rune != '\n' { + appendRune() + if this.eof { err = nil; return } + } default: - err = parse.Errorf ( - this.pos(), "unexpected rune %U", - this.rune) + err = unexpected() } return diff --git a/generate/lex_test.go b/generate/lex_test.go index fc4a967..8715cea 100644 --- a/generate/lex_test.go +++ b/generate/lex_test.go @@ -6,14 +6,21 @@ import "git.tebibyte.media/sashakoshka/goparse" func TestLex(test *testing.T) { lexer, err := Lex("test.pdl", strings.NewReader(` + // User holds profile information about a single user. M0001 User { 0000 Name String, + // dog water comment + + // Users is asdkjsagkj why + // + // wow 0001 Users []User, 0002 Followers U32, }`)) if err != nil { test.Fatal(parse.Format(err)) } correctTokens := []parse.Token { + tok(TokenComment, "// User holds profile information about a single user."), tok(TokenMethod, "0001"), tok(TokenIdent, "User"), tok(TokenLBrace, "{"), @@ -21,6 +28,10 @@ func TestLex(test *testing.T) { tok(TokenIdent, "Name"), tok(TokenIdent, "String"), tok(TokenComma, ","), + tok(TokenComment, "// dog water comment"), + tok(TokenComment, "// Users is asdkjsagkj why"), + tok(TokenComment, "// "), + tok(TokenComment, "// wow"), tok(TokenKey, "0001"), tok(TokenIdent, "Users"), tok(TokenLBracket, "["),