Remove markdown, add goparse from go.modo

design/pdl-language.md

@@ -43,7 +43,6 @@ structures. They are separated by whitespace.
 
 | Name     | Syntax             | Description
 | -------- | ------------------ | -----------
-| Magic    | `PDL/0`            | Must appear at the very start of the file.
 | Method   | `M[0-9A-Fa-f]{4}`  | A 16-bit hexadecimal method code.
 | Key      | `[0-9A-Fa-f]{4}`   | A 16-bit hexadecimal table key.
 | Ident    | `[A-Z][A-Za-z0-9]` | An identifier.
@@ -55,8 +54,6 @@ structures. They are separated by whitespace.
 
 ## Syntax
 
-All files must begin with a Magic token.
-
 Types are expressed with an Ident. A table can be used by either writing the
 name of the type (Table), or by defining a schema with curly braces. Arrays must
 be expressed using two matching square brackets before their element type.
@@ -73,8 +70,6 @@ can be anything.
 Here is an example of all that:
 
 ```
-PDL/0
-
 M0000 Connect {
 	0000 Name String,
 	0001 Password String,
@@ -96,8 +91,7 @@ User {
 Below is an EBNF description of the language.
 
 ```
-<file>    -> <magic> (<message> | <typedef)*
+<file>    -> (<message> | <typedef)*
-<magic>   -> "PDL/0"
 <method>  -> /M[0-9A-Fa-f]{4}/
 <key>     -> /[0-9A-Fa-f]{4}/
 <ident>   -> /[A-Z][A-Za-z0-9]/

generate/lex.go

@@ -0,0 +1,230 @@
+package generate
+
+import "io"
+import "bufio"
+import "unicode"
+import "unicode/utf8"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+const (
+	TokenMethod parse.TokenKind = iota
+	TokenKey
+	TokenIdent
+	TokenComma
+	TokenLBrace
+	TokenRBrace
+	TokenLBracket
+	TokenRBracket
+)
+
+var tokenNames = map[parse.TokenKind] string {
+	TokenMethod:   "Method",
+	TokenKey:      "Key",
+	TokenIdent:    "Ident",
+	TokenComma:    "Comma",
+	TokenLBrace:   "LBrace",
+	TokenRBrace:   "RBrace",
+	TokenLBracket: "LBracket",
+	TokenRBracket: "RBracket",
+}
+
+func Lex(fileName string, reader io.Reader) (parse.Lexer, error) {
+	lex := &lexer {
+		fileName: fileName,
+		lineScanner: bufio.NewScanner(reader),
+	}
+	lex.nextRune()
+	return lex, nil
+}
+
+type lexer struct {
+	fileName    string
+	lineScanner *bufio.Scanner
+	rune        rune
+	line        string
+	lineFood    string
+	
+	offset   int
+	row      int
+	column   int
+
+	eof bool
+
+}
+
+func (this *lexer) Next() (parse.Token, error) {
+	token, err := this.nextInternal()
+	if err == io.EOF { err = this.errUnexpectedEOF() }
+	return token, err
+}
+
+func (this *lexer) nextInternal() (token parse.Token, err error) {
+	err = this.skipWhitespace()
+	token.Position = this.pos()
+	if this.eof {
+		token.Kind = parse.EOF
+		err = nil
+		return
+	}
+	if err != nil { return }
+
+	appendRune := func () {
+		token.Value += string(this.rune)
+		err = this.nextRune()
+	}
+
+	doNumber := func () {
+		for isDigit(this.rune) {
+			appendRune()
+			if this.eof { err = nil; return }
+			if err != nil { return }
+		}
+	}
+
+	defer func () {
+		newPos := this.pos()
+		newPos.End -- // TODO figure out why tf we have to do this
+		token.Position = token.Position.Union(newPos)
+	} ()
+
+	switch {
+	// Method
+	case this.rune == 'M':
+		token.Kind = TokenMethod
+		err = this.nextRune()
+		if err != nil { return }
+		doNumber()
+		if this.eof { err = nil; return }
+	// Key
+	case isDigit(this.rune):
+		token.Kind = TokenKey
+		doNumber()
+		if this.eof { err = nil; return }
+	// Ident
+	case unicode.IsUpper(this.rune):
+		token.Kind = TokenIdent
+		for unicode.IsLetter(this.rune) || isDigit(this.rune) {
+			appendRune()
+			if this.eof { err = nil; return }
+			if err != nil { return }
+		}
+	// Comma
+	case this.rune == ',':
+		token.Kind = TokenComma
+		appendRune()
+		if this.eof { err = nil; return }
+	// LBrace
+	case this.rune == '{':
+		token.Kind = TokenLBrace
+		appendRune()
+		if this.eof { err = nil; return }
+	// RBrace
+	case this.rune == '}':
+		token.Kind = TokenRBrace
+		appendRune()
+		if this.eof { err = nil; return }
+	// LBracket
+	case this.rune == '[':
+		token.Kind = TokenLBracket
+		appendRune()
+		if this.eof { err = nil; return }
+	// RBracket
+	case this.rune == ']':
+		token.Kind = TokenRBracket
+		appendRune()
+		if this.eof { err = nil; return }
+	case unicode.IsPrint(this.rune):
+		err = parse.Errorf (
+			this.pos(), "unexpected rune '%c'",
+			this.rune)
+	default:
+		err = parse.Errorf (
+			this.pos(), "unexpected rune %U",
+			this.rune)
+	}
+
+	return
+}
+
+func (this *lexer) nextRune() error {
+	if this.lineFood == "" {
+		ok := this.lineScanner.Scan()
+		if ok {
+			this.line     = this.lineScanner.Text()
+			this.lineFood = this.line
+			this.rune     = '\n'
+			this.column   = 0
+			this.row      ++
+		} else {
+			err := this.lineScanner.Err()
+			if err == nil {
+				this.eof = true
+				return io.EOF
+			} else {
+				return err
+			}
+		}
+	} else {
+		var ch   rune
+		var size int
+		for ch == 0 && this.lineFood != "" {
+			ch, size = utf8.DecodeRuneInString(this.lineFood)
+			this.lineFood = this.lineFood[size:]
+		}
+		this.rune = ch
+		this.column ++
+	}
+
+	return nil
+}
+
+func (this *lexer) skipWhitespace() error {
+	err := this.skipComment()
+	if err != nil { return err }
+	for isWhitespace(this.rune) {
+		err := this.nextRune()
+		if err != nil { return err }
+		err = this.skipComment()
+		if err != nil { return err }
+	}
+	return nil
+}
+
+func (this *lexer) skipComment() error {
+	if this.rune == ';' {
+		for this.rune != '\n' {
+			err := this.nextRune()
+			if err != nil { return err }
+		}
+	}
+	return nil
+}
+
+func (this *lexer) pos() parse.Position {
+	return parse.Position {
+		File:   this.fileName,
+		Line:   this.lineScanner.Text(),
+		Row:    this.row    - 1,
+		Start:  this.column - 1,
+		End:    this.column,
+	}
+}
+
+func (this *lexer) errUnexpectedEOF() error {
+	return parse.Errorf(this.pos(), "unexpected EOF")
+}
+
+func isWhitespace(char rune) bool {
+	switch char {
+	case ' ', '\t', '\r', '\n': return true
+	default: return false
+	}
+}
+
+func isDigit(char rune) bool {
+	return char >= '0' && char <= '9'
+}
+
+func isHexDigit(char rune) bool {
+	return isDigit(char) || char >= 'a' && char <= 'f' || char >= 'A' && char <= 'F'
+}

generate/lex_test.go

@@ -0,0 +1,54 @@
+package generate
+
+import "strings"
+import "testing"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+func TestLex(test *testing.T) {
+	lexer, err := Lex("test.pdl", strings.NewReader(`
+	M0001 User {
+		0000 Name      String,
+		0001 Users []User,
+		0002 Followers U32,
+	}`))
+	if err != nil { test.Fatal(parse.Format(err)) }
+
+	correctTokens := []parse.Token {
+		tok(TokenMethod,   "0001"),
+		tok(TokenIdent,    "User"),
+		tok(TokenLBrace,   "{"),
+		tok(TokenKey,      "0000"),
+		tok(TokenIdent,    "Name"),
+		tok(TokenIdent,    "String"),
+		tok(TokenComma,    ","),
+		tok(TokenKey,      "0001"),
+		tok(TokenIdent,    "Users"),
+		tok(TokenLBracket, "["),
+		tok(TokenRBracket, "]"),
+		tok(TokenIdent,    "User"),
+		tok(TokenComma,    ","),
+		tok(TokenKey,      "0002"),
+		tok(TokenIdent,    "Followers"),
+		tok(TokenIdent,    "U32"),
+		tok(TokenComma,    ","),
+		tok(TokenRBrace,   "}"),
+		tok(parse.EOF,     ""),
+	}
+
+	for index, correct := range correctTokens {
+		got, err := lexer.Next()
+		if err != nil { test.Fatal(parse.Format(err)) }
+		if got.Kind != correct.Kind || got.Value != correct.Value {
+			test.Logf("token %d mismatch", index)
+			test.Log("GOT:", tokenNames[got.Kind], got.Value)
+			test.Fatal("CORRECT:", tokenNames[correct.Kind], correct.Value)
+		}
+	}
+}
+
+func tok(kind parse.TokenKind, value string) parse.Token {
+	return parse.Token {
+		Kind:  kind,
+		Value: value,
+	}
+}

generate/parse.go

@@ -0,0 +1,206 @@
+package generate
+
+import "io"
+import "strconv"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+func Parse(lx parse.Lexer) (*Protocol, error) {
+	protocol := defaultProtocol()
+	par := parser {
+		Parser: parse.Parser {
+			Lexer: lx,
+			TokenNames: tokenNames,
+		},
+		protocol: &protocol,
+	}
+	err := par.parse()
+	if err != nil { return nil, err }
+	return par.protocol, nil
+}
+
+func defaultProtocol() Protocol {
+	return Protocol {
+		Messages: make(map[uint16] Message),
+		Types: map[string] Type {
+			"U8":     TypeInt { Bits: 8   },
+			"U16":    TypeInt { Bits: 16  },
+			"U32":    TypeInt { Bits: 32  },
+			"U64":    TypeInt { Bits: 64  },
+			"U128":   TypeInt { Bits: 128 },
+			"U256":   TypeInt { Bits: 256 },
+			"I8":     TypeInt { Bits: 8,   Signed: true },
+			"I16":    TypeInt { Bits: 16,  Signed: true },
+			"I32":    TypeInt { Bits: 32,  Signed: true },
+			"I64":    TypeInt { Bits: 64,  Signed: true },
+			"I128":   TypeInt { Bits: 128, Signed: true },
+			"I256":   TypeInt { Bits: 256, Signed: true },
+			"F16":    TypeFloat { Bits: 16  },
+			"F32":    TypeFloat { Bits: 32  },
+			"F64":    TypeFloat { Bits: 64  },
+			"F128":   TypeFloat { Bits: 128 },
+			"F256":   TypeFloat { Bits: 256 },
+			"String": TypeString { },
+			"Buffer": TypeBuffer { },
+			"Table":  TypeTable { },
+		},
+	}
+}
+
+func ParseReader(reader io.Reader) (*Protocol, error) {
+	lx, err := Lex("test.pdl", reader)
+	if err != nil { return nil, err }
+	return Parse(lx)
+}
+
+type parser struct {
+	parse.Parser
+	protocol *Protocol
+}
+
+func (this *parser) parse() error {
+	err := this.Next()
+	if err != nil { return err }
+	for this.Token.Kind != parse.EOF {
+		err = this.parseTopLevel()
+		if err != nil { return err }
+	}
+	return nil
+}
+
+func (this *parser) parseTopLevel() error {
+	err := this.ExpectDesc("message or typedef", TokenMethod, TokenIdent)
+	if err != nil { return err }
+	if this.EOF() { return nil }
+
+	switch this.Kind() {
+	case TokenMethod: return this.parseMessage()
+	case TokenIdent:  return this.parseTypedef()
+	}
+	panic("bug")
+}
+
+func (this *parser) parseMessage() error {
+	err := this.Expect(TokenMethod)
+	if err != nil { return err }
+	method, err := this.parseHexNumber(this.Value(), 0xFFFF)
+	if err != nil { return err }
+	err = this.ExpectNext(TokenIdent)
+	if err != nil { return err }
+	name := this.Value()
+	err = this.Next()
+	if err != nil { return err }
+	typ, err := this.parseType()
+	if err != nil { return err }
+	this.protocol.Messages[uint16(method)] = Message {
+		Name: name,
+		Type: typ,
+	}
+	return nil
+}
+
+func (this *parser) parseTypedef() error {
+	err := this.Expect(TokenIdent)
+	if err != nil { return err }
+	name := this.Value()
+	err = this.Next()
+	if err != nil { return err }
+	typ, err := this.parseType()
+	if err != nil { return err }
+	this.protocol.Types[name] = typ
+	return nil
+}
+
+func (this *parser) parseType() (Type, error) {
+	err := this.ExpectDesc("type", TokenIdent, TokenLBracket, TokenLBrace)
+	if err != nil { return nil, err }
+
+	switch this.Kind() {
+	case TokenIdent:
+		return this.parseTypeNamed()
+	case TokenLBracket:
+		return this.parseTypeArray()
+	case TokenLBrace:
+		return this.parseTypeTable()
+	}
+	panic("bug")
+}
+
+func (this *parser) parseTypeNamed() (TypeNamed, error) {
+	err := this.Expect(TokenIdent)
+	if err != nil { return TypeNamed { }, err }
+	name := this.Value()
+	err = this.Next()
+	if err != nil { return TypeNamed { }, err }
+	return TypeNamed { Name: name }, nil
+}
+
+func (this *parser) parseTypeArray() (TypeArray, error) {
+	err := this.Expect(TokenLBracket)
+	if err != nil { return TypeArray { }, err }
+	err = this.ExpectNext(TokenRBracket)
+	if err != nil { return TypeArray { }, err }
+	err = this.Next()
+	if err != nil { return TypeArray { }, err }
+	typ, err := this.parseType()
+	if err != nil { return TypeArray { }, err }
+	return TypeArray { Element: typ }, nil
+}
+
+func (this *parser) parseTypeTable() (TypeTableDefined, error) {
+	err := this.Expect(TokenLBrace)
+	if err != nil { return TypeTableDefined { }, err }
+	err = this.Next()
+	if err != nil { return TypeTableDefined { }, err }
+	typ := TypeTableDefined {
+		Fields: make(map[uint16] Field),
+	}
+	for {
+		err := this.ExpectDesc("table field", TokenKey, TokenRBrace)
+		if err != nil { return TypeTableDefined { }, err }
+		if this.Is(TokenRBrace) {
+			break
+		}
+		key, field, err := this.parseField()
+		if err != nil { return TypeTableDefined { }, err }
+		typ.Fields[key] = field
+		err = this.Expect(TokenComma, TokenRBrace)
+		if err != nil { return TypeTableDefined { }, err }
+		if this.Is(TokenRBrace) {
+			break
+		}
+		err = this.Next()
+		if err != nil { return TypeTableDefined { }, err }
+	}
+	err = this.Next()
+	if err != nil { return TypeTableDefined { }, err }
+	return typ, nil
+}
+
+func (this *parser) parseField() (uint16, Field, error) {
+	err := this.Expect(TokenKey)
+	if err != nil { return 0, Field { }, err }
+	key, err := this.parseHexNumber(this.Value(), 0xFFFF)
+	if err != nil { return 0, Field { }, err }
+	err = this.ExpectNext(TokenIdent)
+	if err != nil { return 0, Field { }, err }
+	name := this.Value()
+	err = this.Next()
+	if err != nil { return 0, Field { }, err }
+	typ, err := this.parseType()
+	if err != nil { return 0, Field { }, err }
+	return uint16(key), Field {
+		Name: name,
+		Type: typ,
+	}, nil
+}
+
+func (this *parser) parseHexNumber(input string, maxValue int64) (int64, error) {
+	number, err := strconv.ParseInt(input, 16, 64)
+	if err != nil {
+		return 0, parse.Errorf(this.Pos(), "%v", err)
+	}
+	if maxValue > 0 && number > maxValue {
+		return 0, parse.Errorf(this.Pos(), "value too large (max %X)", maxValue)
+	}
+	return number, nil
+}

generate/parse_test.go

@@ -0,0 +1,69 @@
+package generate
+
+import "fmt"
+import "strings"
+import "testing"
+// import "reflect"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+func TestParse(test *testing.T) {
+	correct := defaultProtocol()
+	correct.Messages[0x0000] = Message {
+		Name: "Connect",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Name",      Type: TypeNamed { Name: "String" } },
+				0x0001: Field { Name: "Password",  Type: TypeNamed { Name: "String" } },
+			},
+		},
+	}
+	correct.Messages[0x0001] = Message {
+		Name: "UserList",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Users",  Type: TypeArray { Element: TypeNamed { Name: "User" } } },
+			},
+		},
+	}
+	correct.Types["User"] = TypeTableDefined {
+		Fields: map[uint16] Field {
+			0x0000: Field { Name: "Name",      Type: TypeNamed { Name: "String" } },
+			0x0001: Field { Name: "Bio",       Type: TypeNamed { Name: "String" } },
+			0x0002: Field { Name: "Followers", Type: TypeNamed { Name: "U32" } },
+		},
+	}
+	test.Log("CORRECT:", &correct)
+	
+	got, err := ParseReader(strings.NewReader(`
+	M0000 Connect {
+		0000 Name String,
+		0001 Password String,
+	}
+	
+	M0001 UserList {
+		0000 Users []User,
+	}
+	
+	User {
+		0000 Name      String,
+		0001 Bio       String,
+		0002 Followers U32,
+	}
+	`))
+	if err != nil { test.Fatal(parse.Format(err)) }
+	test.Log("GOT:    ", got)
+
+	correctStr := fmt.Sprint(&correct)
+	gotStr     := fmt.Sprint(got)
+
+	if correctStr != gotStr {
+		test.Error("not equal")
+		for index := range min(len(correctStr), len(gotStr)) {
+			if correctStr[index] == gotStr[index] { continue }
+			test.Log("C:", correctStr[max(0, index - 8):min(len(correctStr), index + 8)])
+			test.Log("G:", gotStr[max(0, index - 8):min(len(gotStr), index + 8)])
+			break
+		}
+		test.FailNow()
+	}
+}

generate/protocol.go

@@ -0,0 +1,47 @@
+package generate
+
+type Protocol struct {
+	Messages map[uint16] Message
+	Types    map[string] Type
+}
+
+type Message struct {
+	Name string
+	Type Type
+}
+
+type Type interface {
+	
+}
+
+type TypeInt struct {
+	Bits   int
+	Signed bool
+}
+
+type TypeFloat struct {
+	Bits int
+}
+
+type TypeString struct { }
+
+type TypeBuffer struct { }
+
+type TypeArray struct {
+	Element Type
+}
+
+type TypeTable struct { }
+
+type TypeTableDefined struct {
+	Fields map[uint16] Field
+}
+
+type Field struct {
+	Name string
+	Type Type
+}
+
+type TypeNamed struct {
+	Name string
+}

go.mod

@@ -4,5 +4,5 @@ go 1.23.0
 
 require (
 	git.tebibyte.media/sashakoshka/go-util v0.9.1
-	github.com/gomarkdown/markdown v0.0.0-20241205020045-f7e15b2f3e62
+	git.tebibyte.media/sashakoshka/goparse v0.2.0
 )

go.sum

@@ -1,4 +1,4 @@
 git.tebibyte.media/sashakoshka/go-util v0.9.1 h1:eGAbLwYhOlh4aq/0w+YnJcxT83yPhXtxnYMzz6K7xGo=
 git.tebibyte.media/sashakoshka/go-util v0.9.1/go.mod h1:0Q1t+PePdx6tFYkRuJNcpM1Mru7wE6X+it1kwuOH+6Y=
-github.com/gomarkdown/markdown v0.0.0-20241205020045-f7e15b2f3e62 h1:pbAFUZisjG4s6sxvRJvf2N7vhpCvx2Oxb3PmS6pDO1g=
+git.tebibyte.media/sashakoshka/goparse v0.2.0 h1:uQmKvOCV2AOlCHEDjg9uclZCXQZzq2PxaXfZ1aIMiQI=
-github.com/gomarkdown/markdown v0.0.0-20241205020045-f7e15b2f3e62/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+git.tebibyte.media/sashakoshka/goparse v0.2.0/go.mod h1:tSQwfuD+EujRoKr6Y1oaRy74ZynatzkRLxjE3sbpCmk=