From b02ff6cda69ae1be6f214c7b7e24f6cc30884320 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 01:47:42 -0400 Subject: [PATCH 01/42] Created structs needed to represent a data section --- parser/body.go | 3 +- parser/tree.go | 132 +++++++++++++++++++++++++++++++++++++ tests/parser/data/main.arf | 26 +++++--- 3 files changed, 152 insertions(+), 9 deletions(-) diff --git a/parser/body.go b/parser/body.go index cd05305..a7d77c1 100644 --- a/parser/body.go +++ b/parser/body.go @@ -10,8 +10,9 @@ func (parser *ParsingOperation) parseBody () (err error) { switch parser.token.Value().(string) { case "data": case "type": - case "func": case "face": + case "enum": + case "func": } return diff --git a/parser/tree.go b/parser/tree.go index 4b95d9d..d93bf11 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -1,5 +1,8 @@ package parser +import "git.tebibyte.media/sashakoshka/arf/file" +import "git.tebibyte.media/sashakoshka/arf/types" + // SyntaxTree represents an abstract syntax tree. It covers an entire module. It // can be expected to be syntactically correct, but it might not be semantically // correct (because it has not been analyzed yet.) @@ -9,3 +12,132 @@ type SyntaxTree struct { requires []string } + +// Identifier represents a chain of arguments separated by a dot. +type Identifier struct { + location file.Location + trail []Argument +} + +// TypeKind represents what kind of type a type is +type TypeKind int + +const ( + // TypeKindBasic either means it's a primitive, or it inherits from + // something. + TypeKindBasic TypeKind = iota + + // TypeKindPointer means it's a pointer + TypeKindPointer + + // TypeKindArray means it's an array. + TypeKindArray +) + +// Type represents a type specifier +type Type struct { + location file.Location + + mutable bool + kind TypeKind + + // only applicable for arrays. a value of zero means it has an + // undefined/dynamic length. + length uint64 + + // not applicable for pointers. + name Identifier + + // only applicable for pointers. + points *Type +} + +// Declaration represents a variable declaration. +type Declaration struct { + location file.Location + name string + + what Type + value []Argument +} + +// ObjectAttribute represents a notation to initialize object attributes. It +// contains a name, and the value that the attribute should be initialized to. +type ObjectAttribute struct { + location file.Location + name string + value Argument +} + +// Phrase represents a function call or operator. In ARF they are the same +// syntactical concept. +type Phrase struct { + location file.Location + command Argument + arguments []Argument + returnsTo []Argument +} + +// ArgumentKind specifies the type of thing the value of an argument should be +// cast to. +type ArgumentKind int + +const ( + // [name argument] + // [name argument argument] + // etc... + ArgumentKindPhrase ArgumentKind = iota + + // , name value + ArgumentKindObjectAttribute + + // name.name + // name.name.name + // etc... + ArgumentKindIdentifier + + // name:Type + // name:{Type} + // name:{Type ...} + // name:{Type 23} + // etc... + ArgumentKindDeclaration + + // -1337 + ArgumentKindInt + + // 1337 + ArgumentKindUInt + + // 0.44 + ArgumentKindFloat + + // "hello world" + ArgumentKindString + + // 'S' + ArgumentKindRune + + // + - * / etc... + ArgumentKindOperator +) + +// Argument represents a value that can be placed anywhere a value goes. This +// allows things like phrases being arguments to other phrases. +type Argument struct { + location file.Location + what ArgumentKind + value any + // TODO: if there is an argument expansion operator its existence should + // be stored here in a boolean. +} + +// DataSection represents a global variable. +type DataSection struct { + location file.Location + name string + + what Type + value []Argument + permission types.Permission +} diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index d3419cd..f781595 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -15,13 +15,23 @@ data wr integerArrayInitialized:{Int 16} 340 0 2304 0 4785 92 data wr object:Obj - : this 324 - : that 2139 + , this 324 + , that 2139 data wr nestedObject:Obj - : this - : bird0 324 - : bird1 "hello world" - : that - : bird2 123.8439 - : bird3 9328.21348239 + , this + , bird0 324 + , bird1 "hello world" + , that + , bird2 123.8439 + , bird3 9328.21348239 + + +# parsing an object literal +func rr main + --- + [let object:Obj + , this 324 + , that 2139] + + let object:Obj , this 324 , that 2139 From d91423863bdba4238b7140a6ea9b002e82d3d3f7 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 14:04:57 -0400 Subject: [PATCH 02/42] Added ToString methods for syntax tree nodes --- parser/data.go | 2 +- parser/parser_test.go | 84 ++++++++++++++++- parser/tree-tostring.go | 179 +++++++++++++++++++++++++++++++++++++ parser/tree.go | 12 +-- tests/parser/data/main.arf | 15 ++-- types/permission.go | 16 ++++ 6 files changed, 293 insertions(+), 15 deletions(-) create mode 100644 parser/tree-tostring.go diff --git a/parser/data.go b/parser/data.go index 9f72e54..17f001d 100644 --- a/parser/data.go +++ b/parser/data.go @@ -2,5 +2,5 @@ package parser // parseData parses a data section func (parser *ParsingOperation) parseData () (err error) { - + return } diff --git a/parser/parser_test.go b/parser/parser_test.go index f4a7181..3824045 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -2,6 +2,7 @@ package parser import "reflect" import "testing" +import "git.tebibyte.media/sashakoshka/arf/types" func checkTree (modulePath string, correct *SyntaxTree, test *testing.T) { tree, err := Parse(modulePath) @@ -20,8 +21,23 @@ func checkTree (modulePath string, correct *SyntaxTree, test *testing.T) { } } +// quickIdentifier returns a simple identifier of names +func quickIdentifier (trail ...string) (identifier Identifier) { + for _, item := range trail { + identifier.trail = append ( + identifier.trail, + Argument { + what: ArgumentKindString, + value: item, + }, + ) + } + + return +} + func TestMeta (test *testing.T) { - checkTree("../tests/parser/meta",&SyntaxTree { + checkTree ("../tests/parser/meta", &SyntaxTree { license: "GPLv3", author: "Sasha Koshka", @@ -31,3 +47,69 @@ func TestMeta (test *testing.T) { }, }, test) } + +func TestData (test *testing.T) { + tree := &SyntaxTree { + dataSections: []DataSection { + DataSection { + name: "integer", + permission: types.PermissionFrom("wr"), + + what: Type { + kind: TypeKindBasic, + name: quickIdentifier("Int"), + }, + value: []Argument { + Argument { + what: ArgumentKindUInt, + value: 3202, + }, + }, + }, + + DataSection { + name: "integerPointer", + permission: types.PermissionFrom("wr"), + + what: Type { + kind: TypeKindPointer, + points: &Type { + kind: TypeKindBasic, + name: quickIdentifier("Int"), + }, + }, + value: []Argument { + Argument { + what: ArgumentKindUInt, + value: 3202, + }, + }, + }, + + DataSection { + name: "integerArray16", + permission: types.PermissionFrom("wr"), + + what: Type { + kind: TypeKindArray, + points: &Type { + kind: TypeKindBasic, + name: quickIdentifier("Int"), + }, + length: Argument { + what: ArgumentKindUInt, + value: 16, + } + }, + value: []Argument { + Argument { + what: ArgumentKindUInt, + value: 3202, + }, + }, + }, + }, + } + + checkTree ("../tests/parser/data", tree, test) +} diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go new file mode 100644 index 0000000..acfeaa9 --- /dev/null +++ b/parser/tree-tostring.go @@ -0,0 +1,179 @@ +package parser + +import "fmt" + +func doIndent (indent int, input ...string) (output string) { + for index := 0; index < indent; index ++ { + output += "\t" + } + for _, inputSection := range input { + output += inputSection + } + return +} + +func (tree *SyntaxTree) ToString (indent int) (output string) { + output += doIndent(indent, ":arf\n") + + if tree.author != "" { + output += doIndent(indent, "author \"", tree.author, "\"\n") + } + + if tree.license != "" { + output += doIndent(indent, "license \"", tree.license, "\"\n") + } + + for _, require := range tree.requires { + output += doIndent(indent, "require \"", require, "\"\n") + } + + output += doIndent(indent, "---\n") + + for _, require := range tree.dataSections { + output += require.ToString(1) + } + return +} + +func (identifier *Identifier) ToString () (output string) { + for index, trailItem := range identifier.trail { + if index > 0 { + output += "." + } + + output += trailItem.ToString(0, false) + } + return +} + +func (what *Type) ToString () (output string) { + if what.kind == TypeKindBasic { + output += what.name.ToString() + } else { + output += "{" + output += what.points.ToString() + output += "}" + } + + if what.mutable { + output += ":mut" + } + + return +} + +func (declaration *Declaration) ToString () (output string) { + output += declaration.name + ":" + output += declaration.what.ToString() + return +} + +func (attribute *ObjectAttribute) ToString ( + indent int, + breakLine bool, +) ( + output string, +) { + if breakLine { + output += doIndent(indent) + } + + output += ", " + attribute.name + if breakLine { + output += "\n" + attribute.value.ToString(indent + 1, true) + } else { + output += " " + attribute.value.ToString(0, false) + } + + return +} + +func (phrase *Phrase) ToString (indent int, breakLine bool) (output string) { + if breakLine { + output += doIndent ( + indent, + "[", phrase.command.ToString(0, false)) + output += "\n" + for _, argument := range phrase.arguments { + output += doIndent ( + indent, + argument.ToString(indent + 1, true)) + } + } else { + output += "[" + phrase.command.ToString(0, false) + for _, argument := range phrase.arguments { + output += " " + argument.ToString(0, false) + } + } + + output += "]" + + if len(phrase.returnsTo) > 0 { + output += " ->" + for _, returnItem := range phrase.returnsTo { + output += " " + returnItem.ToString(0, false) + } + } + + if breakLine { + output += "\n" + } + return +} + +func (argument *Argument) ToString (indent int, breakLine bool) (output string) { + if !breakLine { indent = 0 } + + switch argument.what { + case ArgumentKindPhrase: + output += doIndent ( + indent, + argument.value.(*Phrase).ToString ( + indent, + breakLine)) + + case ArgumentKindObjectAttribute: + output += doIndent ( + indent, + argument.value.(*ObjectAttribute).ToString ( + indent, + breakLine)) + + case ArgumentKindIdentifier: + output += doIndent ( + indent, + argument.value.(*Identifier).ToString()) + + case ArgumentKindDeclaration: + output += doIndent ( + indent, + argument.value.(*Declaration).ToString()) + + case ArgumentKindInt, ArgumentKindUInt, ArgumentKindFloat: + output += doIndent(indent, fmt.Sprint(argument.value)) + + case ArgumentKindString: + output += doIndent ( + indent, + "\"" + argument.value.(string) + "\"") + + case ArgumentKindRune: + output += doIndent ( + indent, + "'" + string(argument.value.(rune)) + "'") + + case ArgumentKindOperator: + // TODO + } + return +} + +func (section *DataSection) ToString (indent int) (output string) { + output += doIndent ( + indent, + "data ", + section.permission.ToString(), " ", + section.name, ":", + section.what.ToString()) + return +} diff --git a/parser/tree.go b/parser/tree.go index d93bf11..f4c095d 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -10,7 +10,8 @@ type SyntaxTree struct { license string author string - requires []string + requires []string + dataSections []DataSection } // Identifier represents a chain of arguments separated by a dot. @@ -41,14 +42,14 @@ type Type struct { mutable bool kind TypeKind - // only applicable for arrays. a value of zero means it has an + // only applicable for arrays. a value of nil means it has an // undefined/dynamic length. - length uint64 + length *Argument - // not applicable for pointers. + // only applicable for basic. name Identifier - // only applicable for pointers. + // not applicable for basic. points *Type } @@ -119,6 +120,7 @@ const ( ArgumentKindRune // + - * / etc... + // this is only used as a phrase command ArgumentKindOperator ) diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index f781595..b7cedc3 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -27,11 +27,10 @@ data wr nestedObject:Obj , bird3 9328.21348239 -# parsing an object literal -func rr main - --- - [let object:Obj - , this 324 - , that 2139] - - let object:Obj , this 324 , that 2139 +# func rr main + # --- + # [let object:Obj + # , this 324 + # , that 2139] + # + # let object:Obj , this 324 , that 2139 diff --git a/types/permission.go b/types/permission.go index 4421058..cd24150 100644 --- a/types/permission.go +++ b/types/permission.go @@ -30,3 +30,19 @@ func PermissionFrom (data string) (permission Permission) { permission.External = ModeFrom(rune(data[1])) return } + +func (mode Mode) ToString () (output string) { + switch mode { + case ModeNone: output = "n" + case ModeRead: output = "r" + case ModeWrite: output = "w" + } + + return +} + +func (permission Permission) ToString () (output string) { + output += permission.Internal.ToString() + output += permission.External.ToString() + return +} From 00bcfaab0be1b71673f33716ce372afb6d449bbb Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 14:17:29 -0400 Subject: [PATCH 03/42] Parser tests now work by checking the ToString() of the parsed tree --- parser/parser_test.go | 99 +++++++++---------------------------------- 1 file changed, 19 insertions(+), 80 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 3824045..ea6c19d 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1,21 +1,26 @@ package parser -import "reflect" +import "io" import "testing" -import "git.tebibyte.media/sashakoshka/arf/types" +// import "git.tebibyte.media/sashakoshka/arf/types" -func checkTree (modulePath string, correct *SyntaxTree, test *testing.T) { +func checkTree (modulePath string, correct string, test *testing.T) { tree, err := Parse(modulePath) - if err != nil { + if err != io.EOF { test.Log("returned error:") test.Log(err.Error()) test.Fail() return } - if !reflect.DeepEqual(tree, correct) { - test.Log("trees not equal") + treeString := tree.ToString(0) + if treeString != correct { + test.Log("trees not equal!") + test.Log("CORRECT TREE:") + test.Log(correct) + test.Log("WHAT WAS PARSED:") + test.Log(treeString) test.Fail() return } @@ -37,79 +42,13 @@ func quickIdentifier (trail ...string) (identifier Identifier) { } func TestMeta (test *testing.T) { - checkTree ("../tests/parser/meta", &SyntaxTree { - license: "GPLv3", - author: "Sasha Koshka", - requires: []string { - "someModule", - "otherModule", - }, - }, test) -} - -func TestData (test *testing.T) { - tree := &SyntaxTree { - dataSections: []DataSection { - DataSection { - name: "integer", - permission: types.PermissionFrom("wr"), - - what: Type { - kind: TypeKindBasic, - name: quickIdentifier("Int"), - }, - value: []Argument { - Argument { - what: ArgumentKindUInt, - value: 3202, - }, - }, - }, - - DataSection { - name: "integerPointer", - permission: types.PermissionFrom("wr"), - - what: Type { - kind: TypeKindPointer, - points: &Type { - kind: TypeKindBasic, - name: quickIdentifier("Int"), - }, - }, - value: []Argument { - Argument { - what: ArgumentKindUInt, - value: 3202, - }, - }, - }, - - DataSection { - name: "integerArray16", - permission: types.PermissionFrom("wr"), - - what: Type { - kind: TypeKindArray, - points: &Type { - kind: TypeKindBasic, - name: quickIdentifier("Int"), - }, - length: Argument { - what: ArgumentKindUInt, - value: 16, - } - }, - value: []Argument { - Argument { - what: ArgumentKindUInt, - value: 3202, - }, - }, - }, - }, - } - - checkTree ("../tests/parser/data", tree, test) + checkTree ("../tests/parser/meta", +`:arf +author "Sasha Koshka" +license "GPLv3" +require "someModule" +require "otherModule" +--- +`, test) } From 3a3c588023dc239245d8327e39bcae07e84afcdd Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 14:23:53 -0400 Subject: [PATCH 04/42] Added data test --- parser/parser_test.go | 35 ++++++++++++++++++++++++++++++++++- parser/tree-tostring.go | 4 ++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index ea6c19d..4e4a21b 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -42,7 +42,6 @@ func quickIdentifier (trail ...string) (identifier Identifier) { } func TestMeta (test *testing.T) { - checkTree ("../tests/parser/meta", `:arf author "Sasha Koshka" @@ -52,3 +51,37 @@ require "otherModule" --- `, test) } + +func TestData (test *testing.T) { + checkTree ("../tests/parser/data", +`:arf +--- +data wr integer:Int 3202 +data wr integerPointer:{Int} +data wr integerArray16:{Int 16} +data wr integerArrayVariable:{Int ...} +data wr integerArrayInitialized:{Int 16} + 3948 + 293 + 293049 + 948 + 912 + 340 + 0 + 2304 + 0 + 4785 + 92 +data wr object:Obj + , this 324 + , that 2139 +data wr nestedObject:Obj + , this + , bird0 324 + , bird1 "hello world" + , that + , bird2 123.8439 + , bird3 9328.21348239 +`, test) +} + diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index acfeaa9..8689f6c 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -175,5 +175,9 @@ func (section *DataSection) ToString (indent int) (output string) { section.permission.ToString(), " ", section.name, ":", section.what.ToString()) + + // TODO: print out initialization values. if there is only one of them, + // keep it on the same line. if there are more than one, give each its + // own line. return } From 7fc51c278f5d48256afccdb82b9d7c184cdf5e5c Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 14:30:54 -0400 Subject: [PATCH 05/42] Fixed issue with Error.Error not positioning marker correctly --- file/error.go | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/file/error.go b/file/error.go index 7cb437b..3f40a72 100644 --- a/file/error.go +++ b/file/error.go @@ -52,20 +52,30 @@ func (err Error) Error () (formattedMessage string) { if err.width > 0 { // print erroneous line + line := err.Location.file.lines[err.Location.row] formattedMessage += err.Location.file.lines[err.Location.row] + "\n" + // position error marker + var index int + for index = 0; index < err.Location.column; index ++ { + if line[index] == '\t' { + formattedMessage += "\t" + } else { + formattedMessage += " " + } + } + // print an arrow with a tail spanning the width of the mistake - columnCountdown := err.Location.column - for columnCountdown > 1 { - // TODO: for tabs, print out a teb instead. - formattedMessage += " " - columnCountdown -- - } for err.width > 1 { - // TODO: for tabs, print out 8 of these instead. - formattedMessage += "-" + if line[index] == '\t' { + formattedMessage += "--------" + } else { + formattedMessage += "-" + } + index ++ } + formattedMessage += "^\n" } formattedMessage += err.message + "\n" From d27c0ff07c915db6b2c1c86a2e0b139bbb5fa85e Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 14:32:59 -0400 Subject: [PATCH 06/42] Added single digit zero and eight to lexer number test --- lexer/lexer_test.go | 4 ++++ tests/lexer/numbers.arf | 1 + 2 files changed, 5 insertions(+) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 1d1b175..4a6fc96 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -91,6 +91,10 @@ func TestTokenizeAll (test *testing.T) { func TestTokenizeNumbers (test *testing.T) { checkTokenSlice("../tests/lexer/numbers.arf", []Token { + Token { kind: TokenKindUInt, value: uint64(0) }, + Token { kind: TokenKindNewline }, + Token { kind: TokenKindUInt, value: uint64(5) }, + Token { kind: TokenKindNewline }, Token { kind: TokenKindUInt, value: uint64(83628266) }, Token { kind: TokenKindNewline }, Token { kind: TokenKindUInt, value: uint64(83628266) }, diff --git a/tests/lexer/numbers.arf b/tests/lexer/numbers.arf index 9e512ca..e0da9bb 100644 --- a/tests/lexer/numbers.arf +++ b/tests/lexer/numbers.arf @@ -1,4 +1,5 @@ :arf +0 83628266 0b100111111000001000011101010 0x4Fc10Ea From 608162fa920b53dfedf19185195a23d6e41d5492 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 14:42:32 -0400 Subject: [PATCH 07/42] Fixed bug in number lexing function that prevented reading zero --- lexer/lexer_test.go | 2 +- lexer/numbers.go | 7 ------- tests/lexer/numbers.arf | 1 + 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 4a6fc96..bd44956 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -93,7 +93,7 @@ func TestTokenizeNumbers (test *testing.T) { checkTokenSlice("../tests/lexer/numbers.arf", []Token { Token { kind: TokenKindUInt, value: uint64(0) }, Token { kind: TokenKindNewline }, - Token { kind: TokenKindUInt, value: uint64(5) }, + Token { kind: TokenKindUInt, value: uint64(8) }, Token { kind: TokenKindNewline }, Token { kind: TokenKindUInt, value: uint64(83628266) }, Token { kind: TokenKindNewline }, diff --git a/lexer/numbers.go b/lexer/numbers.go index 4dce92b..7093750 100644 --- a/lexer/numbers.go +++ b/lexer/numbers.go @@ -1,7 +1,5 @@ package lexer -import "git.tebibyte.media/sashakoshka/arf/file" - // tokenizeSymbolBeginning lexes a token that starts with a number. func (lexer *LexingOperation) tokenizeNumberBeginning (negative bool) (err error) { var number uint64 @@ -23,11 +21,6 @@ func (lexer *LexingOperation) tokenizeNumberBeginning (negative bool) (err error number, fragment, isFloat, err = lexer.tokenizeNumber(10) } else if lexer.char >= '0' && lexer.char <= '9' { number, fragment, isFloat, err = lexer.tokenizeNumber(8) - } else { - return file.NewError ( - lexer.file.Location(1), - "unexpected character in number literal", - file.ErrorKindError) } } else { number, fragment, isFloat, err = lexer.tokenizeNumber(10) diff --git a/tests/lexer/numbers.arf b/tests/lexer/numbers.arf index e0da9bb..2cb6de4 100644 --- a/tests/lexer/numbers.arf +++ b/tests/lexer/numbers.arf @@ -1,5 +1,6 @@ :arf 0 +8 83628266 0b100111111000001000011101010 0x4Fc10Ea From 8b28fe5a4c66fdca64e9c9b3967fa684026389ac Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 14:50:09 -0400 Subject: [PATCH 08/42] Added a comma token --- lexer/lexer.go | 5 +++++ lexer/lexer_test.go | 1 + lexer/token.go | 3 +++ tests/lexer/all.arf | 2 +- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index cbdc5fb..9f6f8ff 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -179,6 +179,11 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) { token.kind = TokenKindDot lexer.addToken(token) err = lexer.nextRune() + case ',': + token := lexer.newToken() + token.kind = TokenKindComma + lexer.addToken(token) + err = lexer.nextRune() case '[': token := lexer.newToken() token.kind = TokenKindLBracket diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index bd44956..94aa630 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -62,6 +62,7 @@ func TestTokenizeAll (test *testing.T) { Token { kind: TokenKindName, value: "helloWorld" }, Token { kind: TokenKindColon }, Token { kind: TokenKindDot }, + Token { kind: TokenKindComma }, Token { kind: TokenKindLBracket }, Token { kind: TokenKindRBracket }, Token { kind: TokenKindLBrace }, diff --git a/lexer/token.go b/lexer/token.go index 0b2683f..cf18b66 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -24,6 +24,7 @@ const ( TokenKindColon TokenKindDot + TokenKindComma TokenKindLBracket TokenKindRBracket @@ -133,6 +134,8 @@ func (tokenKind TokenKind) Describe () (description string) { description = "Colon" case TokenKindDot: description = "Dot" + case TokenKindComma: + description = "Comma" case TokenKindLBracket: description = "LBracket" case TokenKindRBracket: diff --git a/tests/lexer/all.arf b/tests/lexer/all.arf index 6b9f9c6..da816d2 100644 --- a/tests/lexer/all.arf +++ b/tests/lexer/all.arf @@ -1,3 +1,3 @@ :arf ---- rw -> -349820394 932748397 239485.37520 "hello world!\n" 'E' helloWorld:.[]{} +--- rw -> -349820394 932748397 239485.37520 "hello world!\n" 'E' helloWorld:.,[]{} + - ++ -- * / @ ! % ~ < << > >> | || & && From 614b5664fc097a8f907705f3a5ac284f778704f3 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 15:09:07 -0400 Subject: [PATCH 09/42] Parser calls ParseDataSection --- parser/body.go | 7 +++++++ parser/data.go | 5 ++++- parser/tree.go | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/parser/body.go b/parser/body.go index a7d77c1..98bf5f3 100644 --- a/parser/body.go +++ b/parser/body.go @@ -9,6 +9,13 @@ func (parser *ParsingOperation) parseBody () (err error) { switch parser.token.Value().(string) { case "data": + var section *DataSection + section, err = parser.parseDataSection() + if err != nil { return } + if parser.tree.dataSections == nil { + parser.tree.dataSections = make(map[string] *DataSection) + } + parser.tree.dataSections[section.name] = section case "type": case "face": case "enum": diff --git a/parser/data.go b/parser/data.go index 17f001d..a1348f4 100644 --- a/parser/data.go +++ b/parser/data.go @@ -1,6 +1,9 @@ package parser // parseData parses a data section -func (parser *ParsingOperation) parseData () (err error) { +func (parser *ParsingOperation) parseDataSection () ( + section *DataSection, + err error, +) { return } diff --git a/parser/tree.go b/parser/tree.go index f4c095d..19a6654 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -11,7 +11,7 @@ type SyntaxTree struct { author string requires []string - dataSections []DataSection + dataSections map[string] *DataSection } // Identifier represents a chain of arguments separated by a dot. From e42bad5810b0427f22f9cabe6b59707ff1a3be2e Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 17:05:57 -0400 Subject: [PATCH 10/42] Identifiers can no longer have arguments in them Previously [[something something].something] would have been syntactically correct. This can lead to ugly and cluttered syntax due to violating the one thing per line guideline (that I've forgotten to write down) and would make the parser incredibly convoluded. Member selection in arf is not an operator and should not be treated as such. It would be much better to just use variables for this. --- parser/parser_test.go | 17 +---------------- parser/tree-tostring.go | 2 +- parser/tree.go | 2 +- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 4e4a21b..fb2bd17 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -7,7 +7,7 @@ import "testing" func checkTree (modulePath string, correct string, test *testing.T) { tree, err := Parse(modulePath) - if err != io.EOF { + if err != io.EOF && err != nil { test.Log("returned error:") test.Log(err.Error()) test.Fail() @@ -26,21 +26,6 @@ func checkTree (modulePath string, correct string, test *testing.T) { } } -// quickIdentifier returns a simple identifier of names -func quickIdentifier (trail ...string) (identifier Identifier) { - for _, item := range trail { - identifier.trail = append ( - identifier.trail, - Argument { - what: ArgumentKindString, - value: item, - }, - ) - } - - return -} - func TestMeta (test *testing.T) { checkTree ("../tests/parser/meta", `:arf diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index 8689f6c..7135f0f 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -41,7 +41,7 @@ func (identifier *Identifier) ToString () (output string) { output += "." } - output += trailItem.ToString(0, false) + output += trailItem } return } diff --git a/parser/tree.go b/parser/tree.go index 19a6654..08e61eb 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -17,7 +17,7 @@ type SyntaxTree struct { // Identifier represents a chain of arguments separated by a dot. type Identifier struct { location file.Location - trail []Argument + trail []string } // TypeKind represents what kind of type a type is From d081f363b1d47148206ccca3332eb4bf55401383 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 17:21:34 -0400 Subject: [PATCH 11/42] Add guideline I forgot to add earlier --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 6ca2326..21b85c4 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ These are some design goals that I have followed/am following: - Language syntax must have zero ambiguity - The compiler should not generate new functions or complex logic that the user has not written +- One line at a time - the language's syntax should encourage writing code that + flows vertically and not horizontally, with minimal nesting ## Planned features From 85b7938843e18df1a9298612a93a871fb44c2f92 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 15 Aug 2022 21:20:13 -0400 Subject: [PATCH 12/42] ParseBody now has a loop, and errors on unrecognized section type --- parser/body.go | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/parser/body.go b/parser/body.go index 98bf5f3..4f6fae6 100644 --- a/parser/body.go +++ b/parser/body.go @@ -1,26 +1,34 @@ package parser +import "git.tebibyte.media/sashakoshka/arf/file" import "git.tebibyte.media/sashakoshka/arf/lexer" // parse body parses the body of an arf file, after the metadata header. func (parser *ParsingOperation) parseBody () (err error) { - err = parser.nextToken(lexer.TokenKindName) - if err != nil { return } - - switch parser.token.Value().(string) { - case "data": - var section *DataSection - section, err = parser.parseDataSection() + for { + err = parser.expect(lexer.TokenKindName) if err != nil { return } - if parser.tree.dataSections == nil { - parser.tree.dataSections = make(map[string] *DataSection) - } - parser.tree.dataSections[section.name] = section - case "type": - case "face": - case "enum": - case "func": - } - return + sectionType := parser.token.Value().(string) + switch sectionType { + case "data": + var section *DataSection + section, err = parser.parseDataSection() + if err != nil { return } + if parser.tree.dataSections == nil { + parser.tree.dataSections = + make(map[string] *DataSection) + } + parser.tree.dataSections[section.name] = section + case "type": + case "face": + case "enum": + case "func": + default: + err = parser.token.NewError ( + "unknown section type \"" + sectionType + "\"", + file.ErrorKindError) + return + } + } } From ac40fa96e535abc2e13444a129dc35ce95d35cc4 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 01:55:22 -0400 Subject: [PATCH 13/42] Improved parser test Created new cases involving mutable data and phrase initialization arguments, and always print out the correct and parsed trees. --- parser/parser_test.go | 15 +++++++++------ tests/parser/data/main.arf | 7 +++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index fb2bd17..3ec8c0e 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -6,6 +6,12 @@ import "testing" func checkTree (modulePath string, correct string, test *testing.T) { tree, err := Parse(modulePath) + treeString := tree.ToString(0) + + test.Log("CORRECT TREE:") + test.Log(correct) + test.Log("WHAT WAS PARSED:") + test.Log(treeString) if err != io.EOF && err != nil { test.Log("returned error:") @@ -14,13 +20,8 @@ func checkTree (modulePath string, correct string, test *testing.T) { return } - treeString := tree.ToString(0) if treeString != correct { test.Log("trees not equal!") - test.Log("CORRECT TREE:") - test.Log(correct) - test.Log("WHAT WAS PARSED:") - test.Log(treeString) test.Fail() return } @@ -42,7 +43,9 @@ func TestData (test *testing.T) { `:arf --- data wr integer:Int 3202 -data wr integerPointer:{Int} +data wr mutInteger:Int:mut 3202 +data wr integerPointer:{Int} [& integer] +data wr mutIntegerPointer:{Int}:mut [& integer] data wr integerArray16:{Int 16} data wr integerArrayVariable:{Int ...} data wr integerArrayInitialized:{Int 16} diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index b7cedc3..3314221 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -3,8 +3,11 @@ data wr integer:Int 3202 -data wr integerPointer:{Int} -# TODO: data wr integerPointer:{Int} [& integer] +data wr mutInteger:Int:mut 3202 + +data wr integerPointer:{Int} [& integer] + +data wr mutIntegerPointer:{Int}:mut [& integer] data wr integerArray16:{Int 16} From f9786216734a020f5f1c08c20c386331cad7055d Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 10:44:02 -0400 Subject: [PATCH 14/42] Parse basic information about data sections (name, type) --- parser/data.go | 104 ++++++++++++++++++++++++++++++++++++++++++++++++- parser/tree.go | 4 +- 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/parser/data.go b/parser/data.go index a1348f4..c9a9da4 100644 --- a/parser/data.go +++ b/parser/data.go @@ -1,9 +1,111 @@ package parser -// parseData parses a data section +import "git.tebibyte.media/sashakoshka/arf/file" +import "git.tebibyte.media/sashakoshka/arf/types" +import "git.tebibyte.media/sashakoshka/arf/lexer" + +// parseData parses a data section. func (parser *ParsingOperation) parseDataSection () ( section *DataSection, err error, ) { + err = parser.expect(lexer.TokenKindName) + if err != nil { return } + + section = &DataSection { location: parser.token.Location() } + + err = parser.nextToken(lexer.TokenKindPermission) + if err != nil { return } + section.permission = parser.token.Value().(types.Permission) + + err = parser.nextToken(lexer.TokenKindName) + if err != nil { return } + section.name = parser.token.Value().(string) + + err = parser.nextToken(lexer.TokenKindColon) + if err != nil { return } + section.what, err = parser.parseType() + if err != nil { return } + + return +} + +// parseType parses a type notation of the form Name, {Name}, etc. +func (parser *ParsingOperation) parseType () (what Type, err error) { + err = parser.nextToken(lexer.TokenKindName, lexer.TokenKindLBrace) + if err != nil { return } + what.location = parser.token.Location() + + if parser.token.Is(lexer.TokenKindLBrace) { + what.kind = TypeKindPointer + + var points Type + points, err = parser.parseType() + if err != nil { return } + what.points = &points + + err = parser.nextToken ( + lexer.TokenKindUInt, + lexer.TokenKindRBrace) + if err != nil { return } + + if parser.token.Is(lexer.TokenKindUInt) { + what.kind = TypeKindArray + + what.length = parser.token.Value().(uint64) + + err = parser.nextToken ( + lexer.TokenKindUInt, + lexer.TokenKindRBrace) + if err != nil { return } + } + } else { + what.name, err = parser.parseIdentifier() + if err != nil { return } + } + + err = parser.nextToken() + + if parser.token.Is(lexer.TokenKindColon) { + err = parser.nextToken(lexer.TokenKindName) + if err != nil { return } + + qualifier := parser.token.Value().(string) + switch qualifier { + case "mut": + what.mutable = true + default: + err = parser.token.NewError ( + "unknown type qualifier \"" + qualifier + "\"", + file.ErrorKindError) + return + } + } + + return +} + +// parseIdentifier parses an identifier made out of dot separated names. +func (parser *ParsingOperation) parseIdentifier () ( + identifier Identifier, + err error, +) { + err = parser.expect(lexer.TokenKindName) + if err != nil { return } + identifier.location = parser.token.Location() + + for { + if !parser.token.Is(lexer.TokenKindName) { break } + + identifier.trail = append ( + identifier.trail, + parser.token.Value().(string)) + + err = parser.nextToken() + if err != nil { return } + + if !parser.token.Is(lexer.TokenKindDot) { break } + } + return } diff --git a/parser/tree.go b/parser/tree.go index 08e61eb..830c466 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -42,9 +42,9 @@ type Type struct { mutable bool kind TypeKind - // only applicable for arrays. a value of nil means it has an + // only applicable for arrays. a value of zero means it has an // undefined/dynamic length. - length *Argument + length uint64 // only applicable for basic. name Identifier From 4dfb32755803d66cb3f79b3e20ec4e73902212f4 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 13:43:36 -0400 Subject: [PATCH 15/42] Add subscript and dereference argument kinds --- parser/tree.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/parser/tree.go b/parser/tree.go index 830c466..ea81d24 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -89,6 +89,12 @@ const ( // etc... ArgumentKindPhrase ArgumentKind = iota + // {name} + ArgumentKindDereference + + // {name 23} + ArgumentKindSubscript + // , name value ArgumentKindObjectAttribute From 9cb2f68581d338230961a83879ee6be64d4a3e2a Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 15:50:34 -0400 Subject: [PATCH 16/42] Argument kind is now called .kind instead of .what --- parser/tree-tostring.go | 2 +- parser/tree.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index 7135f0f..a46da08 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -124,7 +124,7 @@ func (phrase *Phrase) ToString (indent int, breakLine bool) (output string) { func (argument *Argument) ToString (indent int, breakLine bool) (output string) { if !breakLine { indent = 0 } - switch argument.what { + switch argument.kind { case ArgumentKindPhrase: output += doIndent ( indent, diff --git a/parser/tree.go b/parser/tree.go index ea81d24..7919bde 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -57,9 +57,7 @@ type Type struct { type Declaration struct { location file.Location name string - - what Type - value []Argument + what Type } // ObjectAttribute represents a notation to initialize object attributes. It @@ -134,7 +132,7 @@ const ( // allows things like phrases being arguments to other phrases. type Argument struct { location file.Location - what ArgumentKind + kind ArgumentKind value any // TODO: if there is an argument expansion operator its existence should // be stored here in a boolean. From 45bc798d199fe1d115156cda5cff212610ed5517 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 15:53:32 -0400 Subject: [PATCH 17/42] Added identifier and declaration parsing --- parser/argument.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 parser/argument.go diff --git a/parser/argument.go b/parser/argument.go new file mode 100644 index 0000000..6a94bee --- /dev/null +++ b/parser/argument.go @@ -0,0 +1,76 @@ +package parser + +import "git.tebibyte.media/sashakoshka/arf/file" +import "git.tebibyte.media/sashakoshka/arf/lexer" + +var validArgumentStartTokens = []lexer.TokenKind { + lexer.TokenKindName, + + lexer.TokenKindInt, + lexer.TokenKindUInt, + lexer.TokenKindFloat, + lexer.TokenKindString, + lexer.TokenKindRune, + + lexer.TokenKindLBrace, + lexer.TokenKindLBracket, +} + +func (parser *ParsingOperation) parseArgument () (argument Argument, err error) { + argument.location = parser.token.Location() + + err = parser.expect(validArgumentStartTokens...) + if err != nil { return } + + switch parser.token.Kind() { + case lexer.TokenKindName: + var identifier Identifier + identifier, err = parser.parseIdentifier() + if err != nil { return } + + if parser.token.Is(lexer.TokenKindColon) { + var what Type + what, err = parser.parseType() + if err != nil { return } + + if len(identifier.trail) != 1 { + err = parser.token.NewError ( + "cannot use member selection in " + + "a variable definition", + file.ErrorKindError) + return + } + + argument.kind = ArgumentKindDeclaration + argument.value = Declaration { + location: argument.location, + name: identifier.trail[0], + what: what, + } + } else { + argument.kind = ArgumentKindIdentifier + argument.value = identifier + } + + case lexer.TokenKindInt: + + case lexer.TokenKindUInt: + + case lexer.TokenKindFloat: + + case lexer.TokenKindString: + + case lexer.TokenKindRune: + + case lexer.TokenKindLBrace: + + case lexer.TokenKindLBracket: + + default: + panic ( + "unimplemented argument kind " + + parser.token.Kind().Describe()) + } + + return +} From 0025d03a18d2acf17e71c8b21c8f657ac54f27f3 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 16:16:39 -0400 Subject: [PATCH 18/42] Modified parser data test to have uninitialized pointer cases --- parser/parser_test.go | 6 ++++-- tests/parser/data/main.arf | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 3ec8c0e..24219db 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -44,8 +44,8 @@ func TestData (test *testing.T) { --- data wr integer:Int 3202 data wr mutInteger:Int:mut 3202 -data wr integerPointer:{Int} [& integer] -data wr mutIntegerPointer:{Int}:mut [& integer] +data wr integerPointer:{Int} +data wr mutIntegerPointer:{Int}:mut data wr integerArray16:{Int 16} data wr integerArrayVariable:{Int ...} data wr integerArrayInitialized:{Int 16} @@ -60,6 +60,8 @@ data wr integerArrayInitialized:{Int 16} 0 4785 92 +data wr integerPointerInit:{Int} [& integer] +data wr mutIntegerPointerInit:{Int}:mut [& integer] data wr object:Obj , this 324 , that 2139 diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index 3314221..fbcbf12 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -5,9 +5,9 @@ data wr integer:Int 3202 data wr mutInteger:Int:mut 3202 -data wr integerPointer:{Int} [& integer] +data wr integerPointer:{Int} -data wr mutIntegerPointer:{Int}:mut [& integer] +data wr mutIntegerPointer:{Int}:mut data wr integerArray16:{Int 16} @@ -16,6 +16,10 @@ data wr integerArrayVariable:{Int ...} data wr integerArrayInitialized:{Int 16} 3948 293 293049 948 912 340 0 2304 0 4785 92 + +data wr integerPointerInit:{Int} [& integer] + +data wr mutIntegerPointerInit:{Int}:mut [& integer] data wr object:Obj , this 324 From 5c23c59c925c9bc3b9135c67e055aacb77824a83 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 16:27:52 -0400 Subject: [PATCH 19/42] Added parsing primitive arguments --- parser/argument.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/parser/argument.go b/parser/argument.go index 6a94bee..c7f42b0 100644 --- a/parser/argument.go +++ b/parser/argument.go @@ -53,18 +53,33 @@ func (parser *ParsingOperation) parseArgument () (argument Argument, err error) } case lexer.TokenKindInt: + argument.kind = ArgumentKindInt + argument.value = parser.token.Value().(int64) + err = parser.nextToken() case lexer.TokenKindUInt: + argument.kind = ArgumentKindUInt + argument.value = parser.token.Value().(uint64) + err = parser.nextToken() case lexer.TokenKindFloat: + argument.kind = ArgumentKindFloat + argument.value = parser.token.Value().(float64) + err = parser.nextToken() case lexer.TokenKindString: + argument.kind = ArgumentKindString + argument.value = parser.token.Value().(string) + parser.nextToken() case lexer.TokenKindRune: + argument.kind = ArgumentKindRune + argument.value = parser.token.Value().(rune) + parser.nextToken() - case lexer.TokenKindLBrace: + // case lexer.TokenKindLBrace: - case lexer.TokenKindLBracket: + // case lexer.TokenKindLBracket: default: panic ( From 5e2d8c995508e256dd735095399dccdc8722a621 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 16:37:20 -0400 Subject: [PATCH 20/42] Parser can now ToString data sections properly --- parser/tree-tostring.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index a46da08..6ce75ec 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -30,7 +30,7 @@ func (tree *SyntaxTree) ToString (indent int) (output string) { output += doIndent(indent, "---\n") for _, require := range tree.dataSections { - output += require.ToString(1) + output += require.ToString(indent) } return } @@ -165,6 +165,8 @@ func (argument *Argument) ToString (indent int, breakLine bool) (output string) case ArgumentKindOperator: // TODO } + + if breakLine { output += "\n" } return } @@ -176,8 +178,16 @@ func (section *DataSection) ToString (indent int) (output string) { section.name, ":", section.what.ToString()) - // TODO: print out initialization values. if there is only one of them, - // keep it on the same line. if there are more than one, give each its - // own line. + if len(section.value) == 0 { + output += "\n" + } else if len(section.value) == 1 { + output += " " + section.value[0].ToString(0, false) + output += "\n" + } else { + output += "\n" + for _, argument := range(section.value) { + output += argument.ToString(indent + 1, true) + } + } return } From 16bca57e363b5ca011e78abb4241821366a0e7e9 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 17:21:10 -0400 Subject: [PATCH 21/42] Fixed numerous problems related to type parsing --- parser/data.go | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/parser/data.go b/parser/data.go index c9a9da4..9f81fc4 100644 --- a/parser/data.go +++ b/parser/data.go @@ -24,27 +24,44 @@ func (parser *ParsingOperation) parseDataSection () ( err = parser.nextToken(lexer.TokenKindColon) if err != nil { return } + err = parser.nextToken() + if err != nil { return } section.what, err = parser.parseType() if err != nil { return } + if parser.token.Is(lexer.TokenKindNewline) { + // TODO: parse arguments + } else { + var argument Argument + argument, err = parser.parseArgument() + section.value = append(section.value, argument) + + err = parser.expect(lexer.TokenKindNewline) + if err != nil { return } + err = parser.nextToken() + if err != nil { return } + } return } // parseType parses a type notation of the form Name, {Name}, etc. func (parser *ParsingOperation) parseType () (what Type, err error) { - err = parser.nextToken(lexer.TokenKindName, lexer.TokenKindLBrace) + err = parser.expect(lexer.TokenKindName, lexer.TokenKindLBrace) if err != nil { return } what.location = parser.token.Location() if parser.token.Is(lexer.TokenKindLBrace) { what.kind = TypeKindPointer + + err = parser.nextToken() + if err != nil { return } var points Type points, err = parser.parseType() if err != nil { return } what.points = &points - err = parser.nextToken ( + err = parser.expect ( lexer.TokenKindUInt, lexer.TokenKindRBrace) if err != nil { return } @@ -54,18 +71,17 @@ func (parser *ParsingOperation) parseType () (what Type, err error) { what.length = parser.token.Value().(uint64) - err = parser.nextToken ( - lexer.TokenKindUInt, - lexer.TokenKindRBrace) + err = parser.nextToken(lexer.TokenKindRBrace) if err != nil { return } } + + err = parser.nextToken() + if err != nil { return } } else { what.name, err = parser.parseIdentifier() if err != nil { return } } - err = parser.nextToken() - if parser.token.Is(lexer.TokenKindColon) { err = parser.nextToken(lexer.TokenKindName) if err != nil { return } @@ -80,6 +96,9 @@ func (parser *ParsingOperation) parseType () (what Type, err error) { file.ErrorKindError) return } + + err = parser.nextToken() + if err != nil { return } } return From c172c111d80745a712800a1256a7f09d1ee7c092 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 17:45:31 -0400 Subject: [PATCH 22/42] Rethought how object and array literals will work --- parser/parser_test.go | 16 ++++++++-------- tests/parser/data/main.arf | 28 ++++++++++++++++++---------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 24219db..efefe04 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -63,15 +63,15 @@ data wr integerArrayInitialized:{Int 16} data wr integerPointerInit:{Int} [& integer] data wr mutIntegerPointerInit:{Int}:mut [& integer] data wr object:Obj - , this 324 - , that 2139 + .this 324 + .that 2139 data wr nestedObject:Obj - , this - , bird0 324 - , bird1 "hello world" - , that - , bird2 123.8439 - , bird3 9328.21348239 + .this + .bird0 324 + .bird1 "hello world" + .that + .bird2 123.8439 + .bird3 9328.21348239 `, test) } diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index fbcbf12..7fea405 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -26,18 +26,26 @@ data wr object:Obj , that 2139 data wr nestedObject:Obj - , this - , bird0 324 - , bird1 "hello world" - , that - , bird2 123.8439 - , bird3 9328.21348239 + .this + .bird0 324 + .bird1 "hello world" + .that + .bird2 123.8439 + .bird3 9328.21348239 # func rr main # --- - # [let object:Obj - # , this 324 - # , that 2139] + # # TODO: set should be a special case, checking under itself for object + # member initialization args. it should also check for args in general + # under there which should be treated as array initialization args. + # basically, under a set phrase, it should do the same checks that it + # does under a data section. + # + # [set object:Obj] + # .this 324 + # .that 2139 # - # let object:Obj , this 324 , that 2139 + # set object:Obj + # .this 324 + # .that 2139 From efb3bbe21bacf9cbde6d4528b5c661a6a80fd057 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 20:10:47 -0400 Subject: [PATCH 23/42] Added base for parsing initialization values --- parser/data.go | 31 ++++++++++++++++++++++++++++++- tests/parser/data/main.arf | 4 ++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/parser/data.go b/parser/data.go index 9f81fc4..983fcda 100644 --- a/parser/data.go +++ b/parser/data.go @@ -30,7 +30,11 @@ func (parser *ParsingOperation) parseDataSection () ( if err != nil { return } if parser.token.Is(lexer.TokenKindNewline) { - // TODO: parse arguments + err = parser.nextToken() + if err != nil { return } + + section.value, err = parser.parseInitializationValues(0) + if err != nil { return } } else { var argument Argument argument, err = parser.parseArgument() @@ -44,6 +48,29 @@ func (parser *ParsingOperation) parseDataSection () ( return } +// parseInitializationValues starts on the line after a data section, or a set +// phrase. It checks for an indent greater than the indent of the aforementioned +// data section or set phrase (passed through baseIndent), and if there is, +// it parses initialization values. +func (parser *ParsingOperation) parseInitializationValues ( + baseIndent int, +) ( + values []Argument, + err error, +) { + // check if line is indented one more than baseIndent + if !parser.token.Is(lexer.TokenKindIndent) { return } + if parser.token.Value().(int) != baseIndent + 1 { return } + + if parser.token.Is(lexer.TokenKindDot) { + // TODO: parse as object initialization + } else { + // TODO: parse as array initialization + } + + return +} + // parseType parses a type notation of the form Name, {Name}, etc. func (parser *ParsingOperation) parseType () (what Type, err error) { err = parser.expect(lexer.TokenKindName, lexer.TokenKindLBrace) @@ -114,6 +141,8 @@ func (parser *ParsingOperation) parseIdentifier () ( identifier.location = parser.token.Location() for { + // TODO: eat up newlines and tabs after the dot, but not before + // it. if !parser.token.Is(lexer.TokenKindName) { break } identifier.trail = append ( diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index 7fea405..670fb17 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -22,8 +22,8 @@ data wr integerPointerInit:{Int} [& integer] data wr mutIntegerPointerInit:{Int}:mut [& integer] data wr object:Obj - , this 324 - , that 2139 + .this 324 + .that 2139 data wr nestedObject:Obj .this From 9e01eef45b2a1d84b78854c1ea8bf02f824e647f Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 20:24:27 -0400 Subject: [PATCH 24/42] Added elipsis token --- lexer/lexer.go | 7 ++++++- lexer/lexer_test.go | 1 + lexer/token.go | 3 +++ parser/parser_test.go | 2 +- tests/lexer/all.arf | 2 +- tests/parser/data/main.arf | 2 +- 6 files changed, 13 insertions(+), 4 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 9f6f8ff..4cc7660 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -175,10 +175,15 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) { lexer.addToken(token) err = lexer.nextRune() case '.': + err = lexer.nextRune() + if err != nil { return } token := lexer.newToken() token.kind = TokenKindDot + if lexer.char == '.' { + token.kind = TokenKindElipsis + err = lexer.nextRune() + } lexer.addToken(token) - err = lexer.nextRune() case ',': token := lexer.newToken() token.kind = TokenKindComma diff --git a/lexer/lexer_test.go b/lexer/lexer_test.go index 94aa630..27216d0 100644 --- a/lexer/lexer_test.go +++ b/lexer/lexer_test.go @@ -63,6 +63,7 @@ func TestTokenizeAll (test *testing.T) { Token { kind: TokenKindColon }, Token { kind: TokenKindDot }, Token { kind: TokenKindComma }, + Token { kind: TokenKindElipsis }, Token { kind: TokenKindLBracket }, Token { kind: TokenKindRBracket }, Token { kind: TokenKindLBrace }, diff --git a/lexer/token.go b/lexer/token.go index cf18b66..43cd675 100644 --- a/lexer/token.go +++ b/lexer/token.go @@ -24,6 +24,7 @@ const ( TokenKindColon TokenKindDot + TokenKindElipsis TokenKindComma TokenKindLBracket @@ -134,6 +135,8 @@ func (tokenKind TokenKind) Describe () (description string) { description = "Colon" case TokenKindDot: description = "Dot" + case TokenKindElipsis: + description = "Elipsis" case TokenKindComma: description = "Comma" case TokenKindLBracket: diff --git a/parser/parser_test.go b/parser/parser_test.go index efefe04..46c35e1 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -47,7 +47,7 @@ data wr mutInteger:Int:mut 3202 data wr integerPointer:{Int} data wr mutIntegerPointer:{Int}:mut data wr integerArray16:{Int 16} -data wr integerArrayVariable:{Int ...} +data wr integerArrayVariable:{Int ..} data wr integerArrayInitialized:{Int 16} 3948 293 diff --git a/tests/lexer/all.arf b/tests/lexer/all.arf index da816d2..29834fe 100644 --- a/tests/lexer/all.arf +++ b/tests/lexer/all.arf @@ -1,3 +1,3 @@ :arf ---- rw -> -349820394 932748397 239485.37520 "hello world!\n" 'E' helloWorld:.,[]{} +--- rw -> -349820394 932748397 239485.37520 "hello world!\n" 'E' helloWorld:.,..[]{} + - ++ -- * / @ ! % ~ < << > >> | || & && diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index 670fb17..8b03dcc 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -11,7 +11,7 @@ data wr mutIntegerPointer:{Int}:mut data wr integerArray16:{Int 16} -data wr integerArrayVariable:{Int ...} +data wr integerArrayVariable:{Int ..} data wr integerArrayInitialized:{Int 16} 3948 293 293049 948 912 From 3407aa7c5941b93147773e85e291d08983b835bd Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 20:26:06 -0400 Subject: [PATCH 25/42] Fixed lexing digraph tokens Lexer gave wrong token locations and would skip an extra rune when digraph was not found. --- lexer/lexer.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lexer/lexer.go b/lexer/lexer.go index 4cc7660..1c2b761 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -175,9 +175,9 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) { lexer.addToken(token) err = lexer.nextRune() case '.': + token := lexer.newToken() err = lexer.nextRune() if err != nil { return } - token := lexer.newToken() token.kind = TokenKindDot if lexer.char == '.' { token.kind = TokenKindElipsis @@ -210,15 +210,15 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) { lexer.addToken(token) err = lexer.nextRune() case '+': + token := lexer.newToken() err = lexer.nextRune() if err != nil { return } - token := lexer.newToken() token.kind = TokenKindPlus if lexer.char == '+' { token.kind = TokenKindIncrement + err = lexer.nextRune() } lexer.addToken(token) - err = lexer.nextRune() case '-': err = lexer.tokenizeDashBeginning() case '*': @@ -252,45 +252,45 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) { lexer.addToken(token) err = lexer.nextRune() case '<': + token := lexer.newToken() err = lexer.nextRune() if err != nil { return } - token := lexer.newToken() token.kind = TokenKindLessThan if lexer.char == '<' { token.kind = TokenKindLShift + err = lexer.nextRune() } lexer.addToken(token) - err = lexer.nextRune() case '>': + token := lexer.newToken() err = lexer.nextRune() if err != nil { return } - token := lexer.newToken() token.kind = TokenKindGreaterThan if lexer.char == '>' { token.kind = TokenKindRShift + err = lexer.nextRune() } lexer.addToken(token) - err = lexer.nextRune() case '|': + token := lexer.newToken() err = lexer.nextRune() if err != nil { return } - token := lexer.newToken() token.kind = TokenKindBinaryOr if lexer.char == '|' { token.kind = TokenKindLogicalOr + err = lexer.nextRune() } lexer.addToken(token) - err = lexer.nextRune() case '&': + token := lexer.newToken() err = lexer.nextRune() if err != nil { return } - token := lexer.newToken() token.kind = TokenKindBinaryAnd if lexer.char == '&' { token.kind = TokenKindLogicalAnd + err = lexer.nextRune() } lexer.addToken(token) - err = lexer.nextRune() default: err = file.NewError ( lexer.file.Location(1), From 97cb6e54eb41e5a015a7b05eea2b2c77dd15b0d7 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 20:53:27 -0400 Subject: [PATCH 26/42] Type.ToString now understands array lengths --- parser/tree-tostring.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index 6ce75ec..f598807 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -52,6 +52,16 @@ func (what *Type) ToString () (output string) { } else { output += "{" output += what.points.ToString() + + if what.kind == TypeKindArray { + output += " " + if what.length == 0 { + output += ".." + } else { + output += fmt.Sprint(what.length) + } + } + output += "}" } @@ -164,6 +174,9 @@ func (argument *Argument) ToString (indent int, breakLine bool) (output string) case ArgumentKindOperator: // TODO + // also when parsing this argument kind, don't do it in the + // argument parsing function. do it specifically when parsing a + // phrase command. } if breakLine { output += "\n" } From 210e527b3a05f6a2b4ddef367a648a6e7f4bcd4d Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 20:55:43 -0400 Subject: [PATCH 27/42] parseType method now understands arrays with undefined length --- parser/data.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/parser/data.go b/parser/data.go index 983fcda..fa4130b 100644 --- a/parser/data.go +++ b/parser/data.go @@ -90,7 +90,8 @@ func (parser *ParsingOperation) parseType () (what Type, err error) { err = parser.expect ( lexer.TokenKindUInt, - lexer.TokenKindRBrace) + lexer.TokenKindRBrace, + lexer.TokenKindElipsis) if err != nil { return } if parser.token.Is(lexer.TokenKindUInt) { @@ -98,6 +99,11 @@ func (parser *ParsingOperation) parseType () (what Type, err error) { what.length = parser.token.Value().(uint64) + err = parser.nextToken(lexer.TokenKindRBrace) + if err != nil { return } + } else if parser.token.Is(lexer.TokenKindElipsis) { + what.kind = TypeKindArray + err = parser.nextToken(lexer.TokenKindRBrace) if err != nil { return } } From bb2948d3974bbeb543213a7cb321b57c94f7467a Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 21:31:23 -0400 Subject: [PATCH 28/42] Added parsing array initialization values --- parser/data.go | 37 ++++++++++++++++++++++++++++++++++++- tests/parser/data/main.arf | 4 ++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/parser/data.go b/parser/data.go index fa4130b..918e0d2 100644 --- a/parser/data.go +++ b/parser/data.go @@ -61,16 +61,51 @@ func (parser *ParsingOperation) parseInitializationValues ( // check if line is indented one more than baseIndent if !parser.token.Is(lexer.TokenKindIndent) { return } if parser.token.Value().(int) != baseIndent + 1 { return } + + err = parser.nextToken() + if err != nil { return } if parser.token.Is(lexer.TokenKindDot) { // TODO: parse as object initialization } else { - // TODO: parse as array initialization + values, err = parser.parseArrayInitializationValues ( + baseIndent + 1) } return } +// parseArrayInitializationValues parses a list of array initialization values +// until the indentation lexel is not equal to indent. +func (parser *ParsingOperation) parseArrayInitializationValues ( + indent int, +) ( + values []Argument, + err error, +) { + for { + if parser.token.Is(lexer.TokenKindNewline) { + println("line break") + err = parser.nextToken() + if err != nil { return } + + if !parser.token.Is(lexer.TokenKindIndent) { break } + if parser.token.Value().(int) != indent { break } + err = parser.nextToken() + if err != nil { return } + } + if err != nil { return } + println(parser.token.Describe()) + + var argument Argument + argument, err = parser.parseArgument() + if err != nil { return } + values = append(values, argument) + } + + return +} + // parseType parses a type notation of the form Name, {Name}, etc. func (parser *ParsingOperation) parseType () (what Type, err error) { err = parser.expect(lexer.TokenKindName, lexer.TokenKindLBrace) diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index 8b03dcc..9d6e347 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -17,9 +17,9 @@ data wr integerArrayInitialized:{Int 16} 3948 293 293049 948 912 340 0 2304 0 4785 92 -data wr integerPointerInit:{Int} [& integer] +data wr integerPointerInit:{Int} #[& integer] -data wr mutIntegerPointerInit:{Int}:mut [& integer] +data wr mutIntegerPointerInit:{Int}:mut #[& integer] data wr object:Obj .this 324 From eb3fb65c9b967de7d64d3833e9c137563a925936 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 16 Aug 2022 23:45:25 -0400 Subject: [PATCH 29/42] Changed object initialization values to be a map --- parser/tree-tostring.go | 27 ++++++++++++--------------- parser/tree.go | 18 +++++++++--------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index f598807..74e7753 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -78,23 +78,21 @@ func (declaration *Declaration) ToString () (output string) { return } -func (attribute *ObjectAttribute) ToString ( +func (attributes *ObjectAttributes) ToString ( indent int, - breakLine bool, ) ( output string, ) { - if breakLine { - output += doIndent(indent) + for name, value := range attributes.attributes { + output += doIndent(indent, ".", name, " ") + if value.kind == ArgumentKindObjectAttributes { + output += "\n" + output += value.ToString(indent + 1, true) + } else { + output += value.ToString(0, false) + "\n" + } } - output += ", " + attribute.name - if breakLine { - output += "\n" + attribute.value.ToString(indent + 1, true) - } else { - output += " " + attribute.value.ToString(0, false) - } - return } @@ -142,12 +140,11 @@ func (argument *Argument) ToString (indent int, breakLine bool) (output string) indent, breakLine)) - case ArgumentKindObjectAttribute: + case ArgumentKindObjectAttributes: + // this should only appear in contexts where breakLine is true output += doIndent ( indent, - argument.value.(*ObjectAttribute).ToString ( - indent, - breakLine)) + argument.value.(*ObjectAttributes).ToString (indent)) case ArgumentKindIdentifier: output += doIndent ( diff --git a/parser/tree.go b/parser/tree.go index 7919bde..76b7187 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -60,12 +60,11 @@ type Declaration struct { what Type } -// ObjectAttribute represents a notation to initialize object attributes. It -// contains a name, and the value that the attribute should be initialized to. -type ObjectAttribute struct { - location file.Location - name string - value Argument +// ObjectAttributes represents a list of object member initialization +// attributes. +type ObjectAttributes struct { + location file.Location + attributes map[string] Argument } // Phrase represents a function call or operator. In ARF they are the same @@ -93,8 +92,9 @@ const ( // {name 23} ArgumentKindSubscript - // , name value - ArgumentKindObjectAttribute + // .name value + // but like, a lot of them + ArgumentKindObjectAttributes // name.name // name.name.name @@ -103,7 +103,7 @@ const ( // name:Type // name:{Type} - // name:{Type ...} + // name:{Type ..} // name:{Type 23} // etc... ArgumentKindDeclaration From bd456b72e915a653993b43aacc5b42acbd0c6488 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 00:13:14 -0400 Subject: [PATCH 30/42] Argument.ToString can now recover on nil interface value --- parser/tree-tostring.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index 74e7753..f579550 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -131,6 +131,11 @@ func (phrase *Phrase) ToString (indent int, breakLine bool) (output string) { func (argument *Argument) ToString (indent int, breakLine bool) (output string) { if !breakLine { indent = 0 } + if argument.value == nil { + output += "NIL ARGUMENT" + if breakLine { output += "\n" } + return + } switch argument.kind { case ArgumentKindPhrase: From bd42c95de0212b24aab98a5df61fff8f028c2f27 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 00:14:55 -0400 Subject: [PATCH 31/42] Parser can now sort of parse object member initialization values --- parser/data.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/parser/data.go b/parser/data.go index 918e0d2..c404916 100644 --- a/parser/data.go +++ b/parser/data.go @@ -65,8 +65,20 @@ func (parser *ParsingOperation) parseInitializationValues ( err = parser.nextToken() if err != nil { return } + // TODO: make data sections have only one value argument and create a + // similar data structure to ObjectAttributes for arrays if parser.token.Is(lexer.TokenKindDot) { - // TODO: parse as object initialization + var attributes ObjectAttributes + attributes, err = parser.parseObjectInitializationValues ( + baseIndent + 1) + + values = []Argument { + Argument { + location: attributes.location, + value: &attributes, + kind: ArgumentKindObjectAttributes, + }, + } } else { values, err = parser.parseArrayInitializationValues ( baseIndent + 1) @@ -75,6 +87,56 @@ func (parser *ParsingOperation) parseInitializationValues ( return } +// parseObjectInitializationValues parses a list of object initialization +// values until the indentation level is not equal to indent. +func (parser *ParsingOperation) parseObjectInitializationValues ( + indent int, +) ( + values ObjectAttributes, + err error, +) { + values.attributes = make(map[string] Argument) + values.location = parser.token.Location() + + for { + // get attribute name and value + err = parser.nextToken(lexer.TokenKindName) + if err != nil { return } + name := parser.token.Value().(string) + + _, exists := values.attributes[name] + if exists { + err = parser.token.NewError ( + "duplicate member \"" + name + "\" in object " + + "member initialization", + file.ErrorKindError) + return + } + + err = parser.nextToken() + if err != nil { return } + var value Argument + value, err = parser.parseArgument() + + // store in object + values.attributes[name] = value + + // go onto the next line + err = parser.expect(lexer.TokenKindNewline) + if err != nil { return } + err = parser.nextToken() + if err != nil { return } + if !parser.token.Is(lexer.TokenKindIndent) { break } + // TODO: if indent is greater, recurse instead + if parser.token.Value().(int) != indent { break } + + // the next line must start with a dot + err = parser.nextToken(lexer.TokenKindDot) + if err != nil { return } + } + return +} + // parseArrayInitializationValues parses a list of array initialization values // until the indentation lexel is not equal to indent. func (parser *ParsingOperation) parseArrayInitializationValues ( @@ -85,7 +147,6 @@ func (parser *ParsingOperation) parseArrayInitializationValues ( ) { for { if parser.token.Is(lexer.TokenKindNewline) { - println("line break") err = parser.nextToken() if err != nil { return } @@ -94,9 +155,7 @@ func (parser *ParsingOperation) parseArrayInitializationValues ( err = parser.nextToken() if err != nil { return } } - if err != nil { return } - println(parser.token.Describe()) - + var argument Argument argument, err = parser.parseArgument() if err != nil { return } From 0dd93683938d3e8cea47f60b4e6257ed1a409e0a Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 00:49:49 -0400 Subject: [PATCH 32/42] Reworked data section so it stores single initialization argument --- parser/data.go | 44 +++++++++++++++++------------------- parser/tree-tostring.go | 50 +++++++++++++++++++++++++++-------------- parser/tree.go | 18 +++++++++++---- 3 files changed, 68 insertions(+), 44 deletions(-) diff --git a/parser/data.go b/parser/data.go index c404916..995e673 100644 --- a/parser/data.go +++ b/parser/data.go @@ -36,9 +36,8 @@ func (parser *ParsingOperation) parseDataSection () ( section.value, err = parser.parseInitializationValues(0) if err != nil { return } } else { - var argument Argument - argument, err = parser.parseArgument() - section.value = append(section.value, argument) + section.value, err = parser.parseArgument() + if err != nil { return } err = parser.expect(lexer.TokenKindNewline) if err != nil { return } @@ -55,8 +54,8 @@ func (parser *ParsingOperation) parseDataSection () ( func (parser *ParsingOperation) parseInitializationValues ( baseIndent int, ) ( - values []Argument, - err error, + value Argument, + err error, ) { // check if line is indented one more than baseIndent if !parser.token.Is(lexer.TokenKindIndent) { return } @@ -65,23 +64,20 @@ func (parser *ParsingOperation) parseInitializationValues ( err = parser.nextToken() if err != nil { return } - // TODO: make data sections have only one value argument and create a - // similar data structure to ObjectAttributes for arrays - if parser.token.Is(lexer.TokenKindDot) { - var attributes ObjectAttributes - attributes, err = parser.parseObjectInitializationValues ( - baseIndent + 1) + value.location = parser.token.Location() - values = []Argument { - Argument { - location: attributes.location, - value: &attributes, - kind: ArgumentKindObjectAttributes, - }, - } - } else { - values, err = parser.parseArrayInitializationValues ( + if parser.token.Is(lexer.TokenKindDot) { + var values ObjectInitializationValues + values, err = parser.parseObjectInitializationValues ( baseIndent + 1) + value.kind = ArgumentKindObjectInitializationValues + value.value = &values + } else { + var values ArrayInitializationValues + value.value, err = parser.parseArrayInitializationValues ( + baseIndent + 1) + value.kind = ArgumentKindArrayInitializationValues + value.value = &values } return @@ -92,7 +88,7 @@ func (parser *ParsingOperation) parseInitializationValues ( func (parser *ParsingOperation) parseObjectInitializationValues ( indent int, ) ( - values ObjectAttributes, + values ObjectInitializationValues, err error, ) { values.attributes = make(map[string] Argument) @@ -142,9 +138,11 @@ func (parser *ParsingOperation) parseObjectInitializationValues ( func (parser *ParsingOperation) parseArrayInitializationValues ( indent int, ) ( - values []Argument, + values ArrayInitializationValues, err error, ) { + values.location = parser.token.Location() + for { if parser.token.Is(lexer.TokenKindNewline) { err = parser.nextToken() @@ -159,7 +157,7 @@ func (parser *ParsingOperation) parseArrayInitializationValues ( var argument Argument argument, err = parser.parseArgument() if err != nil { return } - values = append(values, argument) + values.values = append(values.values, argument) } return diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index f579550..6970df7 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -78,14 +78,14 @@ func (declaration *Declaration) ToString () (output string) { return } -func (attributes *ObjectAttributes) ToString ( - indent int, +func (attributes *ObjectInitializationValues) ToString ( + indent int, ) ( output string, ) { for name, value := range attributes.attributes { output += doIndent(indent, ".", name, " ") - if value.kind == ArgumentKindObjectAttributes { + if value.kind == ArgumentKindObjectInitializationValues { output += "\n" output += value.ToString(indent + 1, true) } else { @@ -96,6 +96,18 @@ func (attributes *ObjectAttributes) ToString ( return } +func (values *ArrayInitializationValues) ToString ( + indent int, +) ( + output string, +) { + for _, value := range values.values { + output += value.ToString(indent, true) + } + + return +} + func (phrase *Phrase) ToString (indent int, breakLine bool) (output string) { if breakLine { output += doIndent ( @@ -139,17 +151,19 @@ func (argument *Argument) ToString (indent int, breakLine bool) (output string) switch argument.kind { case ArgumentKindPhrase: - output += doIndent ( - indent, - argument.value.(*Phrase).ToString ( + output += argument.value.(*Phrase).ToString ( indent, - breakLine)) + breakLine) - case ArgumentKindObjectAttributes: + case ArgumentKindObjectInitializationValues: // this should only appear in contexts where breakLine is true - output += doIndent ( - indent, - argument.value.(*ObjectAttributes).ToString (indent)) + output += argument.value.(*ObjectInitializationValues). + ToString (indent) + + case ArgumentKindArrayInitializationValues: + // this should only appear in contexts where breakLine is true + output += argument.value.(*ArrayInitializationValues). + ToString (indent) case ArgumentKindIdentifier: output += doIndent ( @@ -193,16 +207,18 @@ func (section *DataSection) ToString (indent int) (output string) { section.name, ":", section.what.ToString()) - if len(section.value) == 0 { + isComplexInitialization := + section.value.kind == ArgumentKindObjectInitializationValues || + section.value.kind == ArgumentKindArrayInitializationValues + + if section.value.value == nil { output += "\n" - } else if len(section.value) == 1 { - output += " " + section.value[0].ToString(0, false) + } else if isComplexInitialization { output += "\n" + output += section.value.ToString(indent + 1, true) } else { + output += " " + section.value.ToString(0, false) output += "\n" - for _, argument := range(section.value) { - output += argument.ToString(indent + 1, true) - } } return } diff --git a/parser/tree.go b/parser/tree.go index 76b7187..88ffabe 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -60,13 +60,20 @@ type Declaration struct { what Type } -// ObjectAttributes represents a list of object member initialization +// ObjectInitializationValues represents a list of object member initialization // attributes. -type ObjectAttributes struct { +type ObjectInitializationValues struct { location file.Location attributes map[string] Argument } +// ArrayInitializationValues represents a list of attributes initializing an +// array. +type ArrayInitializationValues struct { + location file.Location + values []Argument +} + // Phrase represents a function call or operator. In ARF they are the same // syntactical concept. type Phrase struct { @@ -94,7 +101,10 @@ const ( // .name value // but like, a lot of them - ArgumentKindObjectAttributes + ArgumentKindObjectInitializationValues + + // value value... + ArgumentKindArrayInitializationValues // name.name // name.name.name @@ -144,6 +154,6 @@ type DataSection struct { name string what Type - value []Argument + value Argument permission types.Permission } From 9ca1be22040d95031fb4954461612ca45dfb839d Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 00:55:53 -0400 Subject: [PATCH 33/42] Fixed array initialization value parsing --- parser/data.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/parser/data.go b/parser/data.go index 995e673..69e7fa4 100644 --- a/parser/data.go +++ b/parser/data.go @@ -54,8 +54,8 @@ func (parser *ParsingOperation) parseDataSection () ( func (parser *ParsingOperation) parseInitializationValues ( baseIndent int, ) ( - value Argument, - err error, + initializationArgument Argument, + err error, ) { // check if line is indented one more than baseIndent if !parser.token.Is(lexer.TokenKindIndent) { return } @@ -64,20 +64,20 @@ func (parser *ParsingOperation) parseInitializationValues ( err = parser.nextToken() if err != nil { return } - value.location = parser.token.Location() + initializationArgument.location = parser.token.Location() if parser.token.Is(lexer.TokenKindDot) { - var values ObjectInitializationValues - values, err = parser.parseObjectInitializationValues ( + var initializationValues ObjectInitializationValues + initializationValues, err = parser.parseObjectInitializationValues ( baseIndent + 1) - value.kind = ArgumentKindObjectInitializationValues - value.value = &values + initializationArgument.kind = ArgumentKindObjectInitializationValues + initializationArgument.value = &initializationValues } else { - var values ArrayInitializationValues - value.value, err = parser.parseArrayInitializationValues ( + var initializationValues ArrayInitializationValues + initializationValues, err = parser.parseArrayInitializationValues ( baseIndent + 1) - value.kind = ArgumentKindArrayInitializationValues - value.value = &values + initializationArgument.kind = ArgumentKindArrayInitializationValues + initializationArgument.value = &initializationValues } return From 0ad1c0b2f4c23c880464d3eb784d998e02f01ae6 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 01:04:52 -0400 Subject: [PATCH 34/42] Fixed extraneous newlines after complex initialization values --- parser/tree-tostring.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index 6970df7..e9d5b0e 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -158,35 +158,40 @@ func (argument *Argument) ToString (indent int, breakLine bool) (output string) case ArgumentKindObjectInitializationValues: // this should only appear in contexts where breakLine is true output += argument.value.(*ObjectInitializationValues). - ToString (indent) + ToString(indent) case ArgumentKindArrayInitializationValues: // this should only appear in contexts where breakLine is true output += argument.value.(*ArrayInitializationValues). - ToString (indent) + ToString(indent) case ArgumentKindIdentifier: output += doIndent ( indent, argument.value.(*Identifier).ToString()) + if breakLine { output += "\n" } case ArgumentKindDeclaration: output += doIndent ( indent, argument.value.(*Declaration).ToString()) + if breakLine { output += "\n" } case ArgumentKindInt, ArgumentKindUInt, ArgumentKindFloat: output += doIndent(indent, fmt.Sprint(argument.value)) + if breakLine { output += "\n" } case ArgumentKindString: output += doIndent ( indent, "\"" + argument.value.(string) + "\"") + if breakLine { output += "\n" } case ArgumentKindRune: output += doIndent ( indent, "'" + string(argument.value.(rune)) + "'") + if breakLine { output += "\n" } case ArgumentKindOperator: // TODO @@ -195,7 +200,6 @@ func (argument *Argument) ToString (indent int, breakLine bool) (output string) // phrase command. } - if breakLine { output += "\n" } return } From 7bb6582e0149e263f94df8407a2b34bb5a8a616c Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 01:07:12 -0400 Subject: [PATCH 35/42] Added default nil argument kind --- parser/tree-tostring.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index e9d5b0e..f3253ae 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -143,8 +143,8 @@ func (phrase *Phrase) ToString (indent int, breakLine bool) (output string) { func (argument *Argument) ToString (indent int, breakLine bool) (output string) { if !breakLine { indent = 0 } - if argument.value == nil { - output += "NIL ARGUMENT" + if argument.kind == ArgumentKindNil { + output += "NIL-ARGUMENT" if breakLine { output += "\n" } return } From 8c03aa880b7aac624498f75814fb1ba46da28623 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 11:30:17 -0400 Subject: [PATCH 36/42] Reworked array initialization value parsing --- parser/data.go | 183 ++++++++++++++++++++++++++++++------------------- parser/tree.go | 4 +- 2 files changed, 117 insertions(+), 70 deletions(-) diff --git a/parser/data.go b/parser/data.go index 69e7fa4..92f7dab 100644 --- a/parser/data.go +++ b/parser/data.go @@ -60,104 +60,149 @@ func (parser *ParsingOperation) parseInitializationValues ( // check if line is indented one more than baseIndent if !parser.token.Is(lexer.TokenKindIndent) { return } if parser.token.Value().(int) != baseIndent + 1 { return } - - err = parser.nextToken() - if err != nil { return } initializationArgument.location = parser.token.Location() if parser.token.Is(lexer.TokenKindDot) { + var initializationValues ObjectInitializationValues - initializationValues, err = parser.parseObjectInitializationValues ( - baseIndent + 1) - initializationArgument.kind = ArgumentKindObjectInitializationValues + initializationValues, err = parser.parseObjectInitializationValues() + initializationArgument.kind = ArgumentKindObjectInitializationValues initializationArgument.value = &initializationValues + } else { + var initializationValues ArrayInitializationValues - initializationValues, err = parser.parseArrayInitializationValues ( - baseIndent + 1) - initializationArgument.kind = ArgumentKindArrayInitializationValues + initializationValues, err = parser.parseArrayInitializationValues() + initializationArgument.kind = ArgumentKindArrayInitializationValues initializationArgument.value = &initializationValues + } return } // parseObjectInitializationValues parses a list of object initialization -// values until the indentation level is not equal to indent. -func (parser *ParsingOperation) parseObjectInitializationValues ( - indent int, -) ( +// values until the indentation level drops. +func (parser *ParsingOperation) parseObjectInitializationValues () ( values ObjectInitializationValues, err error, ) { - values.attributes = make(map[string] Argument) - values.location = parser.token.Location() - - for { - // get attribute name and value - err = parser.nextToken(lexer.TokenKindName) - if err != nil { return } - name := parser.token.Value().(string) - - _, exists := values.attributes[name] - if exists { - err = parser.token.NewError ( - "duplicate member \"" + name + "\" in object " + - "member initialization", - file.ErrorKindError) - return - } - - err = parser.nextToken() - if err != nil { return } - var value Argument - value, err = parser.parseArgument() - - // store in object - values.attributes[name] = value - - // go onto the next line - err = parser.expect(lexer.TokenKindNewline) - if err != nil { return } - err = parser.nextToken() - if err != nil { return } - if !parser.token.Is(lexer.TokenKindIndent) { break } - // TODO: if indent is greater, recurse instead - if parser.token.Value().(int) != indent { break } - - // the next line must start with a dot - err = parser.nextToken(lexer.TokenKindDot) - if err != nil { return } - } + // println("PARSING") + // defer println("DONE\n") + // values.attributes = make(map[string] Argument) + // values.location = parser.token.Location() +// + // for { + // // get attribute name and value + // err = parser.nextToken(lexer.TokenKindName) + // if err != nil { return } + // name := parser.token.Value().(string) + // println(" name:", name) + // + // _, exists := values.attributes[name] + // if exists { + // err = parser.token.NewError ( + // "duplicate member \"" + name + "\" in object " + + // "member initialization", + // file.ErrorKindError) + // return + // } + // + // err = parser.nextToken() + // if err != nil { return } +// + // println(" parsing value argument") + // // parse value argument + // var value Argument + // if parser.token.Is(lexer.TokenKindNewline) { + // println(" is newline") + // // if there is none on this line, expect complex + // // initialization below +// + // possibleErrorLocation := parser.token.Location() + // err = parser.nextToken(lexer.TokenKindIndent) + // if err != nil { return } +// + // value, err = parser.parseInitializationValues(indent) + // if err != nil { return } +// + // // TODO: this doesn't seem to produce an error at the + // // correct location. + // if value.value == nil { + // err = possibleErrorLocation.NewError ( + // "empty initialization value", + // file.ErrorKindError) + // return + // } + // + // } else { + // println(" is not newline") + // value, err = parser.parseArgument() + // if err != nil { return } + // err = parser.expect(lexer.TokenKindNewline) + // if err != nil { return } + // } +// + // // store in object + // values.attributes[name] = value +// + // // if indent drops, or does something strange, stop parsing + // err = parser.nextToken() + // if err != nil { return } + // if !parser.token.Is(lexer.TokenKindIndent) { break } + // if parser.token.Value().(int) != indent { break } +// + // // the next line must start with a dot + // err = parser.nextToken(lexer.TokenKindDot) + // if err != nil { return } + // } return } // parseArrayInitializationValues parses a list of array initialization values -// until the indentation lexel is not equal to indent. -func (parser *ParsingOperation) parseArrayInitializationValues ( - indent int, -) ( +// until the indentation lexel drops. +func (parser *ParsingOperation) parseArrayInitializationValues () ( values ArrayInitializationValues, err error, ) { - values.location = parser.token.Location() + baseIndent := 0 + begin := true for { - if parser.token.Is(lexer.TokenKindNewline) { - err = parser.nextToken() - if err != nil { return } - - if !parser.token.Is(lexer.TokenKindIndent) { break } - if parser.token.Value().(int) != indent { break } - err = parser.nextToken() - if err != nil { return } - } + // if there is no indent we can just stop parsing + if !parser.token.Is(lexer.TokenKindIndent) { break} + indent := parser.token.Value().(int) - var argument Argument - argument, err = parser.parseArgument() + if begin == true { + values.location = parser.token.Location() + baseIndent = indent + begin = false + } + + // do not parse any further if the indent has changed + if indent != baseIndent { break } + + // move on to the beginning of the line, which must contain + // arguments + err = parser.nextToken(validArgumentStartTokens...) if err != nil { return } - values.values = append(values.values, argument) + + for { + // stop parsing this line and go on to the next if a + // newline token is encountered + if parser.token.Is(lexer.TokenKindNewline) { + err = parser.nextToken() + if err != nil { return } + break + } + + // otherwise, parse the argument + var argument Argument + argument, err = parser.parseArgument() + if err != nil { return } + values.values = append(values.values, argument) + } } return diff --git a/parser/tree.go b/parser/tree.go index 88ffabe..9ef87e7 100644 --- a/parser/tree.go +++ b/parser/tree.go @@ -88,10 +88,12 @@ type Phrase struct { type ArgumentKind int const ( + ArgumentKindNil ArgumentKind = iota + // [name argument] // [name argument argument] // etc... - ArgumentKindPhrase ArgumentKind = iota + ArgumentKindPhrase = iota // {name} ArgumentKindDereference From 384de58d419db8373310acb7d9ee0954d0623ad6 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 12:39:26 -0400 Subject: [PATCH 37/42] Added previousToken method to parser --- parser/parser.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/parser/parser.go b/parser/parser.go index 3907a53..ac183b6 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -111,3 +111,12 @@ func (parser *ParsingOperation) nextToken (allowed ...lexer.TokenKind) (err erro err = parser.expect(allowed...) return } + +// previousToken goes back one token. If the parser is already at the beginning, +// this does nothing. +func (parser *ParsingOperation) previousToken () { + parser.tokenIndex -- + if parser.tokenIndex < 0 { parser.tokenIndex = 0 } + parser.token = parser.tokens[parser.tokenIndex] + return +} From 31bb36a4f7c63c61d68175070f9bb82074b6ca6a Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 12:53:35 -0400 Subject: [PATCH 38/42] Reworked parsing object initialization values --- parser/data.go | 153 +++++++++++++++++++++++++------------------------ 1 file changed, 79 insertions(+), 74 deletions(-) diff --git a/parser/data.go b/parser/data.go index 92f7dab..ed1b1c0 100644 --- a/parser/data.go +++ b/parser/data.go @@ -62,9 +62,13 @@ func (parser *ParsingOperation) parseInitializationValues ( if parser.token.Value().(int) != baseIndent + 1 { return } initializationArgument.location = parser.token.Location() + + err = parser.nextToken() + if err != nil { return } if parser.token.Is(lexer.TokenKindDot) { + parser.previousToken() var initializationValues ObjectInitializationValues initializationValues, err = parser.parseObjectInitializationValues() initializationArgument.kind = ArgumentKindObjectInitializationValues @@ -72,6 +76,7 @@ func (parser *ParsingOperation) parseInitializationValues ( } else { + parser.previousToken() var initializationValues ArrayInitializationValues initializationValues, err = parser.parseArrayInitializationValues() initializationArgument.kind = ArgumentKindArrayInitializationValues @@ -85,86 +90,84 @@ func (parser *ParsingOperation) parseInitializationValues ( // parseObjectInitializationValues parses a list of object initialization // values until the indentation level drops. func (parser *ParsingOperation) parseObjectInitializationValues () ( - values ObjectInitializationValues, - err error, + initializationValues ObjectInitializationValues, + err error, ) { - // println("PARSING") - // defer println("DONE\n") - // values.attributes = make(map[string] Argument) - // values.location = parser.token.Location() -// - // for { - // // get attribute name and value - // err = parser.nextToken(lexer.TokenKindName) - // if err != nil { return } - // name := parser.token.Value().(string) - // println(" name:", name) - // - // _, exists := values.attributes[name] - // if exists { - // err = parser.token.NewError ( - // "duplicate member \"" + name + "\" in object " + - // "member initialization", - // file.ErrorKindError) - // return - // } - // - // err = parser.nextToken() - // if err != nil { return } -// - // println(" parsing value argument") - // // parse value argument - // var value Argument - // if parser.token.Is(lexer.TokenKindNewline) { - // println(" is newline") - // // if there is none on this line, expect complex - // // initialization below -// - // possibleErrorLocation := parser.token.Location() - // err = parser.nextToken(lexer.TokenKindIndent) + initializationValues.attributes = make(map[string] Argument) + + baseIndent := 0 + begin := true + + for { + println(parser.token.Describe()) + + // if there is no indent we can just stop parsing + if !parser.token.Is(lexer.TokenKindIndent) { break} + indent := parser.token.Value().(int) + + if begin == true { + initializationValues.location = parser.token.Location() + baseIndent = indent + begin = false + } + + // do not parse any further if the indent has changed + if indent != baseIndent { break } + + // move on to the beginning of the line, which must contain + // a member initialization value + err = parser.nextToken(lexer.TokenKindDot) + if err != nil { return } + err = parser.nextToken(lexer.TokenKindName) + if err != nil { return } + name := parser.token.Value().(string) + + // if the member has already been listed, throw an error + _, exists := initializationValues.attributes[name] + if exists { + err = parser.token.NewError ( + "duplicate member \"" + name + "\" in object " + + "member initialization", + file.ErrorKindError) + return + } + + // parse the argument determining the member initialization + // value + err = parser.nextToken() + if err != nil { return } + var value Argument + if parser.token.Is(lexer.TokenKindNewline) { + + // TODO: recurse + err = parser.nextToken(lexer.TokenKindIndent) + if err != nil { return } + // value, err = parser.parseInitializationValues() // if err != nil { return } -// - // value, err = parser.parseInitializationValues(indent) - // if err != nil { return } -// - // // TODO: this doesn't seem to produce an error at the - // // correct location. - // if value.value == nil { - // err = possibleErrorLocation.NewError ( - // "empty initialization value", - // file.ErrorKindError) - // return - // } - // - // } else { - // println(" is not newline") - // value, err = parser.parseArgument() - // if err != nil { return } - // err = parser.expect(lexer.TokenKindNewline) - // if err != nil { return } - // } -// - // // store in object - // values.attributes[name] = value -// - // // if indent drops, or does something strange, stop parsing - // err = parser.nextToken() - // if err != nil { return } - // if !parser.token.Is(lexer.TokenKindIndent) { break } - // if parser.token.Value().(int) != indent { break } -// - // // the next line must start with a dot - // err = parser.nextToken(lexer.TokenKindDot) - // if err != nil { return } - // } + + } else { + + // parse as normal argument + value, err = parser.parseArgument() + if err != nil { return } + err = parser.expect(lexer.TokenKindNewline) + if err != nil { return } + err = parser.nextToken() + if err != nil { return } + } + + // store in object + initializationValues.attributes[name] = value + } + return } // parseArrayInitializationValues parses a list of array initialization values // until the indentation lexel drops. func (parser *ParsingOperation) parseArrayInitializationValues () ( - values ArrayInitializationValues, - err error, + initializationValues ArrayInitializationValues, + err error, ) { baseIndent := 0 begin := true @@ -175,7 +178,7 @@ func (parser *ParsingOperation) parseArrayInitializationValues () ( indent := parser.token.Value().(int) if begin == true { - values.location = parser.token.Location() + initializationValues.location = parser.token.Location() baseIndent = indent begin = false } @@ -201,7 +204,9 @@ func (parser *ParsingOperation) parseArrayInitializationValues () ( var argument Argument argument, err = parser.parseArgument() if err != nil { return } - values.values = append(values.values, argument) + initializationValues.values = append ( + initializationValues.values, + argument) } } From aee90757e3287e2303ff54d2205710d50d0ca45f Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 13:26:18 -0400 Subject: [PATCH 39/42] Object initialization value parsing is now done recursively --- parser/body.go | 2 +- parser/data.go | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/parser/body.go b/parser/body.go index 4f6fae6..459d184 100644 --- a/parser/body.go +++ b/parser/body.go @@ -14,12 +14,12 @@ func (parser *ParsingOperation) parseBody () (err error) { case "data": var section *DataSection section, err = parser.parseDataSection() - if err != nil { return } if parser.tree.dataSections == nil { parser.tree.dataSections = make(map[string] *DataSection) } parser.tree.dataSections[section.name] = section + if err != nil { return } case "type": case "face": case "enum": diff --git a/parser/data.go b/parser/data.go index ed1b1c0..cf32548 100644 --- a/parser/data.go +++ b/parser/data.go @@ -67,7 +67,8 @@ func (parser *ParsingOperation) parseInitializationValues ( if err != nil { return } if parser.token.Is(lexer.TokenKindDot) { - + + // object initialization parser.previousToken() var initializationValues ObjectInitializationValues initializationValues, err = parser.parseObjectInitializationValues() @@ -76,12 +77,12 @@ func (parser *ParsingOperation) parseInitializationValues ( } else { + // array initialization parser.previousToken() var initializationValues ArrayInitializationValues initializationValues, err = parser.parseArrayInitializationValues() initializationArgument.kind = ArgumentKindArrayInitializationValues initializationArgument.value = &initializationValues - } return @@ -93,14 +94,15 @@ func (parser *ParsingOperation) parseObjectInitializationValues () ( initializationValues ObjectInitializationValues, err error, ) { + println("BEGIN") + defer println("END") + initializationValues.attributes = make(map[string] Argument) baseIndent := 0 begin := true for { - println(parser.token.Describe()) - // if there is no indent we can just stop parsing if !parser.token.Is(lexer.TokenKindIndent) { break} indent := parser.token.Value().(int) @@ -114,6 +116,8 @@ func (parser *ParsingOperation) parseObjectInitializationValues () ( // do not parse any further if the indent has changed if indent != baseIndent { break } + println("HIT") + // move on to the beginning of the line, which must contain // a member initialization value err = parser.nextToken(lexer.TokenKindDot) @@ -139,25 +143,26 @@ func (parser *ParsingOperation) parseObjectInitializationValues () ( var value Argument if parser.token.Is(lexer.TokenKindNewline) { - // TODO: recurse + // recurse err = parser.nextToken(lexer.TokenKindIndent) if err != nil { return } - // value, err = parser.parseInitializationValues() - // if err != nil { return } + + value, err = parser.parseInitializationValues(baseIndent) + initializationValues.attributes[name] = value + if err != nil { return } } else { // parse as normal argument value, err = parser.parseArgument() + initializationValues.attributes[name] = value if err != nil { return } + err = parser.expect(lexer.TokenKindNewline) if err != nil { return } err = parser.nextToken() if err != nil { return } } - - // store in object - initializationValues.attributes[name] = value } return From 98fb4e9c660b51ece68d6d61a07f028c769715b6 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 13:50:33 -0400 Subject: [PATCH 40/42] In ToString methods, maps are sorted alphabetically before output This makes the output of ToString methods deterministic, and as such they can be used for testing. --- parser/tree-tostring.go | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/parser/tree-tostring.go b/parser/tree-tostring.go index f3253ae..0c2ed33 100644 --- a/parser/tree-tostring.go +++ b/parser/tree-tostring.go @@ -1,6 +1,7 @@ package parser import "fmt" +import "sort" func doIndent (indent int, input ...string) (output string) { for index := 0; index < indent; index ++ { @@ -12,6 +13,22 @@ func doIndent (indent int, input ...string) (output string) { return } +func sortMapKeysAlphabetically[KEY_TYPE any] ( + unsortedMap map[string] KEY_TYPE, +) ( + sortedKeys []string, +) { + sortedKeys = make([]string, len(unsortedMap)) + index := 0 + for key, _ := range unsortedMap { + sortedKeys[index] = key + index ++ + } + sort.Strings(sortedKeys) + + return +} + func (tree *SyntaxTree) ToString (indent int) (output string) { output += doIndent(indent, ":arf\n") @@ -29,8 +46,9 @@ func (tree *SyntaxTree) ToString (indent int) (output string) { output += doIndent(indent, "---\n") - for _, require := range tree.dataSections { - output += require.ToString(indent) + dataSectionKeys := sortMapKeysAlphabetically(tree.dataSections) + for _, name := range dataSectionKeys { + output += tree.dataSections[name].ToString(indent) } return } @@ -83,7 +101,9 @@ func (attributes *ObjectInitializationValues) ToString ( ) ( output string, ) { - for name, value := range attributes.attributes { + for _, name := range sortMapKeysAlphabetically(attributes.attributes) { + value := attributes.attributes[name] + output += doIndent(indent, ".", name, " ") if value.kind == ArgumentKindObjectInitializationValues { output += "\n" From e5b92009f0575d2494376c868c10b6ecf3874bc1 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 13:53:08 -0400 Subject: [PATCH 41/42] Re-arranged data parsing test case to be alphabetical --- parser/parser_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 46c35e1..6193605 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -43,11 +43,7 @@ func TestData (test *testing.T) { `:arf --- data wr integer:Int 3202 -data wr mutInteger:Int:mut 3202 -data wr integerPointer:{Int} -data wr mutIntegerPointer:{Int}:mut data wr integerArray16:{Int 16} -data wr integerArrayVariable:{Int ..} data wr integerArrayInitialized:{Int 16} 3948 293 @@ -60,18 +56,22 @@ data wr integerArrayInitialized:{Int 16} 0 4785 92 +data wr integerArrayVariable:{Int ..} +data wr integerPointer:{Int} data wr integerPointerInit:{Int} [& integer] +data wr mutInteger:Int:mut 3202 +data wr mutIntegerPointer:{Int}:mut data wr mutIntegerPointerInit:{Int}:mut [& integer] -data wr object:Obj - .this 324 - .that 2139 data wr nestedObject:Obj - .this - .bird0 324 - .bird1 "hello world" .that .bird2 123.8439 .bird3 9328.21348239 + .this + .bird0 324 + .bird1 "hello world" +data wr object:Obj + .this 324 + .that 2139 `, test) } From d78f150336bae983da18594ad01c183b603b9601 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 17 Aug 2022 14:16:54 -0400 Subject: [PATCH 42/42] Removed data test case initializing pointers with phrases Phrase parsing is out of scope for this branch. It will be implemented at the same time as function parsing. --- parser/parser_test.go | 4 +--- tests/parser/data/main.arf | 7 ++++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/parser/parser_test.go b/parser/parser_test.go index 6193605..18e2ee0 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -58,10 +58,8 @@ data wr integerArrayInitialized:{Int 16} 92 data wr integerArrayVariable:{Int ..} data wr integerPointer:{Int} -data wr integerPointerInit:{Int} [& integer] data wr mutInteger:Int:mut 3202 data wr mutIntegerPointer:{Int}:mut -data wr mutIntegerPointerInit:{Int}:mut [& integer] data wr nestedObject:Obj .that .bird2 123.8439 @@ -70,8 +68,8 @@ data wr nestedObject:Obj .bird0 324 .bird1 "hello world" data wr object:Obj - .this 324 .that 2139 + .this 324 `, test) } diff --git a/tests/parser/data/main.arf b/tests/parser/data/main.arf index 9d6e347..8efeb61 100644 --- a/tests/parser/data/main.arf +++ b/tests/parser/data/main.arf @@ -16,10 +16,11 @@ data wr integerArrayVariable:{Int ..} data wr integerArrayInitialized:{Int 16} 3948 293 293049 948 912 340 0 2304 0 4785 92 - -data wr integerPointerInit:{Int} #[& integer] -data wr mutIntegerPointerInit:{Int}:mut #[& integer] +# TODO: reinstate these two after phrase parsing is implemented +# data wr integerPointerInit:{Int} [& integer] + +# data wr mutIntegerPointerInit:{Int}:mut [& integer] data wr object:Obj .this 324