Implement new tokens

I think this is all of them. The test will tell.

lexer/lexer.go

@@ -253,14 +253,26 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 		lexer.addToken(token)
 	case '%':
 		token := lexer.newToken()
-		token.kind = TokenKindPercent
-		lexer.addToken(token)
 		err = lexer.nextRune()
+		if err != nil { return }
+		token.kind = TokenKindPercent
+		if lexer.char == '=' {
+			token.kind = TokenKindPercentAssignment
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
+		}
+		lexer.addToken(token)
 	case '~':
 		token := lexer.newToken()
-		token.kind = TokenKindTilde
-		lexer.addToken(token)
 		err = lexer.nextRune()
+		if err != nil { return }
+		token.kind = TokenKindTilde
+		if lexer.char == '=' {
+			token.kind = TokenKindTildeAssignment
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
+		}
+		lexer.addToken(token)
 	case '=':
 		token := lexer.newToken()
 		token.kind = TokenKindEqualTo
@@ -275,6 +287,11 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 			token.kind = TokenKindLShift
 			err = lexer.nextRune()
 			token.location.SetWidth(2)
+			if lexer.char == '=' {
+				token.kind = TokenKindLShiftAssignment
+				err = lexer.nextRune()
+				token.location.SetWidth(2)
+			}
 		} else if lexer.char == '=' {
 			token.kind = TokenKindLessThanEqualTo
 			err = lexer.nextRune()
@@ -290,6 +307,11 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 			token.kind = TokenKindRShift
 			err = lexer.nextRune()
 			token.location.SetWidth(2)
+			if lexer.char == '=' {
+				token.kind = TokenKindRShiftAssignment
+				err = lexer.nextRune()
+				token.location.SetWidth(2)
+			}
 		} else if lexer.char == '=' {
 			token.kind = TokenKindGreaterThanEqualTo
 			err = lexer.nextRune()
@@ -305,6 +327,10 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 			token.kind = TokenKindLogicalOr
 			err = lexer.nextRune()
 			token.location.SetWidth(2)
+		} else if lexer.char == '=' {
+			token.kind = TokenKindBinaryOrAssignment
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
 		}
 		lexer.addToken(token)
 	case '&':
@@ -316,6 +342,10 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 			token.kind = TokenKindLogicalAnd
 			err = lexer.nextRune()
 			token.location.SetWidth(2)
+		} else if lexer.char == '=' {
+			token.kind = TokenKindBinaryAndAssignment
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
 		}
 		lexer.addToken(token)
 	default:

lexer/lexer_test.go

@@ -152,19 +152,27 @@ func TestTokenizeAll (test *testing.T) {
 		quickToken(1, TokenKindAt, nil),
 		quickToken(1, TokenKindExclamation, nil),
 		quickToken(1, TokenKindPercent, nil),
+		quickToken(2, TokenKindPercentAssignment, nil),
 		quickToken(1, TokenKindTilde, nil),
+		quickToken(2, TokenKindTildeAssignment, nil),
 		quickToken(1, TokenKindEqualTo, nil),
 		quickToken(2, TokenKindNotEqualTo, nil),
 		quickToken(1, TokenKindLessThan, nil),
 		quickToken(2, TokenKindLessThanEqualTo, nil),
 		quickToken(2, TokenKindLShift, nil),
+		quickToken(3, TokenKindLShiftAssignment, nil),
 		quickToken(1, TokenKindGreaterThan, nil),
 		quickToken(2, TokenKindGreaterThanEqualTo, nil),
 		quickToken(2, TokenKindRShift, nil),
+		quickToken(3, TokenKindRShiftAssignment, nil),
 		quickToken(1, TokenKindBinaryOr, nil),
+		quickToken(2, TokenKindBinaryOrAssignment, nil),
 		quickToken(2, TokenKindLogicalOr, nil),
 		quickToken(1, TokenKindBinaryAnd, nil),
+		quickToken(2, TokenKindBinaryAndAssignment, nil),
 		quickToken(2, TokenKindLogicalAnd, nil),
+		quickToken(1, TokenKindBinaryXor, nil),
+		quickToken(2, TokenKindBinaryXorAssignment, nil),
 		quickToken(1, TokenKindNewline, nil),
 	)
 }

lexer/token.go

@@ -43,20 +43,28 @@ const (
         TokenKindAt
         TokenKindExclamation
         TokenKindPercent
+        TokenKindPercentAssignment
         TokenKindTilde
+        TokenKindTildeAssignment
 
         TokenKindEqualTo
         TokenKindNotEqualTo
         TokenKindLessThanEqualTo
         TokenKindLessThan
         TokenKindLShift
+        TokenKindLShiftAssignment
         TokenKindGreaterThan
         TokenKindGreaterThanEqualTo
         TokenKindRShift
+        TokenKindRShiftAssignment
         TokenKindBinaryOr
+        TokenKindBinaryOrAssignment
         TokenKindLogicalOr
         TokenKindBinaryAnd
+        TokenKindBinaryAndAssignment
         TokenKindLogicalAnd
+        TokenKindBinaryXor
+        TokenKindBinaryXorAssignment
 )
 
 // Token represents a single token. It holds its location in the file, as well
@@ -175,8 +183,12 @@ func (tokenKind TokenKind) Describe () (description string) {
 		description = "Exclamation"
 	case TokenKindPercent:
 		description = "Percent"
+	case TokenKindPercentAssignment:
+		description = "PercentAssignment"
 	case TokenKindTilde:
 		description = "Tilde"
+	case TokenKindTildeAssignment:
+		description = "TildeAssignment"
 	case TokenKindEqualTo:
 		description = "EqualTo"
 	case TokenKindNotEqualTo:
@@ -187,20 +199,32 @@ func (tokenKind TokenKind) Describe () (description string) {
 		description = "LessThanEqualTo"
 	case TokenKindLShift:
 		description = "LShift"
+	case TokenKindLShiftAssignment:
+		description = "LShiftAssignment"
 	case TokenKindGreaterThan:
 		description = "GreaterThan"
 	case TokenKindGreaterThanEqualTo:
 		description = "GreaterThanEqualTo"
 	case TokenKindRShift:
 		description = "RShift"
+	case TokenKindRShiftAssignment:
+		description = "RShiftAssignment"
 	case TokenKindBinaryOr:
 		description = "BinaryOr"
+	case TokenKindBinaryOrAssignment:
+		description = "BinaryOrAssignment"
 	case TokenKindLogicalOr:
 		description = "LogicalOr"
 	case TokenKindBinaryAnd:
 		description = "BinaryAnd"
+	case TokenKindBinaryAndAssignment:
+		description = "BinaryAndAssignment"
 	case TokenKindLogicalAnd:
 		description = "LogicalAnd"
+	case TokenKindBinaryXor:
+		description = "BinaryXor"
+	case TokenKindBinaryXorAssignment:
+		description = "BinaryXorAssignment"
 	}
 
 	return

parser/body.go

@@ -57,6 +57,14 @@ func (parser *ParsingOperation) parseBody () (err error) {
 			parser.tree.enumSections[section.name] = section
 			if err != nil { return }
 		case "func":
+			var section *FuncSection
+			section, err = parser.parseFuncSection()
+			if parser.tree.funcSections == nil {
+				parser.tree.funcSections =
+					make(map[string] *FuncSection)
+			}
+			parser.tree.funcSections[section.name] = section
+			if err != nil { return }
 		default:
 			err = parser.token.NewError (
 				"unknown section type \"" + sectionType + "\"",

parser/func.go

@@ -0,0 +1,30 @@
+package parser
+
+import "git.tebibyte.media/sashakoshka/arf/types"
+import "git.tebibyte.media/sashakoshka/arf/lexer"
+// import "git.tebibyte.media/sashakoshka/arf/infoerr"
+
+// parseFunc parses a function section.
+func (parser *ParsingOperation) parseFuncSection () (
+	section *FuncSection,
+	err     error,
+) {
+	err = parser.expect(lexer.TokenKindName)
+	if err != nil { return }
+	
+	section = &FuncSection { location: parser.token.Location() }
+
+	err = parser.nextToken(lexer.TokenKindPermission)
+	if err != nil { return }
+	section.permission = parser.token.Value().(types.Permission)
+
+	err = parser.nextToken(lexer.TokenKindName)
+	if err != nil { return }
+	section.name = parser.token.Value().(string)
+
+	err = parser.nextToken(lexer.TokenKindNewline)
+	if err != nil { return }
+	
+	return
+}
+

parser/func_test.go

@@ -0,0 +1,111 @@
+package parser
+
+import "testing"
+
+func TestFunc (test *testing.T) {
+	checkTree ("../tests/parser/func",
+`:arf
+---
+func ro aBasicExternal
+	> someInput:Int:mut
+	< someOutput:Int 4
+	---
+	external
+func ro bMethod
+	@ bird:{Bird}
+	> someInput:Int:mut
+	< someOutput:Int 4
+	---
+	external
+func ro cBasicPhrases
+	---
+	[fn 329 983 09]
+	[fn 329 983 09]
+	[fn 329 983 091]
+	[fn [gn 329 983 091] 123]
+func ro dArgumentTypes
+	---
+	[bird tree butterfly.wing "hello world" grass:{Int:mut 8}]
+func ro eMath
+	[> x:Int]
+	[> y:Int]
+	[< z:Int]
+	[---]
+	[++ x]
+	[-- y]
+	[set z [+ [* 0392 00] 98 x [/ 9832 y] 930]]
+	[! true]
+	[~ 0b01]
+	[% 873 32]
+	[=  5 5]
+	[!= 4 4]
+	[<=  4 98]
+	[<   4 98]
+	[<<  0x0F 4]
+	[>=  98 4]
+	[>   98 4]
+	[>>  0xF0 4]
+	[|  0b01 0b10]
+	[& 0b110 0b011]
+	[&& true true]
+	[|| true false]
+func ro fReturnDirection
+	< err:Error
+	---
+	[someFunc 498 2980 90] -> thing:Int err
+	[otherFunc] -> thing err:Error
+	[fn 329 983 091] -> thing:Int err
+func ro gControlFlow
+	---
+	[if condition]
+		[something]
+	[if condition]
+		[something]
+	[elseif]
+		[otherThing]
+	[else]
+		[finalThing]
+	[while [< x 432]]
+		[something]
+	[switch value]
+	[: 324]
+		[something]
+	[: 93284]
+		otherThing
+	[: 9128 34738 7328]
+		multipleCases
+	[:]
+		[defaultThing]
+	[for index:Size element:Int someArray]
+		[something]
+		[someNextThing]
+		[justMakingSureBlockParsingWorks]
+	[if condition]
+		[if condition]
+			[nestedThing]
+		[else]
+			[otherThing]
+	[else]
+		[if condition]
+			[nestedThing]
+		[else]
+			[otherThing]
+func hSetPhrase
+	---
+	[set x:Int 3]
+	[set y:{Int} [. x]]
+	[set z:{Int 8}]
+		398
+		9
+		2309
+		983
+		-2387
+		478
+		555
+		123
+	[set bird:Bird]
+		.that
+			.whenYou 99999
+		.this 324
+`, test)
+}

parser/tree.go

@@ -16,6 +16,7 @@ type SyntaxTree struct {
 	enumSections map[string] *EnumSection
 	faceSections map[string] *FaceSection
 	dataSections map[string] *DataSection
+	funcSections map[string] *FuncSection
 }
 
 // Identifier represents a chain of arguments separated by a dot.
@@ -229,3 +230,20 @@ type FaceSection struct {
 	permission types.Permission
 	behaviors  map[string] FaceBehavior
 }
+
+// Block represents a scoped/indented block of code.
+// TODO: blocks will not directly nest. nested blocks will be stored as a part
+// of certain control flow statements.
+type Block []Phrase
+
+// FuncSection represents a function section.
+type FuncSection struct {
+	location   file.Location
+	name       string
+	permission types.Permission
+	
+	receiver *Declaration
+	inputs   []Declaration
+	outputs  []Declaration
+	root     *Block
+}

tests/lexer/all.arf

@@ -1,3 +1,3 @@
 :arf
 --- rw -> -349820394 932748397 239485.37520 "hello world!\n" 'E' helloWorld:.,..[]{}
-+ - ++ -- * / @ ! % ~ = != < <= << > >= >> | || & &&
++ - ++ -- * / @ ! % %= ~ ~= = != < <= << <<= > >= >> >>= | |= || & &= && ^ ^=

tests/parser/func/main.arf

@@ -0,0 +1,134 @@
+:arf
+---
+func ro aBasicExternal
+	> someInput:Int:mut
+	< someOutput:Int 4
+	---
+	external
+
+func ro bMethod
+	@ bird:{Bird}
+	> someInput:Int:mut
+	< someOutput:Int 4
+	---
+	external
+
+func ro cBasicPhrases
+	---
+	fn 329 983 09
+	[fn 329 983 09]
+	[fn
+		329
+		983
+		091]
+	fn [gn
+		329 983
+		091] 123
+
+func ro dArgumentTypes
+	---
+	[bird tree butterfly.wing "hello world"
+		grass:{Int:mut 8}]
+
+func ro eMath
+	> x:Int
+	> y:Int
+	< z:Int
+	---
+	++ x
+	-- y
+	set z [+ [* 0392 00] 98 x [/ 9832 y] 930]
+
+	# TODO: need tokens ~=
+	
+	! true
+	~ 0b01
+	# ~= x
+	% 873 32
+
+	=  5 5
+	!= 4 4
+	
+	<=  4 98
+	<   4 98
+	<<  0x0F 4
+	# <<= x 4
+
+	>=  98 4
+	>   98 4
+	>>  0xF0 4
+	# >>= x 4
+
+	|  0b01 0b10
+	# |= x 0b10
+	& 0b110 0b011
+	# &= x 0b011
+
+	&& true true
+	|| true false
+
+func ro fReturnDirection
+	< err:Error
+	---
+	someFunc 498 2980 90 -> thing:Int err
+	otherFunc -> thing err:Error
+	
+	[fn
+		329
+		983
+		091] -> thing:Int err
+
+func ro gControlFlow
+	---
+	if condition
+		something
+	
+	if condition
+		something
+	elseif
+		[otherThing]
+	else
+		finalThing
+	
+	while [< x 432]
+		something
+	
+	switch value
+	: 324
+		something
+	[: 93284]
+		otherThing
+	: 9128 34738 7328
+		multipleCases
+	:
+		[defaultThing]
+	
+	for index:Size element:Int someArray
+		something
+		someNextThing
+		justMakingSureBlockParsingWorks
+
+	[if condition]
+		if condition
+			nestedThing
+		else
+			otherThing
+	else
+		if condition
+			nestedThing
+		else
+			otherThing
+
+func hSetPhrase
+	---
+	set x:Int 3
+	# TODO: this should be the "location of" phrase. update other things to
+	# match.
+	set y:{Int} [. x]
+	set z:{Int 8}
+		398 9 2309 983 -2387
+		478 555 123
+	set bird:Bird
+		.that
+			.whenYou 99999
+		.this 324