Implement new tokens

I think this is all of them. The test will tell.

README.md

@@ -16,7 +16,7 @@ A directory of ARF files is called a module, and modules will compile to object
 files (one per module) using C as an intermediate language (maybe LLVM IR in the
 future).
 
-## Design aspects
+## Design Aspects
 
 These are some design goals that I have followed/am following:
 
@@ -32,7 +32,7 @@ These are some design goals that I have followed/am following:
 - One line at a time - the language's syntax should encourage writing code that
   flows vertically and not horizontally, with minimal nesting
 
-## Planned features
+## Planned Features
 
 - Type definition through inheritence
 - Struct member functions
@@ -49,3 +49,11 @@ These are some design goals that I have followed/am following:
 - [ ] Semantic tree -> C -> object file
 - [ ] Figure out HOW to implement generics
 - [ ] Create a standard library
+
+## Compiler Progress
+
+<img src="assets/heatmap.png" alt="Progress heatmap" width="400">
+
+- Yellow: needs to be completed for the MVP
+- Lime: ongoing progress in this area
+- Green: Already completed

lexer/lexer.go

@@ -242,17 +242,40 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 		err = lexer.nextRune()
 	case '!':
 		token := lexer.newToken()
-		token.kind = TokenKindExclamation
-		lexer.addToken(token)
 		err = lexer.nextRune()
+		if err != nil { return }
+		token.kind = TokenKindExclamation
+		if lexer.char == '=' {
+			token.kind = TokenKindNotEqualTo
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
+		}
+		lexer.addToken(token)
 	case '%':
 		token := lexer.newToken()
-		token.kind = TokenKindPercent
-		lexer.addToken(token)
 		err = lexer.nextRune()
+		if err != nil { return }
+		token.kind = TokenKindPercent
+		if lexer.char == '=' {
+			token.kind = TokenKindPercentAssignment
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
+		}
+		lexer.addToken(token)
 	case '~':
 		token := lexer.newToken()
+		err = lexer.nextRune()
+		if err != nil { return }
 		token.kind = TokenKindTilde
+		if lexer.char == '=' {
+			token.kind = TokenKindTildeAssignment
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
+		}
+		lexer.addToken(token)
+	case '=':
+		token := lexer.newToken()
+		token.kind = TokenKindEqualTo
 		lexer.addToken(token)
 		err = lexer.nextRune()
 	case '<':
@@ -264,6 +287,15 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 			token.kind = TokenKindLShift
 			err = lexer.nextRune()
 			token.location.SetWidth(2)
+			if lexer.char == '=' {
+				token.kind = TokenKindLShiftAssignment
+				err = lexer.nextRune()
+				token.location.SetWidth(2)
+			}
+		} else if lexer.char == '=' {
+			token.kind = TokenKindLessThanEqualTo
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
 		}
 		lexer.addToken(token)
 	case '>':
@@ -275,6 +307,15 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 			token.kind = TokenKindRShift
 			err = lexer.nextRune()
 			token.location.SetWidth(2)
+			if lexer.char == '=' {
+				token.kind = TokenKindRShiftAssignment
+				err = lexer.nextRune()
+				token.location.SetWidth(2)
+			}
+		} else if lexer.char == '=' {
+			token.kind = TokenKindGreaterThanEqualTo
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
 		}
 		lexer.addToken(token)
 	case '|':
@@ -286,6 +327,10 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 			token.kind = TokenKindLogicalOr
 			err = lexer.nextRune()
 			token.location.SetWidth(2)
+		} else if lexer.char == '=' {
+			token.kind = TokenKindBinaryOrAssignment
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
 		}
 		lexer.addToken(token)
 	case '&':
@@ -297,6 +342,10 @@ func (lexer *LexingOperation) tokenizeSymbolBeginning () (err error) {
 			token.kind = TokenKindLogicalAnd
 			err = lexer.nextRune()
 			token.location.SetWidth(2)
+		} else if lexer.char == '=' {
+			token.kind = TokenKindBinaryAndAssignment
+			err = lexer.nextRune()
+			token.location.SetWidth(2)
 		}
 		lexer.addToken(token)
 	default:

lexer/lexer_test.go

@@ -152,15 +152,27 @@ func TestTokenizeAll (test *testing.T) {
 		quickToken(1, TokenKindAt, nil),
 		quickToken(1, TokenKindExclamation, nil),
 		quickToken(1, TokenKindPercent, nil),
+		quickToken(2, TokenKindPercentAssignment, nil),
 		quickToken(1, TokenKindTilde, nil),
+		quickToken(2, TokenKindTildeAssignment, nil),
+		quickToken(1, TokenKindEqualTo, nil),
+		quickToken(2, TokenKindNotEqualTo, nil),
 		quickToken(1, TokenKindLessThan, nil),
+		quickToken(2, TokenKindLessThanEqualTo, nil),
 		quickToken(2, TokenKindLShift, nil),
+		quickToken(3, TokenKindLShiftAssignment, nil),
 		quickToken(1, TokenKindGreaterThan, nil),
+		quickToken(2, TokenKindGreaterThanEqualTo, nil),
 		quickToken(2, TokenKindRShift, nil),
+		quickToken(3, TokenKindRShiftAssignment, nil),
 		quickToken(1, TokenKindBinaryOr, nil),
+		quickToken(2, TokenKindBinaryOrAssignment, nil),
 		quickToken(2, TokenKindLogicalOr, nil),
 		quickToken(1, TokenKindBinaryAnd, nil),
+		quickToken(2, TokenKindBinaryAndAssignment, nil),
 		quickToken(2, TokenKindLogicalAnd, nil),
+		quickToken(1, TokenKindBinaryXor, nil),
+		quickToken(2, TokenKindBinaryXorAssignment, nil),
 		quickToken(1, TokenKindNewline, nil),
 	)
 }

lexer/token.go

@@ -43,18 +43,28 @@ const (
         TokenKindAt
         TokenKindExclamation
         TokenKindPercent
+        TokenKindPercentAssignment
         TokenKindTilde
+        TokenKindTildeAssignment
 
-	// TODO: add equal to, less than or equal to, greater than or equal to,
+        TokenKindEqualTo
-	// not equal to
+        TokenKindNotEqualTo
+        TokenKindLessThanEqualTo
         TokenKindLessThan
         TokenKindLShift
+        TokenKindLShiftAssignment
         TokenKindGreaterThan
+        TokenKindGreaterThanEqualTo
         TokenKindRShift
+        TokenKindRShiftAssignment
         TokenKindBinaryOr
+        TokenKindBinaryOrAssignment
         TokenKindLogicalOr
         TokenKindBinaryAnd
+        TokenKindBinaryAndAssignment
         TokenKindLogicalAnd
+        TokenKindBinaryXor
+        TokenKindBinaryXorAssignment
 )
 
 // Token represents a single token. It holds its location in the file, as well
@@ -173,24 +183,48 @@ func (tokenKind TokenKind) Describe () (description string) {
 		description = "Exclamation"
 	case TokenKindPercent:
 		description = "Percent"
+	case TokenKindPercentAssignment:
+		description = "PercentAssignment"
 	case TokenKindTilde:
 		description = "Tilde"
+	case TokenKindTildeAssignment:
+		description = "TildeAssignment"
+	case TokenKindEqualTo:
+		description = "EqualTo"
+	case TokenKindNotEqualTo:
+		description = "NotEqualTo"
 	case TokenKindLessThan:
 		description = "LessThan"
+	case TokenKindLessThanEqualTo:
+		description = "LessThanEqualTo"
 	case TokenKindLShift:
 		description = "LShift"
+	case TokenKindLShiftAssignment:
+		description = "LShiftAssignment"
 	case TokenKindGreaterThan:
 		description = "GreaterThan"
+	case TokenKindGreaterThanEqualTo:
+		description = "GreaterThanEqualTo"
 	case TokenKindRShift:
 		description = "RShift"
+	case TokenKindRShiftAssignment:
+		description = "RShiftAssignment"
 	case TokenKindBinaryOr:
 		description = "BinaryOr"
+	case TokenKindBinaryOrAssignment:
+		description = "BinaryOrAssignment"
 	case TokenKindLogicalOr:
 		description = "LogicalOr"
 	case TokenKindBinaryAnd:
 		description = "BinaryAnd"
+	case TokenKindBinaryAndAssignment:
+		description = "BinaryAndAssignment"
 	case TokenKindLogicalAnd:
 		description = "LogicalAnd"
+	case TokenKindBinaryXor:
+		description = "BinaryXor"
+	case TokenKindBinaryXorAssignment:
+		description = "BinaryXorAssignment"
 	}
 
 	return

parser/body.go

@@ -57,6 +57,14 @@ func (parser *ParsingOperation) parseBody () (err error) {
 			parser.tree.enumSections[section.name] = section
 			if err != nil { return }
 		case "func":
+			var section *FuncSection
+			section, err = parser.parseFuncSection()
+			if parser.tree.funcSections == nil {
+				parser.tree.funcSections =
+					make(map[string] *FuncSection)
+			}
+			parser.tree.funcSections[section.name] = section
+			if err != nil { return }
 		default:
 			err = parser.token.NewError (
 				"unknown section type \"" + sectionType + "\"",

parser/data.go

@@ -289,8 +289,6 @@ func (parser *ParsingOperation) parseIdentifier () (
 	identifier.location = parser.token.Location()
 
 	for {
-		// TODO: eat up newlines and tabs after the dot, but not before
-		// it.
 		if !parser.token.Is(lexer.TokenKindName) { break }
 
 		identifier.trail = append (
@@ -301,6 +299,18 @@ func (parser *ParsingOperation) parseIdentifier () (
 		if err != nil { return }
 		
 		if !parser.token.Is(lexer.TokenKindDot) { break }
+
+		err = parser.nextToken()
+		if err != nil { return }
+
+		// allow the identifier to continue on to the next line if there
+		// is a line break right after the dot
+		for parser.token.Is(lexer.TokenKindNewline) ||
+			parser.token.Is(lexer.TokenKindIndent) {
+
+			err = parser.nextToken()
+			if err != nil { return }
+		}
 	}
 
 	return

parser/data_test.go

@@ -31,7 +31,7 @@ data ro nestedObject:Obj
 	.this
 		.bird0 324
 		.bird1 "hello world"
-data ro object:Obj
+data ro object:thing.thing.thing.thing
 	.that 2139
 	.this 324
 `, test)

parser/func.go

@@ -0,0 +1,30 @@
+package parser
+
+import "git.tebibyte.media/sashakoshka/arf/types"
+import "git.tebibyte.media/sashakoshka/arf/lexer"
+// import "git.tebibyte.media/sashakoshka/arf/infoerr"
+
+// parseFunc parses a function section.
+func (parser *ParsingOperation) parseFuncSection () (
+	section *FuncSection,
+	err     error,
+) {
+	err = parser.expect(lexer.TokenKindName)
+	if err != nil { return }
+	
+	section = &FuncSection { location: parser.token.Location() }
+
+	err = parser.nextToken(lexer.TokenKindPermission)
+	if err != nil { return }
+	section.permission = parser.token.Value().(types.Permission)
+
+	err = parser.nextToken(lexer.TokenKindName)
+	if err != nil { return }
+	section.name = parser.token.Value().(string)
+
+	err = parser.nextToken(lexer.TokenKindNewline)
+	if err != nil { return }
+	
+	return
+}
+

parser/func_test.go

@@ -0,0 +1,111 @@
+package parser
+
+import "testing"
+
+func TestFunc (test *testing.T) {
+	checkTree ("../tests/parser/func",
+`:arf
+---
+func ro aBasicExternal
+	> someInput:Int:mut
+	< someOutput:Int 4
+	---
+	external
+func ro bMethod
+	@ bird:{Bird}
+	> someInput:Int:mut
+	< someOutput:Int 4
+	---
+	external
+func ro cBasicPhrases
+	---
+	[fn 329 983 09]
+	[fn 329 983 09]
+	[fn 329 983 091]
+	[fn [gn 329 983 091] 123]
+func ro dArgumentTypes
+	---
+	[bird tree butterfly.wing "hello world" grass:{Int:mut 8}]
+func ro eMath
+	[> x:Int]
+	[> y:Int]
+	[< z:Int]
+	[---]
+	[++ x]
+	[-- y]
+	[set z [+ [* 0392 00] 98 x [/ 9832 y] 930]]
+	[! true]
+	[~ 0b01]
+	[% 873 32]
+	[=  5 5]
+	[!= 4 4]
+	[<=  4 98]
+	[<   4 98]
+	[<<  0x0F 4]
+	[>=  98 4]
+	[>   98 4]
+	[>>  0xF0 4]
+	[|  0b01 0b10]
+	[& 0b110 0b011]
+	[&& true true]
+	[|| true false]
+func ro fReturnDirection
+	< err:Error
+	---
+	[someFunc 498 2980 90] -> thing:Int err
+	[otherFunc] -> thing err:Error
+	[fn 329 983 091] -> thing:Int err
+func ro gControlFlow
+	---
+	[if condition]
+		[something]
+	[if condition]
+		[something]
+	[elseif]
+		[otherThing]
+	[else]
+		[finalThing]
+	[while [< x 432]]
+		[something]
+	[switch value]
+	[: 324]
+		[something]
+	[: 93284]
+		otherThing
+	[: 9128 34738 7328]
+		multipleCases
+	[:]
+		[defaultThing]
+	[for index:Size element:Int someArray]
+		[something]
+		[someNextThing]
+		[justMakingSureBlockParsingWorks]
+	[if condition]
+		[if condition]
+			[nestedThing]
+		[else]
+			[otherThing]
+	[else]
+		[if condition]
+			[nestedThing]
+		[else]
+			[otherThing]
+func hSetPhrase
+	---
+	[set x:Int 3]
+	[set y:{Int} [. x]]
+	[set z:{Int 8}]
+		398
+		9
+		2309
+		983
+		-2387
+		478
+		555
+		123
+	[set bird:Bird]
+		.that
+			.whenYou 99999
+		.this 324
+`, test)
+}

parser/objt.go

@@ -13,10 +13,7 @@ func (parser *ParsingOperation) parseObjtSection () (
 	err = parser.expect(lexer.TokenKindName)
 	if err != nil { return }
 	
-	section = &ObjtSection {
+	section = &ObjtSection { location: parser.token.Location() }
-		location: parser.token.Location(),
-		members:  make(map[string] ObjtMember),
-	}
 
 	// get permission
 	err = parser.nextToken(lexer.TokenKindPermission)
@@ -68,7 +65,7 @@ func (parser *ParsingOperation) parseObjtMembers (
 		// add member to object section
 		var member ObjtMember
 		member, err = parser.parseObjtMember()
-		into.members[member.name] = member
+		into.members = append(into.members, member)
 		if err != nil { return }
 	}
 }

parser/objt_test.go

@@ -13,17 +13,17 @@ objt ro BitFields:Obj
 	ro that:Int & 1
 	ro this:Int & 24 298
 objt ro ComplexInit:Obj
-	ro basic:Int 87
+	ro whatever:{Int 3}
+		230984
+		849
+		394580
 	ro complex0:Bird
 		.that 98
 		.this 2
 	ro complex1:Bird
 		.that 98902
 		.this 235
-	ro whatever:{Int 3}
+	ro basic:Int 87
-		230984
-		849
-		394580
 objt ro Init:Obj
 	ro that:String "hello world"
 	ro this:Int 23

parser/tree.go

@@ -16,6 +16,7 @@ type SyntaxTree struct {
 	enumSections map[string] *EnumSection
 	faceSections map[string] *FaceSection
 	dataSections map[string] *DataSection
+	funcSections map[string] *FuncSection
 }
 
 // Identifier represents a chain of arguments separated by a dot.
@@ -229,3 +230,20 @@ type FaceSection struct {
 	permission types.Permission
 	behaviors  map[string] FaceBehavior
 }
+
+// Block represents a scoped/indented block of code.
+// TODO: blocks will not directly nest. nested blocks will be stored as a part
+// of certain control flow statements.
+type Block []Phrase
+
+// FuncSection represents a function section.
+type FuncSection struct {
+	location   file.Location
+	name       string
+	permission types.Permission
+	
+	receiver *Declaration
+	inputs   []Declaration
+	outputs  []Declaration
+	root     *Block
+}

tests/lexer/all.arf

@@ -1,3 +1,3 @@
 :arf
 --- rw -> -349820394 932748397 239485.37520 "hello world!\n" 'E' helloWorld:.,..[]{}
-+ - ++ -- * / @ ! % ~ < << > >> | || & &&
++ - ++ -- * / @ ! % %= ~ ~= = != < <= << <<= > >= >> >>= | |= || & &= && ^ ^=

tests/parser/data/main.arf

@@ -22,7 +22,9 @@ data ro integerArrayInitialized:{Int 16}
 
 # data wr mutIntegerPointerInit:{Int}:mut [& integer]
 
-data ro object:Obj
+# TODO: maybe test identifiers somewhere else?
+data ro object:thing.thing.
+	thing.thing
 	.this 324
 	.that 2139
 

tests/parser/func/main.arf

@@ -0,0 +1,134 @@
+:arf
+---
+func ro aBasicExternal
+	> someInput:Int:mut
+	< someOutput:Int 4
+	---
+	external
+
+func ro bMethod
+	@ bird:{Bird}
+	> someInput:Int:mut
+	< someOutput:Int 4
+	---
+	external
+
+func ro cBasicPhrases
+	---
+	fn 329 983 09
+	[fn 329 983 09]
+	[fn
+		329
+		983
+		091]
+	fn [gn
+		329 983
+		091] 123
+
+func ro dArgumentTypes
+	---
+	[bird tree butterfly.wing "hello world"
+		grass:{Int:mut 8}]
+
+func ro eMath
+	> x:Int
+	> y:Int
+	< z:Int
+	---
+	++ x
+	-- y
+	set z [+ [* 0392 00] 98 x [/ 9832 y] 930]
+
+	# TODO: need tokens ~=
+	
+	! true
+	~ 0b01
+	# ~= x
+	% 873 32
+
+	=  5 5
+	!= 4 4
+	
+	<=  4 98
+	<   4 98
+	<<  0x0F 4
+	# <<= x 4
+
+	>=  98 4
+	>   98 4
+	>>  0xF0 4
+	# >>= x 4
+
+	|  0b01 0b10
+	# |= x 0b10
+	& 0b110 0b011
+	# &= x 0b011
+
+	&& true true
+	|| true false
+
+func ro fReturnDirection
+	< err:Error
+	---
+	someFunc 498 2980 90 -> thing:Int err
+	otherFunc -> thing err:Error
+	
+	[fn
+		329
+		983
+		091] -> thing:Int err
+
+func ro gControlFlow
+	---
+	if condition
+		something
+	
+	if condition
+		something
+	elseif
+		[otherThing]
+	else
+		finalThing
+	
+	while [< x 432]
+		something
+	
+	switch value
+	: 324
+		something
+	[: 93284]
+		otherThing
+	: 9128 34738 7328
+		multipleCases
+	:
+		[defaultThing]
+	
+	for index:Size element:Int someArray
+		something
+		someNextThing
+		justMakingSureBlockParsingWorks
+
+	[if condition]
+		if condition
+			nestedThing
+		else
+			otherThing
+	else
+		if condition
+			nestedThing
+		else
+			otherThing
+
+func hSetPhrase
+	---
+	set x:Int 3
+	# TODO: this should be the "location of" phrase. update other things to
+	# match.
+	set y:{Int} [. x]
+	set z:{Int 8}
+		398 9 2309 983 -2387
+		478 555 123
+	set bird:Bird
+		.that
+			.whenYou 99999
+		.this 324