Update go-util

Reviewed-on: #22

cmd/hopp-generate/main.go

@@ -55,7 +55,7 @@ func main() {
 	}
 	_, err = generator.Generate(protocol)
 	handleErr(command, 1, err)
-	command.Println(output, "OK")
+	command.Println(destination, "OK")
 }
 
 func handleErr(command *cli.Cli, code int, err error) {

design/pdl-language.md

@@ -33,6 +33,11 @@ PDL allows defining a protocol using HOPP and TAPE.
 | {...}      | KTV     |  * | Table with defined schema
 | Any        | *       |  * | Value of an undefined type
 
+Tables with a defined schema can specify some fields as optional using a
+question mark before the type. This will wrap the field the go-util
+ucontainer.Option type. When encoding, void fields will not be included in the
+output, and when decoding, unspecified fields are left void.
+
 [^1]: Excluding SI and SBA. I5 and U5 cannot be used in an array, but String and
 Buffer are simply forced to use their "long" variant.
 
@@ -48,6 +53,7 @@ structures. They are separated by whitespace.
 | Method   | `M[0-9A-Fa-f]{4}`  | A 16-bit hexadecimal method code.
 | Key      | `[0-9A-Fa-f]{4}`   | A 16-bit hexadecimal table key.
 | Ident    | `[A-Z][A-Za-z0-9]` | An identifier.
+| Option   | `?`                | A question mark.
 | Comma    | `,`                | A comma separator.
 | LBrace   | `{`                | A left curly brace.
 | RBrace   | `}`                | A right curly brace.
@@ -107,7 +113,7 @@ Below is an EBNF description of the language.
 <method>  -> /M[0-9A-Fa-f]{4}/
 <key>     -> /[0-9A-Fa-f]{4}/
 <ident>   -> /[A-Z][A-Za-z0-9]/
-<field>   -> <key> <ident> <type>
+<field>   -> <key> <ident> ["?"] <type>
 <type> -> <ident>
         | "[" "]" <type>
         | "{" (<comment>* <field> ",")* [<comment>* <field>] "}"

generate/generate.go

@@ -11,6 +11,7 @@ import "git.tebibyte.media/sashakoshka/hopp/tape"
 
 const imports =
 `
+import "git.tebibyte.media/sashakoshka/hopp"
 import "git.tebibyte.media/sashakoshka/hopp/tape"
 `
 
@@ -47,6 +48,17 @@ func canAssign(destination, source tape.Tag) bool {
 	}
 	return false
 }
+
+// boolInt converts a bool to an integer.
+func boolInt(input bool) int {
+	if input {
+		return 1
+	} else {
+		return 0
+	}
+}
+
+var _ hopp.Option[int]
 `
 
 // Generator converts protocols into Go code.
@@ -315,6 +327,9 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e
 //   - nn int
 func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource string) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		// SI: (none)
+		// SI stores the value in the tag, so we write nothing here
 	case TypeInt:
 		// SI:     (none)
 		// LI/LSI: <value: IntN>
@@ -467,19 +482,33 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		n += nn; if err != nil { return n, err }
 		this.push()
 			for key, field := range typ.Fields {
+				fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name)
+				if field.Option {
+					nn, err = this.iprintf("if value, ok := %s.Value(); ok {\n", fieldSource)
+					n += nn; if err != nil { return n, err }
+					fieldSource = "value"
+					this.push()
+				}
+
 				nn, err = this.iprintf("nn, err = encoder.WriteUint16(0x%04X)\n", key)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
-				fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name)
 				tagVar, nn, err := this.generateTag(field.Type, fieldSource)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(%s))\n", tagVar)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
+				
 				nn, err = this.generateEncodeValue(field.Type, fieldSource, tagVar)
 				n += nn; if err != nil { return n, err }
+				
+				if field.Option {
+					this.pop()
+					nn, err = this.iprintf("}\n")
+					n += nn; if err != nil { return n, err }
+				}
 			}
 		this.pop()
 		nn, err = this.iprintf("}\n")
@@ -518,6 +547,11 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 // for [Generator.generateDecodeBranch].
 func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagSource string) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		// SI: (none)
+		// SI stores the value in the tag
+		nn, err := this.iprintf("*%s = %s.CN() > 0\n", valueSource, tagSource)
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		// SI:     (none)
 		// LI/LSI: <value: IntN>
@@ -537,13 +571,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 			prefix = "ReadInt"
 		}
 		destinationVar := this.newTemporaryVar("destination")
-		nn, err := this.iprintf("var %s ", destinationVar)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.generateType(typ)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.print("\n")
-		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("%s, nn, err = decoder.%s%d()\n", destinationVar, prefix, typ.Bits)
+		nn, err := this.iprintf("%s, nn, err := decoder.%s%d()\n", destinationVar, prefix, typ.Bits)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
@@ -557,13 +585,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 	case TypeFloat:
 		// FP: <value: FloatN>
 		destinationVar := this.newTemporaryVar("destination")
-		nn, err := this.iprintf("var %s ", destinationVar)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.generateType(typ)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.print("\n")
-		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("%s, nn, err = decoder.ReadFloat%d()\n", destinationVar, typ.Bits)
+		nn, err := this.iprintf("%s, nn, err := decoder.ReadFloat%d()\n", destinationVar, typ.Bits)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
@@ -786,6 +808,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
 		case TypeTableDefined:
+			// TODO: options
 			// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
 			// read header
 			lengthVar := this.newTemporaryVar("length")
@@ -862,10 +885,25 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 					n += nn; if err != nil { return n, err }
 
 					// decode payload
-					nn, err = this.generateDecodeValue(
-						field.Type, "",
-						fmt.Sprintf("(&(this.%s))", field.Name), fieldTagVar)
-					n += nn; if err != nil { return n, err }
+					if field.Option {
+						destination := this.newTemporaryVar("destination")
+						nn, err = this.iprintf("var %s ", destination)
+						n += nn; if err != nil { return n, err }
+						nn, err = this.generateType(field.Type)
+						n += nn; if err != nil { return n, err }
+						nn, err = this.printf("\n")
+						n += nn; if err != nil { return n, err }
+						nn, err = this.generateDecodeValue(
+							field.Type, "", fmt.Sprintf("(&%s)", destination), fieldTagVar)
+						n += nn; if err != nil { return n, err }
+						nn, err = this.iprintf("this.%s = hopp.O(%s)\n", field.Name, destination)
+						n += nn; if err != nil { return n, err }
+					} else {
+						nn, err = this.generateDecodeValue(
+							field.Type, "",
+							fmt.Sprintf("(&(this.%s))", field.Name), fieldTagVar)
+						n += nn; if err != nil { return n, err }
+					}
 					this.pop()
 				}					
 				nn, err = this.iprintf("default:\n")
@@ -878,16 +916,6 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			
-			// TODO once options are implemented, have a set of
-			// bools for each non-optional field, and check here
-			// that they are all true. a counter will not work
-			// because if someone specifies a non-optional field
-			// twice, they can neglect to specify another
-			// non-optional field and we won't even know because the
-			// count will still be even. we shouldn't use a map
-			// either because its an allocation and its way more
-			// memory than just, like 5 bools (on the stack no less)
 		default: return n, fmt.Errorf("unexpected type: %T", typ)
 		}
 
@@ -944,6 +972,9 @@ func (this *Generator) generateBareErrorCheck() (n int, err error) {
 func (this *Generator) generateTag(typ Type, source string) (tagVar string, n int, err error) {
 	tagVar = this.newTemporaryVar("tag")
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.iprintf("%s := tape.SI.WithCN(boolInt(bool(%s)))\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeInt:
 		if typ.Bits <= 5 {
 			nn, err := this.iprintf("%s := tape.SI.WithCN(int(%s))\n", tagVar, source)
@@ -997,6 +1028,9 @@ func (this *Generator) generateTag(typ Type, source string) (tagVar string, n in
 // information is chosen.
 func (this *Generator) generateTN(typ Type) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.printf("tape.SI")
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		if typ.Bits <= 5 {
 			nn, err := this.printf("tape.SI")
@@ -1040,6 +1074,9 @@ func (this *Generator) generateTN(typ Type) (n int, err error) {
 
 func (this *Generator) generateType(typ Type) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.printf("bool")
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		if err := this.validateIntBitSize(typ.Bits); err != nil {
 			return n, err
@@ -1107,8 +1144,16 @@ func (this *Generator) generateTypeTableDefined(typ TypeTableDefined) (n int, er
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf("%s ", field.Name)
 		n += nn; if err != nil { return n, err }
+		if field.Option {
+			nn, err = this.print("hopp.Option[")
+			n += nn; if err != nil { return n, err }
+		}
 		nn, err = this.generateType(field.Type)
 		n += nn; if err != nil { return n, err }
+		if field.Option {
+			nn, err = this.print("]")
+			n += nn; if err != nil { return n, err }
+		}
 		nn, err = this.print("\n")
 		n += nn; if err != nil { return n, err }
 	}

generate/generate_test.go

@@ -59,6 +59,7 @@ func init() {
 				0x000C: Field { Name: "NI16",Type: TypeInt { Bits: 16, Signed: true } },
 				0x000D: Field { Name: "NI32",Type: TypeInt { Bits: 32, Signed: true } },
 				0x000E: Field { Name: "NI64",Type: TypeInt { Bits: 64, Signed: true } },
+				0x000F: Field { Name: "Bool",Type: TypeBool { } },
 			},
 		},
 	}
@@ -83,6 +84,34 @@ func init() {
 			},
 		},
 	}
+	exampleProtocol.Messages[0x0006] = Message {
+		Name: "Option",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "OU5",     Option: true, Type: TypeInt    { Bits: 5  } },
+				0x0001: Field { Name: "OU8",     Option: true, Type: TypeInt    { Bits: 8  } },
+				0x0002: Field { Name: "OU16",    Option: true, Type: TypeInt    { Bits: 16 } },
+				0x0003: Field { Name: "OU32",    Option: true, Type: TypeInt    { Bits: 32 } },
+				0x0004: Field { Name: "OU64",    Option: true, Type: TypeInt    { Bits: 64 } },
+				0x0005: Field { Name: "OI8",     Option: true, Type: TypeInt    { Bits: 8,  Signed: true } },
+				0x0006: Field { Name: "OI16",    Option: true, Type: TypeInt    { Bits: 16, Signed: true } },
+				0x0007: Field { Name: "OI32",    Option: true, Type: TypeInt    { Bits: 32, Signed: true } },
+				0x0008: Field { Name: "OI64",    Option: true, Type: TypeInt    { Bits: 64, Signed: true } },
+				0x0009: Field { Name: "OF32",    Option: true, Type: TypeFloat  { Bits: 32 } },
+				0x000A: Field { Name: "OF64",    Option: true, Type: TypeFloat  { Bits: 64 } },
+				0x000B: Field { Name: "OBool",   Option: true, Type: TypeBool   { } },
+				0x000C: Field { Name: "OString", Option: true, Type: TypeString { } },
+				0x000D: Field { Name: "OArray",  Option: true, Type: TypeArray {
+					Element: TypeInt { Bits: 16, Signed: true } } },
+				0x000E: Field { Name: "OTable",  Option: true, Type: TypeTableDefined {
+					Fields: map[uint16] Field {
+						0x0001: Field { Name: "A", Type: TypeInt   { Bits: 8, Signed: true }, },
+						0x0002: Field { Name: "B", Type: TypeFloat { Bits: 64 }, },
+					} } },
+				0x000F: Field { Name: "T0", Option: true, Type: TypeTable { } },
+			},
+		},
+	}
 	exampleProtocol.Types["User"] = Typedef {
 		Type: TypeTableDefined {
 			Fields: map[uint16] Field {
@@ -201,10 +230,11 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 			NI16: -0x34C9,
 			NI32: -0x10E134C9,
 			NI64: -0x639109BC10E134C9,
+			Bool: true,
 		}
 		testEncodeDecode(
 			&messageIntegers,
-			tu.S(0xE0, 13).AddVar(
+			tu.S(0xE0, 14).AddVar(
 				[]byte { 0x00, 0x00, 0x13 },
 				[]byte { 0x00, 0x01, 0x20, 0xC9 },
 				[]byte { 0x00, 0x02, 0x21, 0x34, 0xC9 },
@@ -218,6 +248,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 				[]byte { 0x00, 0x0C, 0x41, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0D, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0E, 0x47, 0x9C, 0x6E, 0xF6, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
+				[]byte { 0x00, 0x0F, 0x01 },
 				))
 		log.Println("MessageDynamic")
 		messageDynamic := MessageDynamic {

generate/lex.go

@@ -10,6 +10,7 @@ const (
 	TokenMethod parse.TokenKind = iota
 	TokenKey
 	TokenIdent
+	TokenOption
 	TokenComma
 	TokenLBrace
 	TokenRBrace
@@ -22,6 +23,7 @@ var tokenNames = map[parse.TokenKind] string {
 	TokenMethod:   "Method",
 	TokenKey:      "Key",
 	TokenIdent:    "Ident",
+	TokenOption:   "Option",
 	TokenComma:    "Comma",
 	TokenLBrace:   "LBrace",
 	TokenRBrace:   "RBrace",
@@ -122,6 +124,11 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 			if this.eof { err = nil; return }
 			if err != nil { return }
 		}
+	// Option
+	case this.rune == '?':
+		token.Kind = TokenOption
+		appendRune()
+		if this.eof { err = nil; return }
 	// Comma
 	case this.rune == ',':
 		token.Kind = TokenComma

generate/lex_test.go

@@ -16,6 +16,7 @@ func TestLex(test *testing.T) {
 		// wow
 		0001 Users []User,
 		0002 Followers U32,
+		0003 Wings ?Int,
 	}`))
 	if err != nil { test.Fatal(parse.Format(err)) }
 
@@ -42,6 +43,11 @@ func TestLex(test *testing.T) {
 		tok(TokenIdent,    "Followers"),
 		tok(TokenIdent,    "U32"),
 		tok(TokenComma,    ","),
+		tok(TokenKey,      "0003"),
+		tok(TokenIdent,    "Wings"),
+		tok(TokenOption,   "?"),
+		tok(TokenIdent,    "Int"),
+		tok(TokenComma,    ","),
 		tok(TokenRBrace,   "}"),
 		tok(parse.EOF,     ""),
 	}

generate/parse.go

@@ -214,12 +214,19 @@ func (this *parser) parseField(doc string) (uint16, Field, error) {
 	name := this.Value()
 	err = this.Next()
 	if err != nil { return 0, Field { }, err }
+	option := false
+	if this.Kind() == TokenOption {
+		option = true
+		err = this.Next()
+		if err != nil { return 0, Field { }, err }
+	}
 	typ, err := this.parseType()
 	if err != nil { return 0, Field { }, err }
 	return uint16(key), Field {
-		Name: name,
-		Doc:  doc,
-		Type: typ,
+		Name:   name,
+		Doc:    doc,
+		Type:   typ,
+		Option: option,
 	}, nil
 }
 

generate/parse_test.go

@@ -34,6 +34,7 @@ func TestParse(test *testing.T) {
 				0x0001: Field { Name: "Bio",       Type: TypeString { } },
 				0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
 				0x0003: Field { Name: "Bouncy",    Type: TypeBool { } },
+				0x0004: Field { Name: "Wings",     Type: TypeInt { Bits: 32 } },
 			},
 		},
 	}
@@ -63,6 +64,7 @@ func TestParse(test *testing.T) {
 		0001 Bio       String,
 		0002 Followers U32,
 		0003 Bouncy    Bool,
+		0004 Wings     ?U32,
 	}
 
 	Anything Any

generate/protocol.go

@@ -95,9 +95,10 @@ func (typ TypeTableDefined) String() string {
 }
 
 type Field struct {
-	Name string
-	Doc  string
-	Type Type
+	Name   string
+	Doc    string
+	Type   Type
+	Option bool
 }
 
 func (field Field) String() string {

go.mod

@@ -1,9 +1,9 @@
 module git.tebibyte.media/sashakoshka/hopp
 
-go 1.23.0
+go 1.24.0
 
 require (
 	git.tebibyte.media/sashakoshka/go-cli v0.1.3
-	git.tebibyte.media/sashakoshka/go-util v0.9.1
+	git.tebibyte.media/sashakoshka/go-util v0.11.0
 	git.tebibyte.media/sashakoshka/goparse v0.2.0
 )

go.sum

@@ -1,6 +1,6 @@
 git.tebibyte.media/sashakoshka/go-cli v0.1.3 h1:tSkWjyx2JrGu6KotbXWSTKSYGGS1D4O3qwCrRoZuwbs=
 git.tebibyte.media/sashakoshka/go-cli v0.1.3/go.mod h1:JFA3wSdRkXxa4iQJWHfe3DokiG7Dh2XUJBzPmuVlbuY=
-git.tebibyte.media/sashakoshka/go-util v0.9.1 h1:eGAbLwYhOlh4aq/0w+YnJcxT83yPhXtxnYMzz6K7xGo=
-git.tebibyte.media/sashakoshka/go-util v0.9.1/go.mod h1:0Q1t+PePdx6tFYkRuJNcpM1Mru7wE6X+it1kwuOH+6Y=
+git.tebibyte.media/sashakoshka/go-util v0.11.0 h1:ZxLJWHr0ecgVRV5O4MV7EGHK5xHJMppKd1P4x70AtYQ=
+git.tebibyte.media/sashakoshka/go-util v0.11.0/go.mod h1:0Q1t+PePdx6tFYkRuJNcpM1Mru7wE6X+it1kwuOH+6Y=
 git.tebibyte.media/sashakoshka/goparse v0.2.0 h1:uQmKvOCV2AOlCHEDjg9uclZCXQZzq2PxaXfZ1aIMiQI=
 git.tebibyte.media/sashakoshka/goparse v0.2.0/go.mod h1:tSQwfuD+EujRoKr6Y1oaRy74ZynatzkRLxjE3sbpCmk=

internal/testutil/testutil.go

@@ -97,6 +97,32 @@ func (sn Snake) String() string {
 	return out.String()
 }
 
+func (sn Snake) CharsString() string {
+	if len(sn) == 0 || len(sn[0]) == 0 || len(sn[0][0]) == 0 {
+		return "EMPTY"
+	}
+
+	out := strings.Builder { }
+	for index, sector := range sn {
+		if index > 0 { out.WriteString(" : ") }
+		out.WriteRune('[')
+		for index, variation := range sector {
+			if index > 0 { out.WriteString(" / ") }
+			for _, byt := range variation {
+				run := rune(byt)
+				if unicode.IsPrint(run) && run < 0x7F {
+					out.WriteRune(run)
+				} else {
+					out.WriteRune('.')
+				}
+				out.WriteRune(' ')
+			}
+		}
+		out.WriteRune(']')
+	}
+	return out.String()
+}
+
 // HexBytes formats bytes into a hexadecimal string.
 func HexBytes(data []byte) string {
 	if len(data) == 0 { return "EMPTY" }
@@ -107,6 +133,24 @@ func HexBytes(data []byte) string {
 	return out.String()
 }
 
+// HexChars returns all printable bytes in the string, with non-printable ones
+// replaced with a dot. Each character has an extra space after it for placing
+// underneath the result of HexBytes.
+func HexChars(data []byte) string {
+	if len(data) == 0 { return "EMPTY" }
+	out := strings.Builder { }
+	for _, byt := range data {
+		run := rune(byt)
+		if unicode.IsPrint(run) && run < 0x7F {
+			out.WriteRune(run)
+		} else {
+			out.WriteRune('.')
+		}
+		out.WriteRune(' ')
+	}
+	return out.String()
+}
+
 // Describe returns a string representing the type and data of the given value.
 func Describe(value any) string {
 	desc := describer { }

option.go

@@ -2,29 +2,10 @@ package hopp
 
 import "git.tebibyte.media/sashakoshka/go-util/container"
 
-// Option allows an optional value to be defined without using a pointer.
-// TODO make generic alias once go 1.24 releases
-type Option[T any] ucontainer.Optional[T]
+// Option is an alias for ucontainer.Option, defined here for convenience
+type Option[T any] = ucontainer.Option[T]
 
+// O is an alias for ucontainer.O, defined here for convenience
 func O[T any](value T) Option[T] {
-	return Option[T](ucontainer.O(value))
-}
-
-func Void[T any]() Option[T] {
-	return Option[T](ucontainer.Void[T]())
-}
-
-func (option Option[T]) Ok() bool {
-	return (ucontainer.Optional[T])(option).Exists()
-}
-
-func (option Option[T]) Get() (T, bool) {
-	return (ucontainer.Optional[T])(option).Value()
-}
-
-func (option Option[T]) Default(defaul T) T {
-	if value, ok := option.Get(); ok {
-		return value
-	}
-	return defaul
+	return ucontainer.O(value)
 }

pdl.yaml

@@ -0,0 +1,20 @@
+filetype: pdl
+
+detect:
+        filename: "\\.pdl$"
+
+rules:
+        - preproc: "\\bM[0-9a-fA-F]{4}\\b"
+
+        - type: "\\b((U|I)(5|8|16|32|64|128|256)|F(16|32|64|128|256)|Bool|String|Buffer|Table|Any)\\b"
+
+        - symbol.brackets: "(\\{|\\}|\\[\\])"
+        - symbol.operator: "\\?"
+
+        - constant.number: "\\b[0-9a-fA-F]{4}\\b"
+
+        - comment:
+                start: "//"
+                end: "$"
+                rules:
+                        - todo: "(TODO|XXX|FIXME|BUG):?"

tape/dynamic.go

@@ -6,6 +6,10 @@ package tape
 // TODO: add support for struct tags: `tape:"0000"`, tape:"0001"` so they can get
 // transformed into tables with a defined schema
 
+// TODO: support special behavior for options in structs: don't just write a
+// zero value if the option is void, write no field at all. also consider doing
+// this for maps, and maybe slices.
+
 // TODO: test all of these smaller functions individually
 
 // For an explanation as to why this package always treats LBA/SBA as strings,
@@ -59,6 +63,7 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	case reflect.Uint64:  return encoder.WriteUint64(uint64(reflectValue.Uint()))
 	case reflect.Float32: return encoder.WriteFloat32(float32(reflectValue.Float()))
 	case reflect.Float64: return encoder.WriteFloat64(float64(reflectValue.Float()))
+	case reflect.Bool:    return // SI has no payload
 	case reflect.String:
 		if reflectValue.Len() > MaxStructureLength {
 			return 0, ErrTooLong
@@ -78,6 +83,12 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 		return n, nil
 	}
 
+	// option
+	if isTypeOption(reflectValue.Type()) {
+		elemValue, _ := optionValue(reflectValue) // zero value for free
+		return EncodeAny(encoder, elemValue, tag)
+	}
+	
 	// aggregates
 	reflectType := reflect.TypeOf(value)
 	switch reflectType.Kind() {
@@ -298,11 +309,23 @@ func tagAny(reflectValue reflect.Value) (Tag, error) {
 	case reflect.Uint64:  return LI.WithCN(7), nil
 	case reflect.Float32: return FP.WithCN(3), nil
 	case reflect.Float64: return FP.WithCN(7), nil
+	case reflect.Bool:
+		if reflectValue.Bool() {
+			return SI.WithCN(1), nil
+		} else {
+			return SI.WithCN(0), nil
+		}
 	case reflect.String:  return bufferLenTag(reflectValue.Len()), nil
 	}
 	if reflectValue.CanConvert(reflect.TypeOf(dummyBuffer)) {
 		return bufferLenTag(reflectValue.Len()), nil
 	}
+
+	// option
+	if isTypeOption(reflectValue.Type()) {
+		elem, _ := optionValue(reflectValue) // zero value for free
+		return tagAny(elem)
+	}
 	
 	// aggregates
 	reflectType := reflectValue.Type()
@@ -329,9 +352,14 @@ func encodeAnySlice(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	for index := 0; index < reflectValue.Len(); index += 1 {
 		itemTag, err := tagAny(reflectValue.Index(index))
 		if err != nil { return n, err }
+		if itemTag.Is(SBA) {
+			// SBA data in an LBA will always have the tag LBA:0,
+			// because 32 <= 256
+			continue
+		}
 		if itemTag.CN() > oneTag.CN() { oneTag = itemTag }
 	}
-	if oneTag.Is(SBA) { oneTag += 1 << 5 }
+	if oneTag.Is(SBA) { oneTag = LBA.WithCN(oneTag.CN()) }
 	nn, err = encoder.WriteUint8(uint8(oneTag))
 	n += nn; if err != nil { return n, err }
 	for index := 0; index < reflectValue.Len(); index += 1 {
@@ -374,7 +402,8 @@ func canSet(destination reflect.Type, tag Tag) error {
 		switch destination.Kind() {
 		case
 			reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
-			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
+			reflect.Bool:
 		default:
 			return errCantAssignf("cannot assign integer to %v", destination)
 		}
@@ -432,6 +461,8 @@ func setInt(destination reflect.Value, value int64, bytes int) {
 // setUint expects a settable destination.
 func setUint(destination reflect.Value, value uint64, bytes int) {
 	switch {
+	case destination.Kind() == reflect.Bool:
+		destination.Set(reflect.ValueOf(value > 0))
 	case destination.CanInt():
 		destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type()))
 	case destination.CanUint():
@@ -566,6 +597,21 @@ func isTypeAny(typ reflect.Type) bool {
 	return typ.Kind() == reflect.Interface && typ.NumMethod() == 0
 }
 
+// isTypeOption returns whether the given reflect.Type is a ucontainer.Option,
+// and returns the element type if true.
+func isTypeOption(typ reflect.Type) bool {
+	// TODO: change when needed
+	goutilPath := "git.tebibyte.media/sashakoshka/go-util"
+	return typ.Name() == "Option" && typ.PkgPath() == goutilPath + "/container"
+}
+
+// optionValue returns the value of an option. The value MUST be an option, or
+// this function will panic.
+func optionValue(value reflect.Value) (elem reflect.Value, ok bool) {
+	result := value.MethodByName("Value").Call([]reflect.Value { })
+	return result[0], result[1].Bool()
+}
+
 // peekSlice returns the element tag and dimension count of the OTA currently
 // being decoded. It does not use up the decoder, it only peeks.
 func peekSlice(decoder *Decoder, tag Tag) (Tag, int, error) {

tape/dynamic_test.go

@@ -3,37 +3,10 @@ package tape
 import "bytes"
 import "testing"
 import "reflect"
+import "git.tebibyte.media/sashakoshka/go-util/container"
 import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
 
-var samplePayloads = [][]byte {
-	/* int8     */ []byte { byte(LSI.WithCN(0)), 0x45 },
-	/* int16    */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 },
-	/* int32    */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-	/* int64    */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-	/* uint5    */ []byte { byte(SI.WithCN(12)) },
-	/* uint8    */ []byte { byte(LI.WithCN(0)), 0x45 },
-	/* uint16   */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 },
-	/* uint32   */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-	/* uint64   */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-	/* string   */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' },
-	/* []byte   */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' },
-	/* []string */ []byte {
-		byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)),
-		0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
-		0x05, 0x11, 0x11, 0x11, 0x11, 0x11,
-	},
-	/* map[uint16] any */ []byte {
-		byte(KTV.WithCN(0)), 2,
-		0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67,
-		0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB,
-	},
-	/* map[uint16] any */ []byte {
-		byte(KTV.WithCN(0)), 3,
-		0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00,
-		0x00, 0x02, 0x82, 'h', 'i',
-		0x00, 0x03, 0x21, 0x39, 0x92,
-	},
-}
+var samplePayloads [][]byte
 
 var sampleValues = []any {
 	/* int8     */ int8(0x45),
@@ -45,6 +18,8 @@ var sampleValues = []any {
 	/* uint16   */ uint16(0x4567),
 	/* uint32   */ uint32(0x456789AB),
 	/* uint64   */ uint64(0x456789ABCDEF0123),
+	/* bool     */ false,
+	/* bool     */ true,
 	/* string   */ "pupever",
 	/* []byte   */ "blarg",
 	/* []string */ []string {
@@ -60,6 +35,64 @@ var sampleValues = []any {
 		0x0002: "hi",
 		0x0003: uint16(0x3992),
 	},
+	// IMPORTANT: ONLY ADD AT THE END!!!! DO NOT MOVE WHAT IS ALREADY HERE!
+	// IMPORTANT: ONLY ADD AT THE END!!!! DO NOT MOVE WHAT IS ALREADY HERE!
+}
+
+type sample struct {
+	t Tag
+	v any
+	s tu.Snake
+}
+
+var samples = []sample {
+	/* int8     */ sample { t: LSI.WithCN(0), s: tu.S(0x45) },
+	/* int16    */ sample { t: LSI.WithCN(1), s: tu.S(0x45, 0x67) },
+	/* int32    */ sample { t: LSI.WithCN(3), s: tu.S(0x45, 0x67, 0x89, 0xAB) },
+	/* int64    */ sample { t: LSI.WithCN(7), s: tu.S(0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23) },
+	/* uint5    */ sample { t: SI.WithCN(12), s: tu.S() },
+	/* uint8    */ sample { t: LI.WithCN(0),  s: tu.S(0x45) },
+	/* uint16   */ sample { t: LI.WithCN(1),  s: tu.S(0x45, 0x67) },
+	/* uint32   */ sample { t: LI.WithCN(3),  s: tu.S(0x45, 0x67, 0x89, 0xAB) },
+	/* uint64   */ sample { t: LI.WithCN(7),  s: tu.S(0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23) },
+	/* bool     */ sample { t: SI.WithCN(0),  s: tu.S() },
+	/* bool     */ sample { t: SI.WithCN(1),  s: tu.S() },
+	/* string   */ sample { t: SBA.WithCN(7), s: tu.S('p', 'u', 'p', 'e', 'v', 'e', 'r') },
+	/* []byte   */ sample { t: SBA.WithCN(5), s: tu.S('b', 'l', 'a', 'r', 'g') },
+	/* []string */ sample {
+		t: OTA.WithCN(0),
+		s: tu.S(2, byte(LBA.WithCN(0)),
+			0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
+			0x05, 0x11, 0x11, 0x11, 0x11, 0x11),
+	},
+	/* map[uint16] any */ sample {
+		t: KTV.WithCN(0),
+		s: tu.S(2).AddVar(
+			[]byte { 0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67 },
+			[]byte { 0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB }),
+	},
+	/* map[uint16] any */ sample {
+		t: KTV.WithCN(0),
+		s: tu.S(3).AddVar(
+			[]byte { 0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00 },
+			[]byte { 0x00, 0x02, 0x82, 'h', 'i' },
+			[]byte { 0x00, 0x03, 0x21, 0x39, 0x92 }),
+	},
+}
+
+var sampleOptionValues []any
+
+func init() {
+	sampleOptionValues = make([]any, len(sampleValues))
+	for index, value := range sampleValues {
+		sampleOptionValues[index] = ucontainer.O(value)
+		samples[index].v = value
+	}
+	samplePayloads = make([][]byte, len(samples))
+	for index, sample := range samples {
+		item := append([]byte { byte(sample.t) }, sample.s.Flatten()...)
+		samplePayloads[index] = item
+	}
 }
 
 type userDefinedInteger int16
@@ -84,7 +117,9 @@ func TestEncodeAnyTable(test *testing.T) {
 		0x3456: userDefinedInteger(0x3921),
 		0x1F1F: float32(67.26),
 		0x0F0F: float64(5.3),
-	}, KTV.WithCN(0), tu.S(9).AddVar(
+		0xAAAA: false,
+		0xBBBB: true,
+	}, KTV.WithCN(0), tu.S(11).AddVar(
 		[]byte {
 			0xF3, 0xB9,
 			byte(LSI.WithCN(3)),
@@ -138,6 +173,14 @@ func TestEncodeAnyTable(test *testing.T) {
 			byte(FP.WithCN(7)),
 			0x40, 0x15, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33,
 		},
+		[]byte {
+			0xAA, 0xAA,
+			byte(SI.WithCN(0)),
+		},
+		[]byte {
+			0xBB, 0xBB,
+			byte(SI.WithCN(1)),
+		},
 	))
 	if err != nil { test.Fatal(err) }
 }
@@ -146,7 +189,7 @@ func TestDecodeWrongType(test *testing.T) {
 	for index, data := range samplePayloads {
 		test.Logf("data %2d %v [%s]", index, Tag(data[0]), tu.HexBytes(data[1:]))
 		// integers should only assign to other integers
-		if index > 8 {
+		if index > 10 {
 			cas := func(destination any) {
 				n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
 				if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
@@ -155,7 +198,7 @@ func TestDecodeWrongType(test *testing.T) {
 					if reflectValue.Int() != 0 {
 						test.Fatalf("destination not zero: %v", reflectValue.Elem().Interface())
 					}
-				} else {
+				} else if reflectValue.Kind() != reflect.Bool {
 					if reflectValue.Uint() != 0 {
 						test.Fatalf("destination not zero: %v", reflectValue.Elem().Interface())
 					}
@@ -180,6 +223,8 @@ func TestDecodeWrongType(test *testing.T) {
 			{ var dest uint32; cas(&dest) }
 			test.Log("- uint64")
 			{ var dest uint64; cas(&dest) }
+			test.Log("- bool")
+			{ var dest bool; cas(&dest) }
 		}
 		arrayCase := func(destination any) {
 			n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
@@ -194,19 +239,19 @@ func TestDecodeWrongType(test *testing.T) {
 			}
 		}
 		// SBA/LBA types should only assign to other SBA/LBA types
-		if index != 9 && index != 10 {
+		if index != 11 && index != 12 {
 			test.Log("- string")
 			{ var dest string; arrayCase(&dest) }
 			test.Log("- []byte")
 			{ var dest []byte; arrayCase(&dest) }
 		}
 		// arrays should only assign to other arrays
-		if index != 11 {
+		if index != 13 {
 			test.Log("- []string")
 			{ var dest []string; arrayCase(&dest) }
 		}
 		// tables should only assign to other tables
-		if index != 12 && index != 13 {
+		if index != 14 && index != 15 {
 			test.Log("- map[uint16] any")
 			{ var dest = map[uint16] any { }; arrayCase(&dest) }
 		}
@@ -231,6 +276,12 @@ func TestEncodeDecodeAnyTable(test *testing.T) {
 func TestEncodeDecodeAnyDestination(test *testing.T) {
 	var destination any
 	for index, data := range samplePayloads {
+		if _, isBool := sampleValues[index].(bool); isBool {
+			// test is invalid for bools because they are never
+			// created as a skeleton value
+			continue
+		}
+	
 		tag     := Tag(data[0])
 		payload := data[1:]
 		test.Logf("data %2d %v [%s]", index, tag, tu.HexBytes(payload))
@@ -249,6 +300,20 @@ func TestEncodeDecodeAnyDestination(test *testing.T) {
 	}
 }
 
+func TestEncodeOption(test *testing.T) {
+	for _, sample := range samples {
+		snake   := sample.s
+		tag     := sample.t
+		value   := sample.v
+		if _, ok := value.(bool); tag.Is(SI) && !ok {
+			// we will never encode an SI unless its a bool
+			continue
+		}
+		err := testEncodeAny(test, value, tag, snake)
+		if err != nil { test.Fatal(err) }
+	}
+}
+
 func TestPeekSlice(test *testing.T) {
 	buffer := bytes.NewBuffer([]byte {
 		2, byte(OTA.WithCN(3)),
@@ -349,6 +414,12 @@ func TestTagAny(test *testing.T) {
 
 func TestDecodeAny(test *testing.T) {
 	for index, payload := range samplePayloads {
+		if _, isBool := sampleValues[index].(bool); isBool {
+			// test is invalid for bools because they are never
+			// created as a skeleton value
+			continue
+		}
+		
 		correctValue := sampleValues[index]
 		data := payload[1:]
 		decoder := NewDecoder(bytes.NewBuffer(data))

tape/misc_test.go

@@ -32,7 +32,9 @@ func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.S
 	test.Log("n:      ", n)
 	test.Log("tag:    ", tag)
 	test.Log("got:    ", tu.HexBytes(bytes))
+	test.Log("   :    ", tu.HexChars(bytes))
 	test.Log("correct:", correctBytes)
+	test.Log("       :", correctBytes.CharsString())
 	if tag != correctTag {
 		return fmt.Errorf("tag not equal: %v != %v", tag, correctTag)
 	}
@@ -56,6 +58,7 @@ func testEncodeDecodeAny(test *testing.T, value, correctValue any) error {
 	test.Log("n:  ", n)
 	test.Log("tag:", tag)
 	test.Log("got:", tu.HexBytes(bytes))
+	test.Log("   :", tu.HexChars(bytes))
 	test.Log("decoding...", tag)
 	if n != len(bytes) {
 		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))