generate: Add support for options in static system

Reviewed-on: #22

cmd/hopp-generate/main.go

@@ -1,48 +1,66 @@
 package main
 
 import "os"
-import "fmt"
 import "strings"
 import "path/filepath"
+import "git.tebibyte.media/sashakoshka/go-cli"
 import "git.tebibyte.media/sashakoshka/goparse"
 import "git.tebibyte.media/sashakoshka/hopp/generate"
 
 func main() {
-	name := os.Args[0]
+	flagOutput      := cli.NewInputFlag('o', "output", "The output file", "", cli.ValString)
-	if len(os.Args) != 3 {
+	flagPackageName := cli.NewInputFlag('p', "package-name", "The package name of the file", "", cli.ValString)
-		fmt.Fprintf(os.Stderr, "Usage: %s SOURCE DESTINATION\n", name)
+	command := cli.New("Compile PDL files to program source code",
+		flagOutput,
+		flagPackageName)
+	command.Syntax = "FILE [OPTIONS]..."
+	command.ParseOrExit(os.Args)
+
+	if len(command.Args) != 1 {
+		command.Usage()
 		os.Exit(2)
 	}
-	source      := os.Args[1]
+	source      := command.Args[0]
-	destination := os.Args[2]
+	destination := flagOutput.Value
+	if destination == "" {
+		destination = "protocol.go"
+	}
 
 	input, err := os.Open(source)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 	defer input.Close()
 	protocol, err := generate.ParseReader(source, input)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 
-	absDestination, err := filepath.Abs(destination)
+	packageName := flagPackageName.Value
-	handleErr(1, err)
+	if packageName == "" {
-	packageName := cleanPackageName(strings.ReplaceAll(
+		absDestination, err := filepath.Abs(destination)
-		strings.ToLower(filepath.Base(absDestination)),
+		handleErr(command, 1, err)
-		" ", "_"))
+		base := filepath.Base(absDestination)
-	destination = filepath.Join(os.Args[2], "generated.go")
+		if scrounged, ok := scroungeForPackageName(base); ok {
+			packageName = scrounged
+		} else {
+			packageName = strings.ReplaceAll(
+				strings.ToLower(base),
+				" ", "_")
+		}
+	}
+	packageName = cleanPackageName(packageName)
 
 	output, err := os.Create(destination)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 	generator := generate.Generator {
 		Output:      output,
 		PackageName: packageName,
 	}
 	_, err = generator.Generate(protocol)
-	handleErr(1, err)
+	handleErr(command, 1, err)
-	fmt.Fprintf(os.Stderr, "%s: OK\n", name)
+	command.Println(destination, "OK")
 }
 
-func handleErr(code int, err error) {
+func handleErr(command *cli.Cli, code int, err error) {
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[0], parse.Format(err))
+		command.Errorln(parse.Format(err))
 		os.Exit(code)
 	}
 }
@@ -61,3 +79,32 @@ func cleanPackageName(str string) string {
 	}
 	return string(buffer[:j])
 }
+
+func scroungeForPackageName(dir string) (string, bool) {
+	entries, err := os.ReadDir(dir)
+	if err != nil { return "", false}
+	for _, entry := range entries {
+		if !entry.Type().IsRegular() { continue }
+		file, err := os.Open(filepath.Join(dir, entry.Name()))
+		if err != nil { continue }
+		defer file.Close()
+		// FIXME: it is entirely possible that the only file will have
+		// a shitload of doc comments preceeding the package name, and
+		// those comments are usually huge so this is bad
+		buffer := [512]byte { }
+		n, _ := file.Read(buffer[:])
+		text := string(buffer[:n])
+
+		packageIndex := strings.Index(text, "package")
+		if packageIndex < 0 { continue }
+		text = text[packageIndex:]
+		
+		newlineIndex := strings.Index(text, "\n")
+		if packageIndex > 0 { text = text[:newlineIndex] }
+
+		fields := strings.Fields(text)
+		if len(fields) < 2 { continue }
+		return fields[1], true
+	}
+	return "", false
+}

design/pdl-language.md

@@ -25,6 +25,7 @@ PDL allows defining a protocol using HOPP and TAPE.
 | F64        | FP      |  7 | 
 | F128[^2]   | FP      | 15 | 
 | F256[^2]   | FP      | 31 | 
+| Bool       | SI      |    | 
 | String     | SBA/LBA |  * | UTF-8 string
 | Buffer     | SBA/LBA |  * | Byte array
 | []\<TYPE\> | OTA     |  * | Array of any type[^1]
@@ -32,6 +33,11 @@ PDL allows defining a protocol using HOPP and TAPE.
 | {...}      | KTV     |  * | Table with defined schema
 | Any        | *       |  * | Value of an undefined type
 
+Tables with a defined schema can specify some fields as optional using a
+question mark before the type. This will wrap the field the go-util
+ucontainer.Option type. When encoding, void fields will not be included in the
+output, and when decoding, unspecified fields are left void.
+
 [^1]: Excluding SI and SBA. I5 and U5 cannot be used in an array, but String and
 Buffer are simply forced to use their "long" variant.
 
@@ -47,11 +53,13 @@ structures. They are separated by whitespace.
 | Method   | `M[0-9A-Fa-f]{4}`  | A 16-bit hexadecimal method code.
 | Key      | `[0-9A-Fa-f]{4}`   | A 16-bit hexadecimal table key.
 | Ident    | `[A-Z][A-Za-z0-9]` | An identifier.
+| Option   | `?`                | A question mark.
 | Comma    | `,`                | A comma separator.
 | LBrace   | `{`                | A left curly brace.
 | RBrace   | `}`                | A right curly brace.
 | LBracket | `[`                | A left square bracket.
 | RBracket | `]`                | A right square bracket.
+| Comment  | `\/\/.*$`          | A doc comment starting with a double-slash.
 
 ## Syntax
 
@@ -68,18 +76,27 @@ an Ident token respectively. A message consists of the method code (Method), the
 message name (Ident), and the message's root type. This is usually a table, but
 can be anything.
 
+Messages, types, and table fields can all have doc comments preceding them,
+which are used to generate documentation for the protocol. The syntax is the
+same as Go's (for now). Comments aren't allowed anywhere else.
+
 Here is an example of all that:
 
 ```
+// Connect is sent from the client to the server as the first message of an
+// authenticated transaction.
 M0000 Connect {
 	0000 Name String,
 	0001 Password String,
 }
 
+// UserList is sent from the server to the client in response to a Connect
+// message.
 M0001 UserList {
 	0000 Users []User,
 }
 
+// User holds profile information about a single user.
 User {
 	0000 Name      String,
 	0001 Bio       String,
@@ -96,10 +113,10 @@ Below is an EBNF description of the language.
 <method>  -> /M[0-9A-Fa-f]{4}/
 <key>     -> /[0-9A-Fa-f]{4}/
 <ident>   -> /[A-Z][A-Za-z0-9]/
-<field>   -> <key> <ident> <type>
+<field>   -> <key> <ident> ["?"] <type>
 <type> -> <ident>
         | "[" "]" <type>
-        | "{" (<field> ",")* [<field>] "}"
+        | "{" (<comment>* <field> ",")* [<comment>* <field>] "}"
-<message> -> <method> <ident> <type>
+<message> -> <comment>* <method> <ident> <type>
-<typedef> -> <ident> <type>
+<typedef> -> <comment>* <ident> <type>
 ```

generate/generate.go

@@ -11,6 +11,7 @@ import "git.tebibyte.media/sashakoshka/hopp/tape"
 
 const imports =
 `
+import "git.tebibyte.media/sashakoshka/hopp"
 import "git.tebibyte.media/sashakoshka/hopp/tape"
 `
 
@@ -47,6 +48,17 @@ func canAssign(destination, source tape.Tag) bool {
 	}
 	return false
 }
+
+// boolInt converts a bool to an integer.
+func boolInt(input bool) int {
+	if input {
+		return 1
+	} else {
+		return 0
+	}
+}
+
+var _ hopp.Option[int]
 `
 
 // Generator converts protocols into Go code.
@@ -112,13 +124,20 @@ func (this *Generator) Generate(protocol *Protocol) (n int, err error) {
 	return n, nil
 }
 
-func (this *Generator) generateTypedef(name string, typ Type) (n int, err error) {
+func (this *Generator) generateTypedef(name string, typedef Typedef) (n int, err error) {
+	typ := typedef.Type
+
 	// type definition
-	nn, err := this.iprintf(
+	if typedef.Doc == "" {
-		"\n// %s represents the protocol data type %s.\n",
+		nn, err := this.iprintf(
-		name, name)
+			"\n// %s represents the protocol data type %s.\n",
-	n += nn; if err != nil { return n, err }
+			name, name)
-	nn, err = this.iprintf("type %s ", name)
+		n += nn; if err != nil { return n, err }
+	} else {
+		nn, err := this.iprintf("\n%s\n", this.formatComment(typedef.Doc))
+		n += nn; if err != nil { return n, err }
+	}
+	nn, err := this.iprintf("type %s ", name)
 	n += nn; if err != nil { return n, err }
 	nn, err = this.generateType(typ)
 	n += nn; if err != nil { return n, err }
@@ -208,10 +227,16 @@ func (this *Generator) generateTypedef(name string, typ Type) (n int, err error)
 // generateMessage generates the structure, as well as encoding decoding
 // functions for the given message.
 func (this *Generator) generateMessage(method uint16, message Message) (n int, err error) {
-	nn, err := this.iprintf(
+	if message.Doc == "" {
-		"\n// %s represents the protocol message M%04X %s.\n",
+		nn, err := this.iprintf(
-		message.Name, method, message.Name)
+			"\n// %s represents the protocol message M%04X %s.\n",
-	nn, err = this.iprintf("type %s ", this.resolveMessageName(message.Name))
+			message.Name, method, message.Name)
+		n += nn; if err != nil { return n, err }
+	} else {
+		nn, err := this.iprintf("\n%s\n", this.formatComment(message.Doc))
+		n += nn; if err != nil { return n, err }
+	}
+	nn, err := this.iprintf("type %s ", this.resolveMessageName(message.Name))
 	n += nn; if err != nil { return n, err }
 	nn, err = this.generateType(message.Type)
 	n += nn; if err != nil { return n, err }
@@ -302,6 +327,9 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e
 //   - nn int
 func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource string) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		// SI: (none)
+		// SI stores the value in the tag, so we write nothing here
 	case TypeInt:
 		// SI:     (none)
 		// LI/LSI: <value: IntN>
@@ -454,19 +482,33 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		n += nn; if err != nil { return n, err }
 		this.push()
 			for key, field := range typ.Fields {
+				fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name)
+				if field.Option {
+					nn, err = this.iprintf("if value, ok := %s.Value(); ok {\n", fieldSource)
+					n += nn; if err != nil { return n, err }
+					fieldSource = "value"
+					this.push()
+				}
+
 				nn, err = this.iprintf("nn, err = encoder.WriteUint16(0x%04X)\n", key)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
-				fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name)
 				tagVar, nn, err := this.generateTag(field.Type, fieldSource)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(%s))\n", tagVar)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
+				
 				nn, err = this.generateEncodeValue(field.Type, fieldSource, tagVar)
 				n += nn; if err != nil { return n, err }
+				
+				if field.Option {
+					this.pop()
+					nn, err = this.iprintf("}\n")
+					n += nn; if err != nil { return n, err }
+				}
 			}
 		this.pop()
 		nn, err = this.iprintf("}\n")
@@ -505,6 +547,11 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 // for [Generator.generateDecodeBranch].
 func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagSource string) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		// SI: (none)
+		// SI stores the value in the tag
+		nn, err := this.iprintf("*%s = %s.CN() > 0\n", valueSource, tagSource)
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		// SI:     (none)
 		// LI/LSI: <value: IntN>
@@ -524,13 +571,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 			prefix = "ReadInt"
 		}
 		destinationVar := this.newTemporaryVar("destination")
-		nn, err := this.iprintf("var %s ", destinationVar)
+		nn, err := this.iprintf("%s, nn, err := decoder.%s%d()\n", destinationVar, prefix, typ.Bits)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.generateType(typ)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.print("\n")
-		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("%s, nn, err = decoder.%s%d()\n", destinationVar, prefix, typ.Bits)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
@@ -544,13 +585,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 	case TypeFloat:
 		// FP: <value: FloatN>
 		destinationVar := this.newTemporaryVar("destination")
-		nn, err := this.iprintf("var %s ", destinationVar)
+		nn, err := this.iprintf("%s, nn, err := decoder.ReadFloat%d()\n", destinationVar, typ.Bits)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.generateType(typ)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.print("\n")
-		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("%s, nn, err = decoder.ReadFloat%d()\n", destinationVar, typ.Bits)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
@@ -773,6 +808,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
 		case TypeTableDefined:
+			// TODO: options
 			// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
 			// read header
 			lengthVar := this.newTemporaryVar("length")
@@ -849,10 +885,25 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 					n += nn; if err != nil { return n, err }
 
 					// decode payload
-					nn, err = this.generateDecodeValue(
+					if field.Option {
-						field.Type, "",
+						destination := this.newTemporaryVar("destination")
-						fmt.Sprintf("(&(this.%s))", field.Name), fieldTagVar)
+						nn, err = this.iprintf("var %s ", destination)
-					n += nn; if err != nil { return n, err }
+						n += nn; if err != nil { return n, err }
+						nn, err = this.generateType(field.Type)
+						n += nn; if err != nil { return n, err }
+						nn, err = this.printf("\n")
+						n += nn; if err != nil { return n, err }
+						nn, err = this.generateDecodeValue(
+							field.Type, "", fmt.Sprintf("(&%s)", destination), fieldTagVar)
+						n += nn; if err != nil { return n, err }
+						nn, err = this.iprintf("this.%s = hopp.O(%s)\n", field.Name, destination)
+						n += nn; if err != nil { return n, err }
+					} else {
+						nn, err = this.generateDecodeValue(
+							field.Type, "",
+							fmt.Sprintf("(&(this.%s))", field.Name), fieldTagVar)
+						n += nn; if err != nil { return n, err }
+					}
 					this.pop()
 				}					
 				nn, err = this.iprintf("default:\n")
@@ -865,16 +916,6 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			
-			// TODO once options are implemented, have a set of
-			// bools for each non-optional field, and check here
-			// that they are all true. a counter will not work
-			// because if someone specifies a non-optional field
-			// twice, they can neglect to specify another
-			// non-optional field and we won't even know because the
-			// count will still be even. we shouldn't use a map
-			// either because its an allocation and its way more
-			// memory than just, like 5 bools (on the stack no less)
 		default: return n, fmt.Errorf("unexpected type: %T", typ)
 		}
 
@@ -931,6 +972,9 @@ func (this *Generator) generateBareErrorCheck() (n int, err error) {
 func (this *Generator) generateTag(typ Type, source string) (tagVar string, n int, err error) {
 	tagVar = this.newTemporaryVar("tag")
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.iprintf("%s := tape.SI.WithCN(boolInt(bool(%s)))\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeInt:
 		if typ.Bits <= 5 {
 			nn, err := this.iprintf("%s := tape.SI.WithCN(int(%s))\n", tagVar, source)
@@ -984,6 +1028,9 @@ func (this *Generator) generateTag(typ Type, source string) (tagVar string, n in
 // information is chosen.
 func (this *Generator) generateTN(typ Type) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.printf("tape.SI")
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		if typ.Bits <= 5 {
 			nn, err := this.printf("tape.SI")
@@ -1027,6 +1074,9 @@ func (this *Generator) generateTN(typ Type) (n int, err error) {
 
 func (this *Generator) generateType(typ Type) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.printf("bool")
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		if err := this.validateIntBitSize(typ.Bits); err != nil {
 			return n, err
@@ -1090,10 +1140,20 @@ func (this *Generator) generateTypeTableDefined(typ TypeTableDefined) (n int, er
 
 	for _, key := range slices.Sorted(maps.Keys(typ.Fields)) {
 		field := typ.Fields[key]
-		nn, err := this.iprintf("%s ", field.Name)
+		nn, err := this.iprintf("%s\n", this.formatComment(field.Doc))
 		n += nn; if err != nil { return n, err }
+		nn, err = this.iprintf("%s ", field.Name)
+		n += nn; if err != nil { return n, err }
+		if field.Option {
+			nn, err = this.print("hopp.Option[")
+			n += nn; if err != nil { return n, err }
+		}
 		nn, err = this.generateType(field.Type)
 		n += nn; if err != nil { return n, err }
+		if field.Option {
+			nn, err = this.print("]")
+			n += nn; if err != nil { return n, err }
+		}
 		nn, err = this.print("\n")
 		n += nn; if err != nil { return n, err }
 	}
@@ -1175,17 +1235,21 @@ func (this *Generator) iprintf(format string, args ...any) (n int, err error) {
 	return fmt.Fprintf(this.Output, this.indent() + format, args...)
 }
 
+func (this *Generator) formatComment(comment string) string {
+	return "// " + strings.ReplaceAll(comment, "\n", "\n" + this.indent() + "// ")
+}
+
 func (this *Generator) resolveMessageName(message string) string {
 	return "Message" + message
 }
 
 func (this *Generator) resolveTypeName(name string) (Type, error) {	
-	if typ, ok := this.protocol.Types[name]; ok {
+	if typedef, ok := this.protocol.Types[name]; ok {
-		if typ, ok := typ.(TypeNamed); ok {
+		if typ, ok := typedef.Type.(TypeNamed); ok {
 			return this.resolveTypeName(typ.Name)
 		}
 		
-		return typ, nil
+		return typedef.Type, nil
 	}
 	return nil, fmt.Errorf("no type exists called %s", name)
 }

generate/generate_test.go

@@ -59,6 +59,7 @@ func init() {
 				0x000C: Field { Name: "NI16",Type: TypeInt { Bits: 16, Signed: true } },
 				0x000D: Field { Name: "NI32",Type: TypeInt { Bits: 32, Signed: true } },
 				0x000E: Field { Name: "NI64",Type: TypeInt { Bits: 64, Signed: true } },
+				0x000F: Field { Name: "Bool",Type: TypeBool { } },
 			},
 		},
 	}
@@ -83,11 +84,41 @@ func init() {
 			},
 		},
 	}
-	exampleProtocol.Types["User"] = TypeTableDefined {
+	exampleProtocol.Messages[0x0006] = Message {
-		Fields: map[uint16] Field {
+		Name: "Option",
-			0x0000: Field { Name: "Name",      Type: TypeString { } },
+		Type: TypeTableDefined {
-			0x0001: Field { Name: "Bio",       Type: TypeString { } },
+			Fields: map[uint16] Field {
-			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+				0x0000: Field { Name: "OU5",     Option: true, Type: TypeInt    { Bits: 5  } },
+				0x0001: Field { Name: "OU8",     Option: true, Type: TypeInt    { Bits: 8  } },
+				0x0002: Field { Name: "OU16",    Option: true, Type: TypeInt    { Bits: 16 } },
+				0x0003: Field { Name: "OU32",    Option: true, Type: TypeInt    { Bits: 32 } },
+				0x0004: Field { Name: "OU64",    Option: true, Type: TypeInt    { Bits: 64 } },
+				0x0005: Field { Name: "OI8",     Option: true, Type: TypeInt    { Bits: 8,  Signed: true } },
+				0x0006: Field { Name: "OI16",    Option: true, Type: TypeInt    { Bits: 16, Signed: true } },
+				0x0007: Field { Name: "OI32",    Option: true, Type: TypeInt    { Bits: 32, Signed: true } },
+				0x0008: Field { Name: "OI64",    Option: true, Type: TypeInt    { Bits: 64, Signed: true } },
+				0x0009: Field { Name: "OF32",    Option: true, Type: TypeFloat  { Bits: 32 } },
+				0x000A: Field { Name: "OF64",    Option: true, Type: TypeFloat  { Bits: 64 } },
+				0x000B: Field { Name: "OBool",   Option: true, Type: TypeBool   { } },
+				0x000C: Field { Name: "OString", Option: true, Type: TypeString { } },
+				0x000D: Field { Name: "OArray",  Option: true, Type: TypeArray {
+					Element: TypeInt { Bits: 16, Signed: true } } },
+				0x000E: Field { Name: "OTable",  Option: true, Type: TypeTableDefined {
+					Fields: map[uint16] Field {
+						0x0001: Field { Name: "A", Type: TypeInt   { Bits: 8, Signed: true }, },
+						0x0002: Field { Name: "B", Type: TypeFloat { Bits: 64 }, },
+					} } },
+				0x000F: Field { Name: "T0", Option: true, Type: TypeTable { } },
+			},
+		},
+	}
+	exampleProtocol.Types["User"] = Typedef {
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Bio",       Type: TypeString { } },
+				0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+			},
 		},
 	}
 }
@@ -199,10 +230,11 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 			NI16: -0x34C9,
 			NI32: -0x10E134C9,
 			NI64: -0x639109BC10E134C9,
+			Bool: true,
 		}
 		testEncodeDecode(
 			&messageIntegers,
-			tu.S(0xE0, 13).AddVar(
+			tu.S(0xE0, 14).AddVar(
 				[]byte { 0x00, 0x00, 0x13 },
 				[]byte { 0x00, 0x01, 0x20, 0xC9 },
 				[]byte { 0x00, 0x02, 0x21, 0x34, 0xC9 },
@@ -216,6 +248,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 				[]byte { 0x00, 0x0C, 0x41, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0D, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0E, 0x47, 0x9C, 0x6E, 0xF6, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
+				[]byte { 0x00, 0x0F, 0x01 },
 				))
 		log.Println("MessageDynamic")
 		messageDynamic := MessageDynamic {

generate/lex.go

@@ -10,22 +10,26 @@ const (
 	TokenMethod parse.TokenKind = iota
 	TokenKey
 	TokenIdent
+	TokenOption
 	TokenComma
 	TokenLBrace
 	TokenRBrace
 	TokenLBracket
 	TokenRBracket
+	TokenComment
 )
 
 var tokenNames = map[parse.TokenKind] string {
 	TokenMethod:   "Method",
 	TokenKey:      "Key",
 	TokenIdent:    "Ident",
+	TokenOption:   "Option",
 	TokenComma:    "Comma",
 	TokenLBrace:   "LBrace",
 	TokenRBrace:   "RBrace",
 	TokenLBracket: "LBracket",
 	TokenRBracket: "RBracket",
+	TokenComment:  "Comment",
 }
 
 func Lex(fileName string, reader io.Reader) (parse.Lexer, error) {
@@ -81,6 +85,18 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 		}
 	}
 
+	unexpected := func() error {
+		if unicode.IsPrint(this.rune) {
+			return parse.Errorf (
+				this.pos(), "unexpected rune '%c'",
+				this.rune)
+		} else {
+			return parse.Errorf (
+				this.pos(), "unexpected rune %U",
+				this.rune)
+		}
+	}
+
 	defer func () {
 		newPos := this.pos()
 		newPos.End -- // TODO figure out why tf we have to do this
@@ -108,6 +124,11 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 			if this.eof { err = nil; return }
 			if err != nil { return }
 		}
+	// Option
+	case this.rune == '?':
+		token.Kind = TokenOption
+		appendRune()
+		if this.eof { err = nil; return }
 	// Comma
 	case this.rune == ',':
 		token.Kind = TokenComma
@@ -133,14 +154,21 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 		token.Kind = TokenRBracket
 		appendRune()
 		if this.eof { err = nil; return }
-	case unicode.IsPrint(this.rune):
+	// Comment
-		err = parse.Errorf (
+	case this.rune == '/':
-			this.pos(), "unexpected rune '%c'",
+		token.Kind = TokenComment
-			this.rune)
+		appendRune()
+		if this.eof { return }
+		if this.rune != '/' {
+			err = unexpected()
+			return
+		}
+		for this.rune != '\n' {
+			appendRune()
+			if this.eof { err = nil; return }
+		}
 	default:
-		err = parse.Errorf (
+		err = unexpected()
-			this.pos(), "unexpected rune %U",
-			this.rune)
 	}
 
 	return

generate/lex_test.go

@@ -6,14 +6,22 @@ import "git.tebibyte.media/sashakoshka/goparse"
 
 func TestLex(test *testing.T) {
 	lexer, err := Lex("test.pdl", strings.NewReader(`
+	// User holds profile information about a single user.
 	M0001 User {
 		0000 Name      String,
+		// dog water comment
+		
+		// Users is asdkjsagkj why
+		// 
+		// wow
 		0001 Users []User,
 		0002 Followers U32,
+		0003 Wings ?Int,
 	}`))
 	if err != nil { test.Fatal(parse.Format(err)) }
 
 	correctTokens := []parse.Token {
+		tok(TokenComment,  "// User holds profile information about a single user."),
 		tok(TokenMethod,   "0001"),
 		tok(TokenIdent,    "User"),
 		tok(TokenLBrace,   "{"),
@@ -21,6 +29,10 @@ func TestLex(test *testing.T) {
 		tok(TokenIdent,    "Name"),
 		tok(TokenIdent,    "String"),
 		tok(TokenComma,    ","),
+		tok(TokenComment,  "// dog water comment"),
+		tok(TokenComment,  "// Users is asdkjsagkj why"),
+		tok(TokenComment,  "// "),
+		tok(TokenComment,  "// wow"),
 		tok(TokenKey,      "0001"),
 		tok(TokenIdent,    "Users"),
 		tok(TokenLBracket, "["),
@@ -31,6 +43,11 @@ func TestLex(test *testing.T) {
 		tok(TokenIdent,    "Followers"),
 		tok(TokenIdent,    "U32"),
 		tok(TokenComma,    ","),
+		tok(TokenKey,      "0003"),
+		tok(TokenIdent,    "Wings"),
+		tok(TokenOption,   "?"),
+		tok(TokenIdent,    "Int"),
+		tok(TokenComma,    ","),
 		tok(TokenRBrace,   "}"),
 		tok(parse.EOF,     ""),
 	}

generate/parse.go

@@ -1,6 +1,7 @@
 package generate
 
 import "io"
+import "strings"
 import "strconv"
 import "git.tebibyte.media/sashakoshka/goparse"
 
@@ -21,7 +22,7 @@ func Parse(lx parse.Lexer) (*Protocol, error) {
 func defaultProtocol() Protocol {
 	return Protocol {
 		Messages: make(map[uint16] Message),
-		Types: map[string] Type { },
+		Types: map[string] Typedef { },
 	}
 }
 
@@ -47,18 +48,28 @@ func (this *parser) parse() error {
 }
 
 func (this *parser) parseTopLevel() error {
-	err := this.ExpectDesc("message or typedef", TokenMethod, TokenIdent)
+	doc := ""
-	if err != nil { return err }
+	for {
-	if this.EOF() { return nil }
+		err := this.ExpectDesc("message or typedef", TokenMethod, TokenIdent, TokenComment)
+		if err != nil { return err }
+		if this.EOF() { return nil }
+		if this.Kind() == TokenComment {
+			if doc != "" { doc += "\n" }
+			doc += this.parseComment(this.Value())
+			this.Next()
+		} else {
+			break
+		}
+	}
 
 	switch this.Kind() {
-	case TokenMethod: return this.parseMessage()
+	case TokenMethod: return this.parseMessage(doc)
-	case TokenIdent:  return this.parseTypedef()
+	case TokenIdent:  return this.parseTypedef(doc)
 	}
 	panic("bug")
 }
 
-func (this *parser) parseMessage() error {
+func (this *parser) parseMessage(doc string) error {
 	err := this.Expect(TokenMethod)
 	if err != nil { return err }
 	method, err := this.parseHexNumber(this.Value(), 0xFFFF)
@@ -72,12 +83,13 @@ func (this *parser) parseMessage() error {
 	if err != nil { return err }
 	this.protocol.Messages[uint16(method)] = Message {
 		Name: name,
+		Doc:  doc,
 		Type: typ,
 	}
 	return nil
 }
 
-func (this *parser) parseTypedef() error {
+func (this *parser) parseTypedef(doc string) error {
 	err := this.Expect(TokenIdent)
 	if err != nil { return err }
 	name := this.Value()
@@ -85,7 +97,10 @@ func (this *parser) parseTypedef() error {
 	if err != nil { return err }
 	typ, err := this.parseType()
 	if err != nil { return err }
-	this.protocol.Types[name] = typ
+	this.protocol.Types[name] = Typedef {
+		Doc:  doc,
+		Type: typ,
+	}
 	return nil
 }
 
@@ -117,6 +132,7 @@ func (this *parser) parseType() (Type, error) {
 		case "Buffer": return TypeBuffer { }, this.Next()
 		case "Table":  return TypeTable { }, this.Next()
 		case "Any":    return TypeAny { }, this.Next()
+		case "Bool":   return TypeBool { }, this.Next()
 		}
 		return this.parseTypeNamed()
 	case TokenLBracket:
@@ -157,12 +173,22 @@ func (this *parser) parseTypeTable() (TypeTableDefined, error) {
 		Fields: make(map[uint16] Field),
 	}
 	for {
-		err := this.ExpectDesc("table field", TokenKey, TokenRBrace)
+		doc := ""
-		if err != nil { return TypeTableDefined { }, err }
+		for {
+			err := this.ExpectDesc("table field", TokenKey, TokenRBrace, TokenComment)
+			if err != nil { return TypeTableDefined { }, err }
+			if this.Kind() == TokenComment {
+				if doc != "" { doc += "\n" }
+				doc += this.parseComment(this.Value())
+				this.Next()
+			} else {
+				break
+			}
+		}
 		if this.Is(TokenRBrace) {
 			break
 		}
-		key, field, err := this.parseField()
+		key, field, err := this.parseField(doc)
 		if err != nil { return TypeTableDefined { }, err }
 		typ.Fields[key] = field
 		err = this.Expect(TokenComma, TokenRBrace)
@@ -178,7 +204,7 @@ func (this *parser) parseTypeTable() (TypeTableDefined, error) {
 	return typ, nil
 }
 
-func (this *parser) parseField() (uint16, Field, error) {
+func (this *parser) parseField(doc string) (uint16, Field, error) {
 	err := this.Expect(TokenKey)
 	if err != nil { return 0, Field { }, err }
 	key, err := this.parseHexNumber(this.Value(), 0xFFFF)
@@ -188,11 +214,19 @@ func (this *parser) parseField() (uint16, Field, error) {
 	name := this.Value()
 	err = this.Next()
 	if err != nil { return 0, Field { }, err }
+	option := false
+	if this.Kind() == TokenOption {
+		option = true
+		err = this.Next()
+		if err != nil { return 0, Field { }, err }
+	}
 	typ, err := this.parseType()
 	if err != nil { return 0, Field { }, err }
 	return uint16(key), Field {
-		Name: name,
+		Name:   name,
-		Type: typ,
+		Doc:    doc,
+		Type:   typ,
+		Option: option,
 	}, nil
 }
 
@@ -206,3 +240,7 @@ func (this *parser) parseHexNumber(input string, maxValue int64) (int64, error)
 	}
 	return number, nil
 }
+
+func (this *parser) parseComment(input string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(input, "//"), " ")
+}

generate/parse_test.go

@@ -9,6 +9,7 @@ func TestParse(test *testing.T) {
 	correct := defaultProtocol()
 	correct.Messages[0x0000] = Message {
 		Name: "Connect",
+		Doc:  "Connect is sent from the client to the server as the first message of an\nauthenticated transaction.",
 		Type: TypeTableDefined {
 			Fields: map[uint16] Field {
 				0x0000: Field { Name: "Name",      Type: TypeString { } },
@@ -18,36 +19,52 @@ func TestParse(test *testing.T) {
 	}
 	correct.Messages[0x0001] = Message {
 		Name: "UserList",
+		Doc:  "UserList is sent from the server to the client in response to a Connect\nmessage.",
 		Type: TypeTableDefined {
 			Fields: map[uint16] Field {
 				0x0000: Field { Name: "Users",  Type: TypeArray { Element: TypeNamed { Name: "User" } } },
 			},
 		},
 	}
-	correct.Types["User"] = TypeTableDefined {
+	correct.Types["User"] = Typedef {
-		Fields: map[uint16] Field {
+		Doc:  "User holds profile information about a single user.",
-			0x0000: Field { Name: "Name",      Type: TypeString { } },
+		Type: TypeTableDefined {
-			0x0001: Field { Name: "Bio",       Type: TypeString { } },
+			Fields: map[uint16] Field {
-			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Bio",       Type: TypeString { } },
+				0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+				0x0003: Field { Name: "Bouncy",    Type: TypeBool { } },
+				0x0004: Field { Name: "Wings",     Type: TypeInt { Bits: 32 } },
+			},
 		},
 	}
-	correct.Types["Anything"] = TypeAny { }
+	correct.Types["Anything"] = Typedef {
+		Type: TypeAny { },
+	}
 	test.Log("CORRECT:", &correct)
 	
 	got, err := ParseReader("test.pdl", strings.NewReader(`
+	// Connect is sent from the client to the server as the first message of an
+	// authenticated transaction.
 	M0000 Connect {
 		0000 Name String,
+		// Password is where you put your secrets, your shameful secrets
 		0001 Password String,
 	}
 	
+	// UserList is sent from the server to the client in response to a Connect
+	// message.
 	M0001 UserList {
 		0000 Users []User,
 	}
 	
+	// User holds profile information about a single user.
 	User {
 		0000 Name      String,
 		0001 Bio       String,
 		0002 Followers U32,
+		0003 Bouncy    Bool,
+		0004 Wings     ?U32,
 	}
 
 	Anything Any

generate/protocol.go

@@ -7,11 +7,17 @@ import "crypto/md5"
 
 type Protocol struct {
 	Messages map[uint16] Message
-	Types    map[string] Type
+	Types    map[string] Typedef
 }
 
 type Message struct {
 	Name string
+	Doc  string
+	Type Type
+}
+
+type Typedef struct {
+	Doc  string
 	Type Type
 }
 
@@ -43,6 +49,12 @@ func (typ TypeFloat) String() string {
 	return fmt.Sprintf("F%d", typ.Bits)
 }
 
+type TypeBool struct { }
+
+func (TypeBool) String() string {
+	return "Bool"
+}
+
 type TypeString struct { }
 
 func (TypeString) String() string {
@@ -83,8 +95,10 @@ func (typ TypeTableDefined) String() string {
 }
 
 type Field struct {
-	Name string
+	Name   string
-	Type Type
+	Doc    string
+	Type   Type
+	Option bool
 }
 
 func (field Field) String() string {

go.mod

@@ -1,8 +1,9 @@
 module git.tebibyte.media/sashakoshka/hopp
 
-go 1.23.0
+go 1.24.0
 
 require (
+	git.tebibyte.media/sashakoshka/go-cli v0.1.3
 	git.tebibyte.media/sashakoshka/go-util v0.9.1
 	git.tebibyte.media/sashakoshka/goparse v0.2.0
 )

go.sum

@@ -1,3 +1,5 @@
+git.tebibyte.media/sashakoshka/go-cli v0.1.3 h1:tSkWjyx2JrGu6KotbXWSTKSYGGS1D4O3qwCrRoZuwbs=
+git.tebibyte.media/sashakoshka/go-cli v0.1.3/go.mod h1:JFA3wSdRkXxa4iQJWHfe3DokiG7Dh2XUJBzPmuVlbuY=
 git.tebibyte.media/sashakoshka/go-util v0.9.1 h1:eGAbLwYhOlh4aq/0w+YnJcxT83yPhXtxnYMzz6K7xGo=
 git.tebibyte.media/sashakoshka/go-util v0.9.1/go.mod h1:0Q1t+PePdx6tFYkRuJNcpM1Mru7wE6X+it1kwuOH+6Y=
 git.tebibyte.media/sashakoshka/goparse v0.2.0 h1:uQmKvOCV2AOlCHEDjg9uclZCXQZzq2PxaXfZ1aIMiQI=

internal/testutil/testutil.go

@@ -97,6 +97,32 @@ func (sn Snake) String() string {
 	return out.String()
 }
 
+func (sn Snake) CharsString() string {
+	if len(sn) == 0 || len(sn[0]) == 0 || len(sn[0][0]) == 0 {
+		return "EMPTY"
+	}
+
+	out := strings.Builder { }
+	for index, sector := range sn {
+		if index > 0 { out.WriteString(" : ") }
+		out.WriteRune('[')
+		for index, variation := range sector {
+			if index > 0 { out.WriteString(" / ") }
+			for _, byt := range variation {
+				run := rune(byt)
+				if unicode.IsPrint(run) && run < 0x7F {
+					out.WriteRune(run)
+				} else {
+					out.WriteRune('.')
+				}
+				out.WriteRune(' ')
+			}
+		}
+		out.WriteRune(']')
+	}
+	return out.String()
+}
+
 // HexBytes formats bytes into a hexadecimal string.
 func HexBytes(data []byte) string {
 	if len(data) == 0 { return "EMPTY" }
@@ -107,6 +133,24 @@ func HexBytes(data []byte) string {
 	return out.String()
 }
 
+// HexChars returns all printable bytes in the string, with non-printable ones
+// replaced with a dot. Each character has an extra space after it for placing
+// underneath the result of HexBytes.
+func HexChars(data []byte) string {
+	if len(data) == 0 { return "EMPTY" }
+	out := strings.Builder { }
+	for _, byt := range data {
+		run := rune(byt)
+		if unicode.IsPrint(run) && run < 0x7F {
+			out.WriteRune(run)
+		} else {
+			out.WriteRune('.')
+		}
+		out.WriteRune(' ')
+	}
+	return out.String()
+}
+
 // Describe returns a string representing the type and data of the given value.
 func Describe(value any) string {
 	desc := describer { }

option.go

@@ -2,29 +2,10 @@ package hopp
 
 import "git.tebibyte.media/sashakoshka/go-util/container"
 
-// Option allows an optional value to be defined without using a pointer.
+// Option is an alias for ucontainer.Option, defined here for convenience
-// TODO make generic alias once go 1.24 releases
+type Option[T any] = ucontainer.Option[T]
-type Option[T any] ucontainer.Optional[T]
 
+// O is an alias for ucontainer.O, defined here for convenience
 func O[T any](value T) Option[T] {
-	return Option[T](ucontainer.O(value))
+	return ucontainer.O(value)
-}
-
-func Void[T any]() Option[T] {
-	return Option[T](ucontainer.Void[T]())
-}
-
-func (option Option[T]) Ok() bool {
-	return (ucontainer.Optional[T])(option).Exists()
-}
-
-func (option Option[T]) Get() (T, bool) {
-	return (ucontainer.Optional[T])(option).Value()
-}
-
-func (option Option[T]) Default(defaul T) T {
-	if value, ok := option.Get(); ok {
-		return value
-	}
-	return defaul
 }

pdl.yaml

@@ -0,0 +1,20 @@
+filetype: pdl
+
+detect:
+        filename: "\\.pdl$"
+
+rules:
+        - preproc: "\\bM[0-9a-fA-F]{4}\\b"
+
+        - type: "\\b((U|I)(5|8|16|32|64|128|256)|F(16|32|64|128|256)|Bool|String|Buffer|Table|Any)\\b"
+
+        - symbol.brackets: "(\\{|\\}|\\[\\])"
+        - symbol.operator: "\\?"
+
+        - constant.number: "\\b[0-9a-fA-F]{4}\\b"
+
+        - comment:
+                start: "//"
+                end: "$"
+                rules:
+                        - todo: "(TODO|XXX|FIXME|BUG):?"

tape/dynamic.go

@@ -6,6 +6,10 @@ package tape
 // TODO: add support for struct tags: `tape:"0000"`, tape:"0001"` so they can get
 // transformed into tables with a defined schema
 
+// TODO: support special behavior for options in structs: don't just write a
+// zero value if the option is void, write no field at all. also consider doing
+// this for maps, and maybe slices.
+
 // TODO: test all of these smaller functions individually
 
 // For an explanation as to why this package always treats LBA/SBA as strings,
@@ -59,6 +63,7 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	case reflect.Uint64:  return encoder.WriteUint64(uint64(reflectValue.Uint()))
 	case reflect.Float32: return encoder.WriteFloat32(float32(reflectValue.Float()))
 	case reflect.Float64: return encoder.WriteFloat64(float64(reflectValue.Float()))
+	case reflect.Bool:    return // SI has no payload
 	case reflect.String:
 		if reflectValue.Len() > MaxStructureLength {
 			return 0, ErrTooLong
@@ -78,6 +83,12 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 		return n, nil
 	}
 
+	// option
+	if isTypeOption(reflectValue.Type()) {
+		elemValue, _ := optionValue(reflectValue) // zero value for free
+		return EncodeAny(encoder, elemValue, tag)
+	}
+	
 	// aggregates
 	reflectType := reflect.TypeOf(value)
 	switch reflectType.Kind() {
@@ -252,8 +263,8 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 		// not we receive an empty any value) actually makes it fucking
 		// work. go figure().
 		//
-		// (the map allocation functionality in skeletonPointer has been
+		// (the map allocation functionality in skeletonPointer was
-		// removed)
+		// removed after this comment was written)
 		value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast)
 		destination.Set(value)
 		destination = value
@@ -298,12 +309,24 @@ func tagAny(reflectValue reflect.Value) (Tag, error) {
 	case reflect.Uint64:  return LI.WithCN(7), nil
 	case reflect.Float32: return FP.WithCN(3), nil
 	case reflect.Float64: return FP.WithCN(7), nil
+	case reflect.Bool:
+		if reflectValue.Bool() {
+			return SI.WithCN(1), nil
+		} else {
+			return SI.WithCN(0), nil
+		}
 	case reflect.String:  return bufferLenTag(reflectValue.Len()), nil
 	}
 	if reflectValue.CanConvert(reflect.TypeOf(dummyBuffer)) {
 		return bufferLenTag(reflectValue.Len()), nil
 	}
 
+	// option
+	if isTypeOption(reflectValue.Type()) {
+		elem, _ := optionValue(reflectValue) // zero value for free
+		return tagAny(elem)
+	}
+	
 	// aggregates
 	reflectType := reflectValue.Type()
 	switch reflectType.Kind() {
@@ -329,9 +352,14 @@ func encodeAnySlice(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	for index := 0; index < reflectValue.Len(); index += 1 {
 		itemTag, err := tagAny(reflectValue.Index(index))
 		if err != nil { return n, err }
+		if itemTag.Is(SBA) {
+			// SBA data in an LBA will always have the tag LBA:0,
+			// because 32 <= 256
+			continue
+		}
 		if itemTag.CN() > oneTag.CN() { oneTag = itemTag }
 	}
-	if oneTag.Is(SBA) { oneTag += 1 << 5 }
+	if oneTag.Is(SBA) { oneTag = LBA.WithCN(oneTag.CN()) }
 	nn, err = encoder.WriteUint8(uint8(oneTag))
 	n += nn; if err != nil { return n, err }
 	for index := 0; index < reflectValue.Len(); index += 1 {
@@ -374,7 +402,8 @@ func canSet(destination reflect.Type, tag Tag) error {
 		switch destination.Kind() {
 		case
 			reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
-			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
+			reflect.Bool:
 		default:
 			return errCantAssignf("cannot assign integer to %v", destination)
 		}
@@ -432,6 +461,8 @@ func setInt(destination reflect.Value, value int64, bytes int) {
 // setUint expects a settable destination.
 func setUint(destination reflect.Value, value uint64, bytes int) {
 	switch {
+	case destination.Kind() == reflect.Bool:
+		destination.Set(reflect.ValueOf(value > 0))
 	case destination.CanInt():
 		destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type()))
 	case destination.CanUint():
@@ -566,6 +597,21 @@ func isTypeAny(typ reflect.Type) bool {
 	return typ.Kind() == reflect.Interface && typ.NumMethod() == 0
 }
 
+// isTypeOption returns whether the given reflect.Type is a ucontainer.Option,
+// and returns the element type if true.
+func isTypeOption(typ reflect.Type) bool {
+	// TODO: change when needed
+	goutilPath := "git.tebibyte.media/sashakoshka/go-util"
+	return typ.Name() == "Option" && typ.PkgPath() == goutilPath + "/container"
+}
+
+// optionValue returns the value of an option. The value MUST be an option, or
+// this function will panic.
+func optionValue(value reflect.Value) (elem reflect.Value, ok bool) {
+	result := value.MethodByName("Value").Call([]reflect.Value { })
+	return result[0], result[1].Bool()
+}
+
 // peekSlice returns the element tag and dimension count of the OTA currently
 // being decoded. It does not use up the decoder, it only peeks.
 func peekSlice(decoder *Decoder, tag Tag) (Tag, int, error) {

tape/dynamic_test.go

@@ -3,37 +3,10 @@ package tape
 import "bytes"
 import "testing"
 import "reflect"
+import "git.tebibyte.media/sashakoshka/go-util/container"
 import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
 
-var samplePayloads = [][]byte {
+var samplePayloads [][]byte
-	/* int8     */ []byte { byte(LSI.WithCN(0)), 0x45 },
-	/* int16    */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 },
-	/* int32    */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-	/* int64    */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-	/* uint5    */ []byte { byte(SI.WithCN(12)) },
-	/* uint8    */ []byte { byte(LI.WithCN(0)), 0x45 },
-	/* uint16   */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 },
-	/* uint32   */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-	/* uint64   */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-	/* string   */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' },
-	/* []byte   */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' },
-	/* []string */ []byte {
-		byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)),
-		0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
-		0x05, 0x11, 0x11, 0x11, 0x11, 0x11,
-	},
-	/* map[uint16] any */ []byte {
-		byte(KTV.WithCN(0)), 2,
-		0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67,
-		0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB,
-	},
-	/* map[uint16] any */ []byte {
-		byte(KTV.WithCN(0)), 3,
-		0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00,
-		0x00, 0x02, 0x82, 'h', 'i',
-		0x00, 0x03, 0x21, 0x39, 0x92,
-	},
-}
 
 var sampleValues = []any {
 	/* int8     */ int8(0x45),
@@ -45,6 +18,8 @@ var sampleValues = []any {
 	/* uint16   */ uint16(0x4567),
 	/* uint32   */ uint32(0x456789AB),
 	/* uint64   */ uint64(0x456789ABCDEF0123),
+	/* bool     */ false,
+	/* bool     */ true,
 	/* string   */ "pupever",
 	/* []byte   */ "blarg",
 	/* []string */ []string {
@@ -60,6 +35,64 @@ var sampleValues = []any {
 		0x0002: "hi",
 		0x0003: uint16(0x3992),
 	},
+	// IMPORTANT: ONLY ADD AT THE END!!!! DO NOT MOVE WHAT IS ALREADY HERE!
+	// IMPORTANT: ONLY ADD AT THE END!!!! DO NOT MOVE WHAT IS ALREADY HERE!
+}
+
+type sample struct {
+	t Tag
+	v any
+	s tu.Snake
+}
+
+var samples = []sample {
+	/* int8     */ sample { t: LSI.WithCN(0), s: tu.S(0x45) },
+	/* int16    */ sample { t: LSI.WithCN(1), s: tu.S(0x45, 0x67) },
+	/* int32    */ sample { t: LSI.WithCN(3), s: tu.S(0x45, 0x67, 0x89, 0xAB) },
+	/* int64    */ sample { t: LSI.WithCN(7), s: tu.S(0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23) },
+	/* uint5    */ sample { t: SI.WithCN(12), s: tu.S() },
+	/* uint8    */ sample { t: LI.WithCN(0),  s: tu.S(0x45) },
+	/* uint16   */ sample { t: LI.WithCN(1),  s: tu.S(0x45, 0x67) },
+	/* uint32   */ sample { t: LI.WithCN(3),  s: tu.S(0x45, 0x67, 0x89, 0xAB) },
+	/* uint64   */ sample { t: LI.WithCN(7),  s: tu.S(0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23) },
+	/* bool     */ sample { t: SI.WithCN(0),  s: tu.S() },
+	/* bool     */ sample { t: SI.WithCN(1),  s: tu.S() },
+	/* string   */ sample { t: SBA.WithCN(7), s: tu.S('p', 'u', 'p', 'e', 'v', 'e', 'r') },
+	/* []byte   */ sample { t: SBA.WithCN(5), s: tu.S('b', 'l', 'a', 'r', 'g') },
+	/* []string */ sample {
+		t: OTA.WithCN(0),
+		s: tu.S(2, byte(LBA.WithCN(0)),
+			0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
+			0x05, 0x11, 0x11, 0x11, 0x11, 0x11),
+	},
+	/* map[uint16] any */ sample {
+		t: KTV.WithCN(0),
+		s: tu.S(2).AddVar(
+			[]byte { 0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67 },
+			[]byte { 0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB }),
+	},
+	/* map[uint16] any */ sample {
+		t: KTV.WithCN(0),
+		s: tu.S(3).AddVar(
+			[]byte { 0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00 },
+			[]byte { 0x00, 0x02, 0x82, 'h', 'i' },
+			[]byte { 0x00, 0x03, 0x21, 0x39, 0x92 }),
+	},
+}
+
+var sampleOptionValues []any
+
+func init() {
+	sampleOptionValues = make([]any, len(sampleValues))
+	for index, value := range sampleValues {
+		sampleOptionValues[index] = ucontainer.O(value)
+		samples[index].v = value
+	}
+	samplePayloads = make([][]byte, len(samples))
+	for index, sample := range samples {
+		item := append([]byte { byte(sample.t) }, sample.s.Flatten()...)
+		samplePayloads[index] = item
+	}
 }
 
 type userDefinedInteger int16
@@ -84,7 +117,9 @@ func TestEncodeAnyTable(test *testing.T) {
 		0x3456: userDefinedInteger(0x3921),
 		0x1F1F: float32(67.26),
 		0x0F0F: float64(5.3),
-	}, KTV.WithCN(0), tu.S(9).AddVar(
+		0xAAAA: false,
+		0xBBBB: true,
+	}, KTV.WithCN(0), tu.S(11).AddVar(
 		[]byte {
 			0xF3, 0xB9,
 			byte(LSI.WithCN(3)),
@@ -138,6 +173,14 @@ func TestEncodeAnyTable(test *testing.T) {
 			byte(FP.WithCN(7)),
 			0x40, 0x15, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33,
 		},
+		[]byte {
+			0xAA, 0xAA,
+			byte(SI.WithCN(0)),
+		},
+		[]byte {
+			0xBB, 0xBB,
+			byte(SI.WithCN(1)),
+		},
 	))
 	if err != nil { test.Fatal(err) }
 }
@@ -146,7 +189,7 @@ func TestDecodeWrongType(test *testing.T) {
 	for index, data := range samplePayloads {
 		test.Logf("data %2d %v [%s]", index, Tag(data[0]), tu.HexBytes(data[1:]))
 		// integers should only assign to other integers
-		if index > 8 {
+		if index > 10 {
 			cas := func(destination any) {
 				n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
 				if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
@@ -155,7 +198,7 @@ func TestDecodeWrongType(test *testing.T) {
 					if reflectValue.Int() != 0 {
 						test.Fatalf("destination not zero: %v", reflectValue.Elem().Interface())
 					}
-				} else {
+				} else if reflectValue.Kind() != reflect.Bool {
 					if reflectValue.Uint() != 0 {
 						test.Fatalf("destination not zero: %v", reflectValue.Elem().Interface())
 					}
@@ -180,6 +223,8 @@ func TestDecodeWrongType(test *testing.T) {
 			{ var dest uint32; cas(&dest) }
 			test.Log("- uint64")
 			{ var dest uint64; cas(&dest) }
+			test.Log("- bool")
+			{ var dest bool; cas(&dest) }
 		}
 		arrayCase := func(destination any) {
 			n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
@@ -194,19 +239,19 @@ func TestDecodeWrongType(test *testing.T) {
 			}
 		}
 		// SBA/LBA types should only assign to other SBA/LBA types
-		if index != 9 && index != 10 {
+		if index != 11 && index != 12 {
 			test.Log("- string")
 			{ var dest string; arrayCase(&dest) }
 			test.Log("- []byte")
 			{ var dest []byte; arrayCase(&dest) }
 		}
 		// arrays should only assign to other arrays
-		if index != 11 {
+		if index != 13 {
 			test.Log("- []string")
 			{ var dest []string; arrayCase(&dest) }
 		}
 		// tables should only assign to other tables
-		if index != 12 && index != 13 {
+		if index != 14 && index != 15 {
 			test.Log("- map[uint16] any")
 			{ var dest = map[uint16] any { }; arrayCase(&dest) }
 		}
@@ -231,6 +276,12 @@ func TestEncodeDecodeAnyTable(test *testing.T) {
 func TestEncodeDecodeAnyDestination(test *testing.T) {
 	var destination any
 	for index, data := range samplePayloads {
+		if _, isBool := sampleValues[index].(bool); isBool {
+			// test is invalid for bools because they are never
+			// created as a skeleton value
+			continue
+		}
+	
 		tag     := Tag(data[0])
 		payload := data[1:]
 		test.Logf("data %2d %v [%s]", index, tag, tu.HexBytes(payload))
@@ -249,6 +300,20 @@ func TestEncodeDecodeAnyDestination(test *testing.T) {
 	}
 }
 
+func TestEncodeOption(test *testing.T) {
+	for _, sample := range samples {
+		snake   := sample.s
+		tag     := sample.t
+		value   := sample.v
+		if _, ok := value.(bool); tag.Is(SI) && !ok {
+			// we will never encode an SI unless its a bool
+			continue
+		}
+		err := testEncodeAny(test, value, tag, snake)
+		if err != nil { test.Fatal(err) }
+	}
+}
+
 func TestPeekSlice(test *testing.T) {
 	buffer := bytes.NewBuffer([]byte {
 		2, byte(OTA.WithCN(3)),
@@ -349,6 +414,12 @@ func TestTagAny(test *testing.T) {
 
 func TestDecodeAny(test *testing.T) {
 	for index, payload := range samplePayloads {
+		if _, isBool := sampleValues[index].(bool); isBool {
+			// test is invalid for bools because they are never
+			// created as a skeleton value
+			continue
+		}
+		
 		correctValue := sampleValues[index]
 		data := payload[1:]
 		decoder := NewDecoder(bytes.NewBuffer(data))

tape/misc_test.go

@@ -32,7 +32,9 @@ func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.S
 	test.Log("n:      ", n)
 	test.Log("tag:    ", tag)
 	test.Log("got:    ", tu.HexBytes(bytes))
+	test.Log("   :    ", tu.HexChars(bytes))
 	test.Log("correct:", correctBytes)
+	test.Log("       :", correctBytes.CharsString())
 	if tag != correctTag {
 		return fmt.Errorf("tag not equal: %v != %v", tag, correctTag)
 	}
@@ -56,6 +58,7 @@ func testEncodeDecodeAny(test *testing.T, value, correctValue any) error {
 	test.Log("n:  ", n)
 	test.Log("tag:", tag)
 	test.Log("got:", tu.HexBytes(bytes))
+	test.Log("   :", tu.HexChars(bytes))
 	test.Log("decoding...", tag)
 	if n != len(bytes) {
 		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))