Update go-util

Reviewed-on: #22

cmd/hopp-generate/main.go

@@ -1,48 +1,66 @@
 package main
 
 import "os"
-import "fmt"
 import "strings"
 import "path/filepath"
+import "git.tebibyte.media/sashakoshka/go-cli"
 import "git.tebibyte.media/sashakoshka/goparse"
 import "git.tebibyte.media/sashakoshka/hopp/generate"
 
 func main() {
-	name := os.Args[0]
-	if len(os.Args) != 3 {
-		fmt.Fprintf(os.Stderr, "Usage: %s SOURCE DESTINATION\n", name)
+	flagOutput      := cli.NewInputFlag('o', "output", "The output file", "", cli.ValString)
+	flagPackageName := cli.NewInputFlag('p', "package-name", "The package name of the file", "", cli.ValString)
+	command := cli.New("Compile PDL files to program source code",
+		flagOutput,
+		flagPackageName)
+	command.Syntax = "FILE [OPTIONS]..."
+	command.ParseOrExit(os.Args)
+
+	if len(command.Args) != 1 {
+		command.Usage()
 		os.Exit(2)
 	}
-	source      := os.Args[1]
-	destination := os.Args[2]
+	source      := command.Args[0]
+	destination := flagOutput.Value
+	if destination == "" {
+		destination = "protocol.go"
+	}
 
 	input, err := os.Open(source)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 	defer input.Close()
 	protocol, err := generate.ParseReader(source, input)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 
-	absDestination, err := filepath.Abs(destination)
-	handleErr(1, err)
-	packageName := cleanPackageName(strings.ReplaceAll(
-		strings.ToLower(filepath.Base(absDestination)),
-		" ", "_"))
-	destination = filepath.Join(os.Args[2], "generated.go")
+	packageName := flagPackageName.Value
+	if packageName == "" {
+		absDestination, err := filepath.Abs(destination)
+		handleErr(command, 1, err)
+		base := filepath.Base(absDestination)
+		if scrounged, ok := scroungeForPackageName(base); ok {
+			packageName = scrounged
+		} else {
+			packageName = strings.ReplaceAll(
+				strings.ToLower(base),
+				" ", "_")
+		}
+	}
+	packageName = cleanPackageName(packageName)
 
 	output, err := os.Create(destination)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 	generator := generate.Generator {
 		Output:      output,
 		PackageName: packageName,
 	}
 	_, err = generator.Generate(protocol)
-	handleErr(1, err)
-	fmt.Fprintf(os.Stderr, "%s: OK\n", name)
+	handleErr(command, 1, err)
+	command.Println(destination, "OK")
 }
 
-func handleErr(code int, err error) {
+func handleErr(command *cli.Cli, code int, err error) {
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[0], parse.Format(err))
+		command.Errorln(parse.Format(err))
 		os.Exit(code)
 	}
 }
@@ -61,3 +79,32 @@ func cleanPackageName(str string) string {
 	}
 	return string(buffer[:j])
 }
+
+func scroungeForPackageName(dir string) (string, bool) {
+	entries, err := os.ReadDir(dir)
+	if err != nil { return "", false}
+	for _, entry := range entries {
+		if !entry.Type().IsRegular() { continue }
+		file, err := os.Open(filepath.Join(dir, entry.Name()))
+		if err != nil { continue }
+		defer file.Close()
+		// FIXME: it is entirely possible that the only file will have
+		// a shitload of doc comments preceeding the package name, and
+		// those comments are usually huge so this is bad
+		buffer := [512]byte { }
+		n, _ := file.Read(buffer[:])
+		text := string(buffer[:n])
+
+		packageIndex := strings.Index(text, "package")
+		if packageIndex < 0 { continue }
+		text = text[packageIndex:]
+		
+		newlineIndex := strings.Index(text, "\n")
+		if packageIndex > 0 { text = text[:newlineIndex] }
+
+		fields := strings.Fields(text)
+		if len(fields) < 2 { continue }
+		return fields[1], true
+	}
+	return "", false
+}

design/pdl-language.md

@@ -6,7 +6,7 @@ PDL allows defining a protocol using HOPP and TAPE.
 
 | Syntax     | TN      | CN | Description
 | ---------- | ------- | -: | -----------
-| I5         | SI      |    |   
+| I5         | SI      |    | 
 | I8         | LSI     |  0 | 
 | I16        | LSI     |  1 | 
 | I32        | LSI     |  3 | 
@@ -25,6 +25,7 @@ PDL allows defining a protocol using HOPP and TAPE.
 | F64        | FP      |  7 | 
 | F128[^2]   | FP      | 15 | 
 | F256[^2]   | FP      | 31 | 
+| Bool       | SI      |    | 
 | String     | SBA/LBA |  * | UTF-8 string
 | Buffer     | SBA/LBA |  * | Byte array
 | []\<TYPE\> | OTA     |  * | Array of any type[^1]
@@ -32,6 +33,11 @@ PDL allows defining a protocol using HOPP and TAPE.
 | {...}      | KTV     |  * | Table with defined schema
 | Any        | *       |  * | Value of an undefined type
 
+Tables with a defined schema can specify some fields as optional using a
+question mark before the type. This will wrap the field the go-util
+ucontainer.Option type. When encoding, void fields will not be included in the
+output, and when decoding, unspecified fields are left void.
+
 [^1]: Excluding SI and SBA. I5 and U5 cannot be used in an array, but String and
 Buffer are simply forced to use their "long" variant.
 
@@ -47,11 +53,13 @@ structures. They are separated by whitespace.
 | Method   | `M[0-9A-Fa-f]{4}`  | A 16-bit hexadecimal method code.
 | Key      | `[0-9A-Fa-f]{4}`   | A 16-bit hexadecimal table key.
 | Ident    | `[A-Z][A-Za-z0-9]` | An identifier.
+| Option   | `?`                | A question mark.
 | Comma    | `,`                | A comma separator.
 | LBrace   | `{`                | A left curly brace.
 | RBrace   | `}`                | A right curly brace.
 | LBracket | `[`                | A left square bracket.
 | RBracket | `]`                | A right square bracket.
+| Comment  | `\/\/.*$`          | A doc comment starting with a double-slash.
 
 ## Syntax
 
@@ -68,18 +76,27 @@ an Ident token respectively. A message consists of the method code (Method), the
 message name (Ident), and the message's root type. This is usually a table, but
 can be anything.
 
+Messages, types, and table fields can all have doc comments preceding them,
+which are used to generate documentation for the protocol. The syntax is the
+same as Go's (for now). Comments aren't allowed anywhere else.
+
 Here is an example of all that:
 
 ```
+// Connect is sent from the client to the server as the first message of an
+// authenticated transaction.
 M0000 Connect {
 	0000 Name String,
 	0001 Password String,
 }
 
+// UserList is sent from the server to the client in response to a Connect
+// message.
 M0001 UserList {
 	0000 Users []User,
 }
 
+// User holds profile information about a single user.
 User {
 	0000 Name      String,
 	0001 Bio       String,
@@ -96,10 +113,10 @@ Below is an EBNF description of the language.
 <method>  -> /M[0-9A-Fa-f]{4}/
 <key>     -> /[0-9A-Fa-f]{4}/
 <ident>   -> /[A-Z][A-Za-z0-9]/
-<field>   -> <key> <ident> <type>
+<field>   -> <key> <ident> ["?"] <type>
 <type> -> <ident>
         | "[" "]" <type>
-        | "{" (<field> ",")* [<field>] "}"
-<message> -> <method> <ident> <type>
-<typedef> -> <ident> <type>
+        | "{" (<comment>* <field> ",")* [<comment>* <field>] "}"
+<message> -> <comment>* <method> <ident> <type>
+<typedef> -> <comment>* <ident> <type>
 ```

design/protocol.md

@@ -75,16 +75,16 @@ connection. Thus, the value may range from 0 to 31 if unsigned, and from -16 to
 17 if signed.
 
 #### Large Integer (LI)
-LI encodes an integer of up to 256 bits, which are stored in the payload. The CN
-determine the length of the payload in bytes. The integer is big-endian. Whether
+LI encodes an integer of up to 256 bits, which are stored in the payload. The
+length of the payload (in bytes) is CN + 1. The integer is big-endian. Whether
 the payload is interpreted as unsigned or as signed two's complement is semantic
 information and must be agreed upon by both sides of the connection. Thus, the
 value may range from 0 to 31 if unsigned, and from -16 to 17 if signed.
 
 #### Floating Point (FP)
 FP encodes an IEEE 754 floating point number of up to 256 bits, which are stored
-in the payload. The CN determines the length of the payload in bytes, and it may
-only be one of these values: 16, 32, 64, 128, or 256.
+in the payload. The length of the payload (in bytes) is CN + 1. The only
+supported bit widths for floats are as follows: 16, 32, 64, 128, and 256.
 
 #### Small Byte Array (SBA)
 SBA encodes an array of up to 32 bytes, which are stored in the paylod. The
@@ -98,15 +98,16 @@ in bytes is determined by the CN.
 #### One-Tag Array (OTA)
 OTA encodes an array of up to 2^256 items, which are stored in the payload after
 the length field and the item tag, where the length field comes first. Each item
-must be the same length, as they all share the same tag. The length of the data
-length field in bytes is determined by the CN.
+must be the same length, as they all share the same tag. The length of the
+length field (in bytes) is CN + 1.
 
 #### Key-Tag-Value Table (KTV)
 KTV encodes a table of up to 2^256 key/value pairs, which are stored in the
 payload after the length field. The pairs themselves consist of a 16-bit
 unsigned big-endian key followed by a tag and then the payload. Pair values can
 be of different types and sizes. The order of the pairs is not significant and
-should never be treated as such.
+should never be treated as such. The length of the length field (in bytes) is
+CN + 1.
 
 ## Transports
 A transport is a protocol that HOPP connections can run on top of. HOPP

generate/generate.go

@@ -11,6 +11,7 @@ import "git.tebibyte.media/sashakoshka/hopp/tape"
 
 const imports =
 `
+import "git.tebibyte.media/sashakoshka/hopp"
 import "git.tebibyte.media/sashakoshka/hopp/tape"
 `
 
@@ -24,7 +25,7 @@ const preamble = `
 
 const static = `
 // Table is a KTV table with an undefined schema.
-type Table map[uint16] any
+type Table = map[uint16] any
 
 // Message is any message that can be sent along this protocol.
 type Message interface {
@@ -47,6 +48,17 @@ func canAssign(destination, source tape.Tag) bool {
 	}
 	return false
 }
+
+// boolInt converts a bool to an integer.
+func boolInt(input bool) int {
+	if input {
+		return 1
+	} else {
+		return 0
+	}
+}
+
+var _ hopp.Option[int]
 `
 
 // Generator converts protocols into Go code.
@@ -112,13 +124,20 @@ func (this *Generator) Generate(protocol *Protocol) (n int, err error) {
 	return n, nil
 }
 
-func (this *Generator) generateTypedef(name string, typ Type) (n int, err error) {
+func (this *Generator) generateTypedef(name string, typedef Typedef) (n int, err error) {
+	typ := typedef.Type
+
 	// type definition
-	nn, err := this.iprintf(
-		"\n// %s represents the protocol data type %s.\n",
-		name, name)
-	n += nn; if err != nil { return n, err }
-	nn, err = this.iprintf("type %s ", name)
+	if typedef.Doc == "" {
+		nn, err := this.iprintf(
+			"\n// %s represents the protocol data type %s.\n",
+			name, name)
+		n += nn; if err != nil { return n, err }
+	} else {
+		nn, err := this.iprintf("\n%s\n", this.formatComment(typedef.Doc))
+		n += nn; if err != nil { return n, err }
+	}
+	nn, err := this.iprintf("type %s ", name)
 	n += nn; if err != nil { return n, err }
 	nn, err = this.generateType(typ)
 	n += nn; if err != nil { return n, err }
@@ -208,10 +227,16 @@ func (this *Generator) generateTypedef(name string, typ Type) (n int, err error)
 // generateMessage generates the structure, as well as encoding decoding
 // functions for the given message.
 func (this *Generator) generateMessage(method uint16, message Message) (n int, err error) {
-	nn, err := this.iprintf(
-		"\n// %s represents the protocol message M%04X %s.\n",
-		message.Name, method, message.Name)
-	nn, err = this.iprintf("type %s ", this.resolveMessageName(message.Name))
+	if message.Doc == "" {
+		nn, err := this.iprintf(
+			"\n// %s represents the protocol message M%04X %s.\n",
+			message.Name, method, message.Name)
+		n += nn; if err != nil { return n, err }
+	} else {
+		nn, err := this.iprintf("\n%s\n", this.formatComment(message.Doc))
+		n += nn; if err != nil { return n, err }
+	}
+	nn, err := this.iprintf("type %s ", this.resolveMessageName(message.Name))
 	n += nn; if err != nil { return n, err }
 	nn, err = this.generateType(message.Type)
 	n += nn; if err != nil { return n, err }
@@ -302,6 +327,9 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e
 //   - nn int
 func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource string) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		// SI: (none)
+		// SI stores the value in the tag, so we write nothing here
 	case TypeInt:
 		// SI:     (none)
 		// LI/LSI: <value: IntN>
@@ -372,7 +400,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		nn, err = this.iprintf("}\n")
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf(
-			"nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN())\n",
+			"nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN() + 1)\n",
 			valueSource, tagSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -407,8 +435,8 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 			n += nn; if err != nil { return n, err }
 			nn, err = this.iprintf("nn, err = encoder.WriteTag(itemTag)\n")
 			n += nn; if err != nil { return n, err }
-			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
+			n += nn; if err != nil { return n, err }
 			nn, err = this.iprintf("for _, item := range %s {\n", valueSource)
 			n += nn; if err != nil { return n, err }
 			this.push()
@@ -445,7 +473,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		nn, err = this.iprintf("}\n")
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf(
-			"nn, err = encoder.WriteUintN(%d, %s.CN())\n",
+			"nn, err = encoder.WriteUintN(%d, %s.CN() + 1)\n",
 			len(typ.Fields), tagSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -454,19 +482,33 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		n += nn; if err != nil { return n, err }
 		this.push()
 			for key, field := range typ.Fields {
+				fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name)
+				if field.Option {
+					nn, err = this.iprintf("if value, ok := %s.Value(); ok {\n", fieldSource)
+					n += nn; if err != nil { return n, err }
+					fieldSource = "value"
+					this.push()
+				}
+
 				nn, err = this.iprintf("nn, err = encoder.WriteUint16(0x%04X)\n", key)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
-				fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name)
 				tagVar, nn, err := this.generateTag(field.Type, fieldSource)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(%s))\n", tagVar)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
+				
 				nn, err = this.generateEncodeValue(field.Type, fieldSource, tagVar)
 				n += nn; if err != nil { return n, err }
+				
+				if field.Option {
+					this.pop()
+					nn, err = this.iprintf("}\n")
+					n += nn; if err != nil { return n, err }
+				}
 			}
 		this.pop()
 		nn, err = this.iprintf("}\n")
@@ -505,6 +547,11 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 // for [Generator.generateDecodeBranch].
 func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagSource string) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		// SI: (none)
+		// SI stores the value in the tag
+		nn, err := this.iprintf("*%s = %s.CN() > 0\n", valueSource, tagSource)
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		// SI:     (none)
 		// LI/LSI: <value: IntN>
@@ -524,13 +571,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 			prefix = "ReadInt"
 		}
 		destinationVar := this.newTemporaryVar("destination")
-		nn, err := this.iprintf("var %s ", destinationVar)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.generateType(typ)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.print("\n")
-		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("%s, nn, err = decoder.%s%d()\n", destinationVar, prefix, typ.Bits)
+		nn, err := this.iprintf("%s, nn, err := decoder.%s%d()\n", destinationVar, prefix, typ.Bits)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
@@ -544,13 +585,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 	case TypeFloat:
 		// FP: <value: FloatN>
 		destinationVar := this.newTemporaryVar("destination")
-		nn, err := this.iprintf("var %s ", destinationVar)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.generateType(typ)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.print("\n")
-		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("%s, nn, err = decoder.ReadFloat%d()\n", destinationVar, typ.Bits)
+		nn, err := this.iprintf("%s, nn, err := decoder.ReadFloat%d()\n", destinationVar, typ.Bits)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
@@ -709,7 +744,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar)
+			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()) + 1)\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
 			n += nn; if err != nil { return n, err }
@@ -773,6 +808,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
 		case TypeTableDefined:
+			// TODO: options
 			// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
 			// read header
 			lengthVar := this.newTemporaryVar("length")
@@ -785,7 +821,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar)
+			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()) + 1)\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
 			n += nn; if err != nil { return n, err }
@@ -849,10 +885,25 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 					n += nn; if err != nil { return n, err }
 
 					// decode payload
-					nn, err = this.generateDecodeValue(
-						field.Type, "",
-						fmt.Sprintf("(&(this.%s))", field.Name), fieldTagVar)
-					n += nn; if err != nil { return n, err }
+					if field.Option {
+						destination := this.newTemporaryVar("destination")
+						nn, err = this.iprintf("var %s ", destination)
+						n += nn; if err != nil { return n, err }
+						nn, err = this.generateType(field.Type)
+						n += nn; if err != nil { return n, err }
+						nn, err = this.printf("\n")
+						n += nn; if err != nil { return n, err }
+						nn, err = this.generateDecodeValue(
+							field.Type, "", fmt.Sprintf("(&%s)", destination), fieldTagVar)
+						n += nn; if err != nil { return n, err }
+						nn, err = this.iprintf("this.%s = hopp.O(%s)\n", field.Name, destination)
+						n += nn; if err != nil { return n, err }
+					} else {
+						nn, err = this.generateDecodeValue(
+							field.Type, "",
+							fmt.Sprintf("(&(this.%s))", field.Name), fieldTagVar)
+						n += nn; if err != nil { return n, err }
+					}
 					this.pop()
 				}					
 				nn, err = this.iprintf("default:\n")
@@ -865,16 +916,6 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			
-			// TODO once options are implemented, have a set of
-			// bools for each non-optional field, and check here
-			// that they are all true. a counter will not work
-			// because if someone specifies a non-optional field
-			// twice, they can neglect to specify another
-			// non-optional field and we won't even know because the
-			// count will still be even. we shouldn't use a map
-			// either because its an allocation and its way more
-			// memory than just, like 5 bools (on the stack no less)
 		default: return n, fmt.Errorf("unexpected type: %T", typ)
 		}
 
@@ -931,6 +972,9 @@ func (this *Generator) generateBareErrorCheck() (n int, err error) {
 func (this *Generator) generateTag(typ Type, source string) (tagVar string, n int, err error) {
 	tagVar = this.newTemporaryVar("tag")
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.iprintf("%s := tape.SI.WithCN(boolInt(bool(%s)))\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeInt:
 		if typ.Bits <= 5 {
 			nn, err := this.iprintf("%s := tape.SI.WithCN(int(%s))\n", tagVar, source)
@@ -952,13 +996,13 @@ func (this *Generator) generateTag(typ Type, source string) (tagVar string, n in
 		nn, err := this.iprintf("%s := tape.BufferTag([]byte(%s))\n", tagVar, source)
 		n += nn; if err != nil { return tagVar, n, err }
 	case TypeArray:
-		nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source)
+		nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))) - 1)\n", tagVar, source)
 		n += nn; if err != nil { return tagVar, n, err }
 	case TypeTable:
-		nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source)
+		nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))) - 1)\n", tagVar, source)
 		n += nn; if err != nil { return tagVar, n, err }
 	case TypeTableDefined:
-		nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields))))
+		nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields))) - 1)
 		n += nn; if err != nil { return tagVar, n, err }
 	case TypeNamed:
 		resolved, err := this.resolveTypeName(typ.Name)
@@ -984,6 +1028,9 @@ func (this *Generator) generateTag(typ Type, source string) (tagVar string, n in
 // information is chosen.
 func (this *Generator) generateTN(typ Type) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.printf("tape.SI")
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		if typ.Bits <= 5 {
 			nn, err := this.printf("tape.SI")
@@ -1027,6 +1074,9 @@ func (this *Generator) generateTN(typ Type) (n int, err error) {
 
 func (this *Generator) generateType(typ Type) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.printf("bool")
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		if err := this.validateIntBitSize(typ.Bits); err != nil {
 			return n, err
@@ -1090,10 +1140,20 @@ func (this *Generator) generateTypeTableDefined(typ TypeTableDefined) (n int, er
 
 	for _, key := range slices.Sorted(maps.Keys(typ.Fields)) {
 		field := typ.Fields[key]
-		nn, err := this.iprintf("%s ", field.Name)
+		nn, err := this.iprintf("%s\n", this.formatComment(field.Doc))
 		n += nn; if err != nil { return n, err }
+		nn, err = this.iprintf("%s ", field.Name)
+		n += nn; if err != nil { return n, err }
+		if field.Option {
+			nn, err = this.print("hopp.Option[")
+			n += nn; if err != nil { return n, err }
+		}
 		nn, err = this.generateType(field.Type)
 		n += nn; if err != nil { return n, err }
+		if field.Option {
+			nn, err = this.print("]")
+			n += nn; if err != nil { return n, err }
+		}
 		nn, err = this.print("\n")
 		n += nn; if err != nil { return n, err }
 	}
@@ -1175,17 +1235,21 @@ func (this *Generator) iprintf(format string, args ...any) (n int, err error) {
 	return fmt.Fprintf(this.Output, this.indent() + format, args...)
 }
 
+func (this *Generator) formatComment(comment string) string {
+	return "// " + strings.ReplaceAll(comment, "\n", "\n" + this.indent() + "// ")
+}
+
 func (this *Generator) resolveMessageName(message string) string {
 	return "Message" + message
 }
 
 func (this *Generator) resolveTypeName(name string) (Type, error) {	
-	if typ, ok := this.protocol.Types[name]; ok {
-		if typ, ok := typ.(TypeNamed); ok {
+	if typedef, ok := this.protocol.Types[name]; ok {
+		if typ, ok := typedef.Type.(TypeNamed); ok {
 			return this.resolveTypeName(typ.Name)
 		}
 		
-		return typ, nil
+		return typedef.Type, nil
 	}
 	return nil, fmt.Errorf("no type exists called %s", name)
 }

generate/generate_test.go

@@ -59,6 +59,7 @@ func init() {
 				0x000C: Field { Name: "NI16",Type: TypeInt { Bits: 16, Signed: true } },
 				0x000D: Field { Name: "NI32",Type: TypeInt { Bits: 32, Signed: true } },
 				0x000E: Field { Name: "NI64",Type: TypeInt { Bits: 64, Signed: true } },
+				0x000F: Field { Name: "Bool",Type: TypeBool { } },
 			},
 		},
 	}
@@ -83,11 +84,41 @@ func init() {
 			},
 		},
 	}
-	exampleProtocol.Types["User"] = TypeTableDefined {
-		Fields: map[uint16] Field {
-			0x0000: Field { Name: "Name",      Type: TypeString { } },
-			0x0001: Field { Name: "Bio",       Type: TypeString { } },
-			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+	exampleProtocol.Messages[0x0006] = Message {
+		Name: "Option",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "OU5",     Option: true, Type: TypeInt    { Bits: 5  } },
+				0x0001: Field { Name: "OU8",     Option: true, Type: TypeInt    { Bits: 8  } },
+				0x0002: Field { Name: "OU16",    Option: true, Type: TypeInt    { Bits: 16 } },
+				0x0003: Field { Name: "OU32",    Option: true, Type: TypeInt    { Bits: 32 } },
+				0x0004: Field { Name: "OU64",    Option: true, Type: TypeInt    { Bits: 64 } },
+				0x0005: Field { Name: "OI8",     Option: true, Type: TypeInt    { Bits: 8,  Signed: true } },
+				0x0006: Field { Name: "OI16",    Option: true, Type: TypeInt    { Bits: 16, Signed: true } },
+				0x0007: Field { Name: "OI32",    Option: true, Type: TypeInt    { Bits: 32, Signed: true } },
+				0x0008: Field { Name: "OI64",    Option: true, Type: TypeInt    { Bits: 64, Signed: true } },
+				0x0009: Field { Name: "OF32",    Option: true, Type: TypeFloat  { Bits: 32 } },
+				0x000A: Field { Name: "OF64",    Option: true, Type: TypeFloat  { Bits: 64 } },
+				0x000B: Field { Name: "OBool",   Option: true, Type: TypeBool   { } },
+				0x000C: Field { Name: "OString", Option: true, Type: TypeString { } },
+				0x000D: Field { Name: "OArray",  Option: true, Type: TypeArray {
+					Element: TypeInt { Bits: 16, Signed: true } } },
+				0x000E: Field { Name: "OTable",  Option: true, Type: TypeTableDefined {
+					Fields: map[uint16] Field {
+						0x0001: Field { Name: "A", Type: TypeInt   { Bits: 8, Signed: true }, },
+						0x0002: Field { Name: "B", Type: TypeFloat { Bits: 64 }, },
+					} } },
+				0x000F: Field { Name: "T0", Option: true, Type: TypeTable { } },
+			},
+		},
+	}
+	exampleProtocol.Types["User"] = Typedef {
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Bio",       Type: TypeString { } },
+				0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+			},
 		},
 	}
 }
@@ -103,7 +134,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageConnect,
-			tu.S(0xE1, 0x02).AddVar(
+			tu.S(0xE0, 0x02).AddVar(
 				[]byte { 0x00, 0x00, 0x86, 'r', 'a', 'r', 'i', 't', 'y' },
 				[]byte { 0x00, 0x01, 0x84, 'g', 'e', 'm', 's' },
 				))
@@ -129,8 +160,8 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageUserList,
-			tu.S(0xE1, 0x01, 0x00, 0x00,
-				0xC1, 0x03, 0xE1,
+			tu.S(0xE0, 0x01, 0x00, 0x00,
+				0xC0, 0x03, 0xE0,
 					).Add(0x03).AddVar(
 						[]byte { 0x00, 0x00, 0x86, 'r', 'a', 'r', 'i', 't', 'y' },
 						[]byte { 0x00, 0x01, 0x87, 'a', 's', 'd', 'j', 'a', 'd', 's' },
@@ -155,7 +186,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messagePulse,
-			tu.S(0xE1, 0x05).AddVar(
+			tu.S(0xE0, 0x05).AddVar(
 				[]byte { 0x00, 0x00, 0x09 },
 				[]byte { 0x00, 0x01, 0x41, 0xCA, 0xDF },
 				[]byte { 0x00, 0x02, 0x61, 0x51, 0xAC },
@@ -176,7 +207,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageNestedArray,
-			tu.S(0xC1, 0x02, 0xC1,
+			tu.S(0xC0, 0x02, 0xC0,
 				0x06, 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6,
 				35,   0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6,
 				            0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC,
@@ -199,10 +230,11 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 			NI16: -0x34C9,
 			NI32: -0x10E134C9,
 			NI64: -0x639109BC10E134C9,
+			Bool: true,
 		}
 		testEncodeDecode(
 			&messageIntegers,
-			tu.S(0xE1, 13).AddVar(
+			tu.S(0xE0, 14).AddVar(
 				[]byte { 0x00, 0x00, 0x13 },
 				[]byte { 0x00, 0x01, 0x20, 0xC9 },
 				[]byte { 0x00, 0x02, 0x21, 0x34, 0xC9 },
@@ -216,6 +248,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 				[]byte { 0x00, 0x0C, 0x41, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0D, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0E, 0x47, 0x9C, 0x6E, 0xF6, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
+				[]byte { 0x00, 0x0F, 0x01 },
 				))
 		log.Println("MessageDynamic")
 		messageDynamic := MessageDynamic {
@@ -243,7 +276,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageDynamic,
-			tu.S(0xE1, 14).AddVar(
+			tu.S(0xE0, 14).AddVar(
 				[]byte { 0x00, 0x00, 0x20, 0x23 },
 				[]byte { 0x00, 0x01, 0x21, 0x32, 0x47 },
 				[]byte { 0x00, 0x02, 0x23, 0x87, 0x32, 0x45, 0x23 },
@@ -255,11 +288,15 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 				[]byte { 0x00, 0x08, 0x63, 0x45, 0x12, 0x63, 0xCE },
 				[]byte { 0x00, 0x09, 0x67, 0x40, 0x74, 0x4E, 0x3D, 0x6F, 0xCD, 0x17, 0x75 },
 				[]byte { 0x00, 0x0A, 0x87, 'f', 'o', 'x', ' ', 'b', 'e', 'd' },
-				[]byte { 0x00, 0x0B, 0xC4, 0x00, 0x07, 0x00, 0x06, 0x00, 0x05, 0x00, 0x04 },
-				[]byte { 0x00, 0x0C, 0xE1, 0x02,
-					0x00, 0x01, 0x20, 0x08,
+				[]byte { 0x00, 0x0B, 0xC0, 0x04, 0x41,
+					0x00, 0x07,
+					0x00, 0x06,
+					0x00, 0x05,
+					0x00, 0x04 },
+				[]byte { 0x00, 0x0C, 0xE0, 0x02,
+					0x00, 0x01, 0x40, 0x08,
 					0x00, 0x02, 0x67, 0x40, 0x11, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A },
-				[]byte { 0x00, 0x0D, 0xE1, 0x03,
+				[]byte { 0x00, 0x0D, 0xE0, 0x03, // ERR
 					0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00,
 					0x00, 0x02, 0x82, 'h', 'i',
 					0x00, 0x03, 0x21, 0x39, 0x92 },

generate/lex.go

@@ -10,22 +10,26 @@ const (
 	TokenMethod parse.TokenKind = iota
 	TokenKey
 	TokenIdent
+	TokenOption
 	TokenComma
 	TokenLBrace
 	TokenRBrace
 	TokenLBracket
 	TokenRBracket
+	TokenComment
 )
 
 var tokenNames = map[parse.TokenKind] string {
 	TokenMethod:   "Method",
 	TokenKey:      "Key",
 	TokenIdent:    "Ident",
+	TokenOption:   "Option",
 	TokenComma:    "Comma",
 	TokenLBrace:   "LBrace",
 	TokenRBrace:   "RBrace",
 	TokenLBracket: "LBracket",
 	TokenRBracket: "RBracket",
+	TokenComment:  "Comment",
 }
 
 func Lex(fileName string, reader io.Reader) (parse.Lexer, error) {
@@ -81,6 +85,18 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 		}
 	}
 
+	unexpected := func() error {
+		if unicode.IsPrint(this.rune) {
+			return parse.Errorf (
+				this.pos(), "unexpected rune '%c'",
+				this.rune)
+		} else {
+			return parse.Errorf (
+				this.pos(), "unexpected rune %U",
+				this.rune)
+		}
+	}
+
 	defer func () {
 		newPos := this.pos()
 		newPos.End -- // TODO figure out why tf we have to do this
@@ -108,6 +124,11 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 			if this.eof { err = nil; return }
 			if err != nil { return }
 		}
+	// Option
+	case this.rune == '?':
+		token.Kind = TokenOption
+		appendRune()
+		if this.eof { err = nil; return }
 	// Comma
 	case this.rune == ',':
 		token.Kind = TokenComma
@@ -133,14 +154,21 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 		token.Kind = TokenRBracket
 		appendRune()
 		if this.eof { err = nil; return }
-	case unicode.IsPrint(this.rune):
-		err = parse.Errorf (
-			this.pos(), "unexpected rune '%c'",
-			this.rune)
+	// Comment
+	case this.rune == '/':
+		token.Kind = TokenComment
+		appendRune()
+		if this.eof { return }
+		if this.rune != '/' {
+			err = unexpected()
+			return
+		}
+		for this.rune != '\n' {
+			appendRune()
+			if this.eof { err = nil; return }
+		}
 	default:
-		err = parse.Errorf (
-			this.pos(), "unexpected rune %U",
-			this.rune)
+		err = unexpected()
 	}
 
 	return

generate/lex_test.go

@@ -6,14 +6,22 @@ import "git.tebibyte.media/sashakoshka/goparse"
 
 func TestLex(test *testing.T) {
 	lexer, err := Lex("test.pdl", strings.NewReader(`
+	// User holds profile information about a single user.
 	M0001 User {
 		0000 Name      String,
+		// dog water comment
+		
+		// Users is asdkjsagkj why
+		// 
+		// wow
 		0001 Users []User,
 		0002 Followers U32,
+		0003 Wings ?Int,
 	}`))
 	if err != nil { test.Fatal(parse.Format(err)) }
 
 	correctTokens := []parse.Token {
+		tok(TokenComment,  "// User holds profile information about a single user."),
 		tok(TokenMethod,   "0001"),
 		tok(TokenIdent,    "User"),
 		tok(TokenLBrace,   "{"),
@@ -21,6 +29,10 @@ func TestLex(test *testing.T) {
 		tok(TokenIdent,    "Name"),
 		tok(TokenIdent,    "String"),
 		tok(TokenComma,    ","),
+		tok(TokenComment,  "// dog water comment"),
+		tok(TokenComment,  "// Users is asdkjsagkj why"),
+		tok(TokenComment,  "// "),
+		tok(TokenComment,  "// wow"),
 		tok(TokenKey,      "0001"),
 		tok(TokenIdent,    "Users"),
 		tok(TokenLBracket, "["),
@@ -31,6 +43,11 @@ func TestLex(test *testing.T) {
 		tok(TokenIdent,    "Followers"),
 		tok(TokenIdent,    "U32"),
 		tok(TokenComma,    ","),
+		tok(TokenKey,      "0003"),
+		tok(TokenIdent,    "Wings"),
+		tok(TokenOption,   "?"),
+		tok(TokenIdent,    "Int"),
+		tok(TokenComma,    ","),
 		tok(TokenRBrace,   "}"),
 		tok(parse.EOF,     ""),
 	}

generate/misc_test.go

@@ -100,12 +100,12 @@ func testGenerateRun(test *testing.T, protocol *Protocol, title, imports, testCa
 			log.Println("decoding:")
 			destination := reflect.New(reflect.ValueOf(message).Elem().Type()).Interface().(Message)
 			flat := data.Flatten()
-			log.Println("before: ", destination)
+			log.Println("before: ", tu.Describe(destination))
 			decoder := tape.NewDecoder(bytes.NewBuffer(flat))
 			n, err = destination.Decode(decoder)
 			if err != nil { log.Fatalf("at %d: %v\n", n, err) }
-			log.Println("got:    ", destination)
-			log.Println("correct:", message)
+			log.Println("got:    ", tu.Describe(destination))
+			log.Println("correct:", tu.Describe(message))
 			if n != len(flat) {
 				log.Fatalf("n incorrect: %d != %d\n", n, len(flat))
 			}

generate/parse.go

@@ -1,6 +1,7 @@
 package generate
 
 import "io"
+import "strings"
 import "strconv"
 import "git.tebibyte.media/sashakoshka/goparse"
 
@@ -21,7 +22,7 @@ func Parse(lx parse.Lexer) (*Protocol, error) {
 func defaultProtocol() Protocol {
 	return Protocol {
 		Messages: make(map[uint16] Message),
-		Types: map[string] Type { },
+		Types: map[string] Typedef { },
 	}
 }
 
@@ -47,18 +48,28 @@ func (this *parser) parse() error {
 }
 
 func (this *parser) parseTopLevel() error {
-	err := this.ExpectDesc("message or typedef", TokenMethod, TokenIdent)
-	if err != nil { return err }
-	if this.EOF() { return nil }
+	doc := ""
+	for {
+		err := this.ExpectDesc("message or typedef", TokenMethod, TokenIdent, TokenComment)
+		if err != nil { return err }
+		if this.EOF() { return nil }
+		if this.Kind() == TokenComment {
+			if doc != "" { doc += "\n" }
+			doc += this.parseComment(this.Value())
+			this.Next()
+		} else {
+			break
+		}
+	}
 
 	switch this.Kind() {
-	case TokenMethod: return this.parseMessage()
-	case TokenIdent:  return this.parseTypedef()
+	case TokenMethod: return this.parseMessage(doc)
+	case TokenIdent:  return this.parseTypedef(doc)
 	}
 	panic("bug")
 }
 
-func (this *parser) parseMessage() error {
+func (this *parser) parseMessage(doc string) error {
 	err := this.Expect(TokenMethod)
 	if err != nil { return err }
 	method, err := this.parseHexNumber(this.Value(), 0xFFFF)
@@ -72,12 +83,13 @@ func (this *parser) parseMessage() error {
 	if err != nil { return err }
 	this.protocol.Messages[uint16(method)] = Message {
 		Name: name,
+		Doc:  doc,
 		Type: typ,
 	}
 	return nil
 }
 
-func (this *parser) parseTypedef() error {
+func (this *parser) parseTypedef(doc string) error {
 	err := this.Expect(TokenIdent)
 	if err != nil { return err }
 	name := this.Value()
@@ -85,7 +97,10 @@ func (this *parser) parseTypedef() error {
 	if err != nil { return err }
 	typ, err := this.parseType()
 	if err != nil { return err }
-	this.protocol.Types[name] = typ
+	this.protocol.Types[name] = Typedef {
+		Doc:  doc,
+		Type: typ,
+	}
 	return nil
 }
 
@@ -117,6 +132,7 @@ func (this *parser) parseType() (Type, error) {
 		case "Buffer": return TypeBuffer { }, this.Next()
 		case "Table":  return TypeTable { }, this.Next()
 		case "Any":    return TypeAny { }, this.Next()
+		case "Bool":   return TypeBool { }, this.Next()
 		}
 		return this.parseTypeNamed()
 	case TokenLBracket:
@@ -157,12 +173,22 @@ func (this *parser) parseTypeTable() (TypeTableDefined, error) {
 		Fields: make(map[uint16] Field),
 	}
 	for {
-		err := this.ExpectDesc("table field", TokenKey, TokenRBrace)
-		if err != nil { return TypeTableDefined { }, err }
+		doc := ""
+		for {
+			err := this.ExpectDesc("table field", TokenKey, TokenRBrace, TokenComment)
+			if err != nil { return TypeTableDefined { }, err }
+			if this.Kind() == TokenComment {
+				if doc != "" { doc += "\n" }
+				doc += this.parseComment(this.Value())
+				this.Next()
+			} else {
+				break
+			}
+		}
 		if this.Is(TokenRBrace) {
 			break
 		}
-		key, field, err := this.parseField()
+		key, field, err := this.parseField(doc)
 		if err != nil { return TypeTableDefined { }, err }
 		typ.Fields[key] = field
 		err = this.Expect(TokenComma, TokenRBrace)
@@ -178,7 +204,7 @@ func (this *parser) parseTypeTable() (TypeTableDefined, error) {
 	return typ, nil
 }
 
-func (this *parser) parseField() (uint16, Field, error) {
+func (this *parser) parseField(doc string) (uint16, Field, error) {
 	err := this.Expect(TokenKey)
 	if err != nil { return 0, Field { }, err }
 	key, err := this.parseHexNumber(this.Value(), 0xFFFF)
@@ -188,11 +214,19 @@ func (this *parser) parseField() (uint16, Field, error) {
 	name := this.Value()
 	err = this.Next()
 	if err != nil { return 0, Field { }, err }
+	option := false
+	if this.Kind() == TokenOption {
+		option = true
+		err = this.Next()
+		if err != nil { return 0, Field { }, err }
+	}
 	typ, err := this.parseType()
 	if err != nil { return 0, Field { }, err }
 	return uint16(key), Field {
-		Name: name,
-		Type: typ,
+		Name:   name,
+		Doc:    doc,
+		Type:   typ,
+		Option: option,
 	}, nil
 }
 
@@ -206,3 +240,7 @@ func (this *parser) parseHexNumber(input string, maxValue int64) (int64, error)
 	}
 	return number, nil
 }
+
+func (this *parser) parseComment(input string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(input, "//"), " ")
+}

generate/parse_test.go

@@ -9,6 +9,7 @@ func TestParse(test *testing.T) {
 	correct := defaultProtocol()
 	correct.Messages[0x0000] = Message {
 		Name: "Connect",
+		Doc:  "Connect is sent from the client to the server as the first message of an\nauthenticated transaction.",
 		Type: TypeTableDefined {
 			Fields: map[uint16] Field {
 				0x0000: Field { Name: "Name",      Type: TypeString { } },
@@ -18,36 +19,52 @@ func TestParse(test *testing.T) {
 	}
 	correct.Messages[0x0001] = Message {
 		Name: "UserList",
+		Doc:  "UserList is sent from the server to the client in response to a Connect\nmessage.",
 		Type: TypeTableDefined {
 			Fields: map[uint16] Field {
 				0x0000: Field { Name: "Users",  Type: TypeArray { Element: TypeNamed { Name: "User" } } },
 			},
 		},
 	}
-	correct.Types["User"] = TypeTableDefined {
-		Fields: map[uint16] Field {
-			0x0000: Field { Name: "Name",      Type: TypeString { } },
-			0x0001: Field { Name: "Bio",       Type: TypeString { } },
-			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+	correct.Types["User"] = Typedef {
+		Doc:  "User holds profile information about a single user.",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Bio",       Type: TypeString { } },
+				0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+				0x0003: Field { Name: "Bouncy",    Type: TypeBool { } },
+				0x0004: Field { Name: "Wings",     Type: TypeInt { Bits: 32 } },
+			},
 		},
 	}
-	correct.Types["Anything"] = TypeAny { }
+	correct.Types["Anything"] = Typedef {
+		Type: TypeAny { },
+	}
 	test.Log("CORRECT:", &correct)
 	
 	got, err := ParseReader("test.pdl", strings.NewReader(`
+	// Connect is sent from the client to the server as the first message of an
+	// authenticated transaction.
 	M0000 Connect {
 		0000 Name String,
+		// Password is where you put your secrets, your shameful secrets
 		0001 Password String,
 	}
 	
+	// UserList is sent from the server to the client in response to a Connect
+	// message.
 	M0001 UserList {
 		0000 Users []User,
 	}
 	
+	// User holds profile information about a single user.
 	User {
 		0000 Name      String,
 		0001 Bio       String,
 		0002 Followers U32,
+		0003 Bouncy    Bool,
+		0004 Wings     ?U32,
 	}
 
 	Anything Any

generate/protocol.go

@@ -7,11 +7,17 @@ import "crypto/md5"
 
 type Protocol struct {
 	Messages map[uint16] Message
-	Types    map[string] Type
+	Types    map[string] Typedef
 }
 
 type Message struct {
 	Name string
+	Doc  string
+	Type Type
+}
+
+type Typedef struct {
+	Doc  string
 	Type Type
 }
 
@@ -43,6 +49,12 @@ func (typ TypeFloat) String() string {
 	return fmt.Sprintf("F%d", typ.Bits)
 }
 
+type TypeBool struct { }
+
+func (TypeBool) String() string {
+	return "Bool"
+}
+
 type TypeString struct { }
 
 func (TypeString) String() string {
@@ -83,8 +95,10 @@ func (typ TypeTableDefined) String() string {
 }
 
 type Field struct {
-	Name string
-	Type Type
+	Name   string
+	Doc    string
+	Type   Type
+	Option bool
 }
 
 func (field Field) String() string {

go.mod

@@ -1,8 +1,9 @@
 module git.tebibyte.media/sashakoshka/hopp
 
-go 1.23.0
+go 1.24.0
 
 require (
-	git.tebibyte.media/sashakoshka/go-util v0.9.1
+	git.tebibyte.media/sashakoshka/go-cli v0.1.3
+	git.tebibyte.media/sashakoshka/go-util v0.11.0
 	git.tebibyte.media/sashakoshka/goparse v0.2.0
 )

go.sum

@@ -1,4 +1,6 @@
-git.tebibyte.media/sashakoshka/go-util v0.9.1 h1:eGAbLwYhOlh4aq/0w+YnJcxT83yPhXtxnYMzz6K7xGo=
-git.tebibyte.media/sashakoshka/go-util v0.9.1/go.mod h1:0Q1t+PePdx6tFYkRuJNcpM1Mru7wE6X+it1kwuOH+6Y=
+git.tebibyte.media/sashakoshka/go-cli v0.1.3 h1:tSkWjyx2JrGu6KotbXWSTKSYGGS1D4O3qwCrRoZuwbs=
+git.tebibyte.media/sashakoshka/go-cli v0.1.3/go.mod h1:JFA3wSdRkXxa4iQJWHfe3DokiG7Dh2XUJBzPmuVlbuY=
+git.tebibyte.media/sashakoshka/go-util v0.11.0 h1:ZxLJWHr0ecgVRV5O4MV7EGHK5xHJMppKd1P4x70AtYQ=
+git.tebibyte.media/sashakoshka/go-util v0.11.0/go.mod h1:0Q1t+PePdx6tFYkRuJNcpM1Mru7wE6X+it1kwuOH+6Y=
 git.tebibyte.media/sashakoshka/goparse v0.2.0 h1:uQmKvOCV2AOlCHEDjg9uclZCXQZzq2PxaXfZ1aIMiQI=
 git.tebibyte.media/sashakoshka/goparse v0.2.0/go.mod h1:tSQwfuD+EujRoKr6Y1oaRy74ZynatzkRLxjE3sbpCmk=

internal/testutil/testutil.go

@@ -2,8 +2,9 @@ package testutil
 
 import "fmt"
 import "slices"
-import "strings"
 import "reflect"
+import "strings"
+import "unicode"
 
 // Snake lets you compare blocks of data where the ordering of certain parts may
 // be swapped every which way. It is designed for comparing the encoding of
@@ -96,6 +97,32 @@ func (sn Snake) String() string {
 	return out.String()
 }
 
+func (sn Snake) CharsString() string {
+	if len(sn) == 0 || len(sn[0]) == 0 || len(sn[0][0]) == 0 {
+		return "EMPTY"
+	}
+
+	out := strings.Builder { }
+	for index, sector := range sn {
+		if index > 0 { out.WriteString(" : ") }
+		out.WriteRune('[')
+		for index, variation := range sector {
+			if index > 0 { out.WriteString(" / ") }
+			for _, byt := range variation {
+				run := rune(byt)
+				if unicode.IsPrint(run) && run < 0x7F {
+					out.WriteRune(run)
+				} else {
+					out.WriteRune('.')
+				}
+				out.WriteRune(' ')
+			}
+		}
+		out.WriteRune(']')
+	}
+	return out.String()
+}
+
 // HexBytes formats bytes into a hexadecimal string.
 func HexBytes(data []byte) string {
 	if len(data) == 0 { return "EMPTY" }
@@ -106,6 +133,24 @@ func HexBytes(data []byte) string {
 	return out.String()
 }
 
+// HexChars returns all printable bytes in the string, with non-printable ones
+// replaced with a dot. Each character has an extra space after it for placing
+// underneath the result of HexBytes.
+func HexChars(data []byte) string {
+	if len(data) == 0 { return "EMPTY" }
+	out := strings.Builder { }
+	for _, byt := range data {
+		run := rune(byt)
+		if unicode.IsPrint(run) && run < 0x7F {
+			out.WriteRune(run)
+		} else {
+			out.WriteRune('.')
+		}
+		out.WriteRune(' ')
+	}
+	return out.String()
+}
+
 // Describe returns a string representing the type and data of the given value.
 func Describe(value any) string {
 	desc := describer { }
@@ -124,6 +169,10 @@ func (this *describer) describe(value reflect.Value) {
 		return
 	}
 	value = reflect.ValueOf(value.Interface())
+	if !value.IsValid() {
+		this.printf("<invalid>")
+		return
+	}
 	switch value.Kind() {
 	case reflect.Array, reflect.Slice:
 		this.printf("[\n")
@@ -141,12 +190,20 @@ func (this *describer) describe(value reflect.Value) {
 		typ := value.Type()
 		for index := range typ.NumField() {
 			indexBuffer := [1]int { index }
-			this.iprintf("%s: ", typ.Field(index).Name)
-			this.describe(value.FieldByIndex(indexBuffer[:]))
+			field := typ.Field(index)
+			this.iprintf("%s: ", field.Name)
+			for _, char := range field.Name {
+				if unicode.IsUpper(char) {
+					this.describe(value.FieldByIndex(indexBuffer[:]))
+				} else {
+					this.printf("<private>")
+				}
+				break
+			}
 			this.iprintf("\n")
 		}
 		this.indent -= 1
-		this.iprintf("}\n")
+		this.iprintf("}")
 	case reflect.Map:
 		this.printf("map {\n")
 		this.indent += 1
@@ -159,7 +216,7 @@ func (this *describer) describe(value reflect.Value) {
 			this.iprintf("\n")
 		}
 		this.indent -= 1
-		this.iprintf("}\n")
+		this.iprintf("}")
 	case reflect.Pointer:
 		this.printf("& ")
 		this.describe(value.Elem())

option.go

@@ -2,29 +2,10 @@ package hopp
 
 import "git.tebibyte.media/sashakoshka/go-util/container"
 
-// Option allows an optional value to be defined without using a pointer.
-// TODO make generic alias once go 1.24 releases
-type Option[T any] ucontainer.Optional[T]
+// Option is an alias for ucontainer.Option, defined here for convenience
+type Option[T any] = ucontainer.Option[T]
 
+// O is an alias for ucontainer.O, defined here for convenience
 func O[T any](value T) Option[T] {
-	return Option[T](ucontainer.O(value))
-}
-
-func Void[T any]() Option[T] {
-	return Option[T](ucontainer.Void[T]())
-}
-
-func (option Option[T]) Ok() bool {
-	return (ucontainer.Optional[T])(option).Exists()
-}
-
-func (option Option[T]) Get() (T, bool) {
-	return (ucontainer.Optional[T])(option).Value()
-}
-
-func (option Option[T]) Default(defaul T) T {
-	if value, ok := option.Get(); ok {
-		return value
-	}
-	return defaul
+	return ucontainer.O(value)
 }

pdl.yaml

@@ -0,0 +1,20 @@
+filetype: pdl
+
+detect:
+        filename: "\\.pdl$"
+
+rules:
+        - preproc: "\\bM[0-9a-fA-F]{4}\\b"
+
+        - type: "\\b((U|I)(5|8|16|32|64|128|256)|F(16|32|64|128|256)|Bool|String|Buffer|Table|Any)\\b"
+
+        - symbol.brackets: "(\\{|\\}|\\[\\])"
+        - symbol.operator: "\\?"
+
+        - constant.number: "\\b[0-9a-fA-F]{4}\\b"
+
+        - comment:
+                start: "//"
+                end: "$"
+                rules:
+                        - todo: "(TODO|XXX|FIXME|BUG):?"

tape/dynamic.go

@@ -6,6 +6,10 @@ package tape
 // TODO: add support for struct tags: `tape:"0000"`, tape:"0001"` so they can get
 // transformed into tables with a defined schema
 
+// TODO: support special behavior for options in structs: don't just write a
+// zero value if the option is void, write no field at all. also consider doing
+// this for maps, and maybe slices.
+
 // TODO: test all of these smaller functions individually
 
 // For an explanation as to why this package always treats LBA/SBA as strings,
@@ -59,6 +63,7 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	case reflect.Uint64:  return encoder.WriteUint64(uint64(reflectValue.Uint()))
 	case reflect.Float32: return encoder.WriteFloat32(float32(reflectValue.Float()))
 	case reflect.Float64: return encoder.WriteFloat64(float64(reflectValue.Float()))
+	case reflect.Bool:    return // SI has no payload
 	case reflect.String:
 		if reflectValue.Len() > MaxStructureLength {
 			return 0, ErrTooLong
@@ -78,6 +83,12 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 		return n, nil
 	}
 
+	// option
+	if isTypeOption(reflectValue.Type()) {
+		elemValue, _ := optionValue(reflectValue) // zero value for free
+		return EncodeAny(encoder, elemValue, tag)
+	}
+	
 	// aggregates
 	reflectType := reflect.TypeOf(value)
 	switch reflectType.Kind() {
@@ -116,9 +127,9 @@ func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error
 func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) {
 	destination, err := skeletonPointer(decoder, tag)
 	if err != nil { return nil, n, err }
-	nn, err := DecodeAnyInto(decoder, destination, tag)
+	nn, err := decodeAny(decoder, destination.Elem(), tag)
 	n += nn; if err != nil { return nil, n, err }
-	return destination, n, err
+	return destination.Elem().Interface(), n, err
 }
 
 // unknownSlicePlaceholder is inserted by skeletonValue and informs the program
@@ -242,12 +253,22 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 		}
 		lengthCast, err := Uint64ToIntSafe(length)
 		if err != nil { return n, err }
-		if isTypeAny(destination.Type()) {
-			// need a skeleton value if we are assigning to any.
-			value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast)
-			destination.Set(value)
-			destination = value
-		}
+
+		// im fucking so done dude. im so fucking done. this was
+		// supposed to only run when we need it but i guess it runs all
+		// the time, because when we get a map destination (a valid,
+		// allocated one) we break apart on SetMapIndex because of a nil
+		// map. yeah thats right. a fucking nil map panic. on the map we
+		// just allocated. but running this unconditionally (whether or
+		// not we receive an empty any value) actually makes it fucking
+		// work. go figure().
+		//
+		// (the map allocation functionality in skeletonPointer was
+		// removed after this comment was written)
+		value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast)
+		destination.Set(value)
+		destination = value
+		
 		destination.Clear()
 		for _ = range lengthCast {
 			key, nn, err := decoder.ReadUint16()
@@ -288,11 +309,23 @@ func tagAny(reflectValue reflect.Value) (Tag, error) {
 	case reflect.Uint64:  return LI.WithCN(7), nil
 	case reflect.Float32: return FP.WithCN(3), nil
 	case reflect.Float64: return FP.WithCN(7), nil
+	case reflect.Bool:
+		if reflectValue.Bool() {
+			return SI.WithCN(1), nil
+		} else {
+			return SI.WithCN(0), nil
+		}
 	case reflect.String:  return bufferLenTag(reflectValue.Len()), nil
 	}
 	if reflectValue.CanConvert(reflect.TypeOf(dummyBuffer)) {
 		return bufferLenTag(reflectValue.Len()), nil
 	}
+
+	// option
+	if isTypeOption(reflectValue.Type()) {
+		elem, _ := optionValue(reflectValue) // zero value for free
+		return tagAny(elem)
+	}
 	
 	// aggregates
 	reflectType := reflectValue.Type()
@@ -319,9 +352,14 @@ func encodeAnySlice(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	for index := 0; index < reflectValue.Len(); index += 1 {
 		itemTag, err := tagAny(reflectValue.Index(index))
 		if err != nil { return n, err }
+		if itemTag.Is(SBA) {
+			// SBA data in an LBA will always have the tag LBA:0,
+			// because 32 <= 256
+			continue
+		}
 		if itemTag.CN() > oneTag.CN() { oneTag = itemTag }
 	}
-	if oneTag.Is(SBA) { oneTag += 1 << 5 }
+	if oneTag.Is(SBA) { oneTag = LBA.WithCN(oneTag.CN()) }
 	nn, err = encoder.WriteUint8(uint8(oneTag))
 	n += nn; if err != nil { return n, err }
 	for index := 0; index < reflectValue.Len(); index += 1 {
@@ -364,7 +402,8 @@ func canSet(destination reflect.Type, tag Tag) error {
 		switch destination.Kind() {
 		case
 			reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
-			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
+			reflect.Bool:
 		default:
 			return errCantAssignf("cannot assign integer to %v", destination)
 		}
@@ -387,7 +426,11 @@ func canSet(destination reflect.Type, tag Tag) error {
 			return errCantAssignf("cannot assign array to %v", destination)
 		}
 	case KTV:
-		if destination != reflect.TypeOf(dummyMap) {
+		cantAssign :=
+			destination.Kind() != reflect.Map ||
+			destination.Key().Kind() != reflect.Uint16 ||
+			!isTypeAny(destination.Elem())
+		if cantAssign {
 			return errCantAssignf("cannot assign table to %v", destination)
 		}
 	default:
@@ -418,6 +461,8 @@ func setInt(destination reflect.Value, value int64, bytes int) {
 // setUint expects a settable destination.
 func setUint(destination reflect.Value, value uint64, bytes int) {
 	switch {
+	case destination.Kind() == reflect.Bool:
+		destination.Set(reflect.ValueOf(value > 0))
 	case destination.CanInt():
 		destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type()))
 	case destination.CanUint():
@@ -496,7 +541,8 @@ func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) {
 func skeletonPointer(decoder *Decoder, tag Tag) (reflect.Value, error) {
 	typ, err := typeOf(decoder, tag)
 	if err != nil { return reflect.Value { }, err }
-	return reflect.New(typ), nil
+	value := reflect.New(typ)
+	return value, nil
 }
 
 // typeOf returns the type of the current tag being decoded. It does not use up
@@ -551,6 +597,21 @@ func isTypeAny(typ reflect.Type) bool {
 	return typ.Kind() == reflect.Interface && typ.NumMethod() == 0
 }
 
+// isTypeOption returns whether the given reflect.Type is a ucontainer.Option,
+// and returns the element type if true.
+func isTypeOption(typ reflect.Type) bool {
+	// TODO: change when needed
+	goutilPath := "git.tebibyte.media/sashakoshka/go-util"
+	return typ.Name() == "Option" && typ.PkgPath() == goutilPath + "/container"
+}
+
+// optionValue returns the value of an option. The value MUST be an option, or
+// this function will panic.
+func optionValue(value reflect.Value) (elem reflect.Value, ok bool) {
+	result := value.MethodByName("Value").Call([]reflect.Value { })
+	return result[0], result[1].Bool()
+}
+
 // peekSlice returns the element tag and dimension count of the OTA currently
 // being decoded. It does not use up the decoder, it only peeks.
 func peekSlice(decoder *Decoder, tag Tag) (Tag, int, error) {

tape/dynamic_test.go

@@ -3,31 +3,10 @@ package tape
 import "bytes"
 import "testing"
 import "reflect"
+import "git.tebibyte.media/sashakoshka/go-util/container"
 import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
 
-var samplePayloads = [][]byte {
-	/* int8     */ []byte { byte(LSI.WithCN(0)), 0x45 },
-	/* int16    */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 },
-	/* int32    */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-	/* int64    */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-	/* uint5    */ []byte { byte(SI.WithCN(12)) },
-	/* uint8    */ []byte { byte(LI.WithCN(0)), 0x45 },
-	/* uint16   */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 },
-	/* uint32   */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-	/* uint64   */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-	/* string   */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' },
-	/* []byte   */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' },
-	/* []string */ []byte {
-		byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)),
-		0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
-		0x05, 0x11, 0x11, 0x11, 0x11, 0x11,
-	},
-	/* map[uint16] any */ []byte {
-		byte(KTV.WithCN(0)), 2,
-		0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67,
-		0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB,
-	},
-}
+var samplePayloads [][]byte
 
 var sampleValues = []any {
 	/* int8     */ int8(0x45),
@@ -39,6 +18,8 @@ var sampleValues = []any {
 	/* uint16   */ uint16(0x4567),
 	/* uint32   */ uint32(0x456789AB),
 	/* uint64   */ uint64(0x456789ABCDEF0123),
+	/* bool     */ false,
+	/* bool     */ true,
 	/* string   */ "pupever",
 	/* []byte   */ "blarg",
 	/* []string */ []string {
@@ -49,6 +30,69 @@ var sampleValues = []any {
 		0x0223: int16(0x4567),
 		0x0224: uint32(0x456789AB),
 	},
+	/* map[uint16] any */ map[uint16] any {
+		0x0001: float32(489.5),
+		0x0002: "hi",
+		0x0003: uint16(0x3992),
+	},
+	// IMPORTANT: ONLY ADD AT THE END!!!! DO NOT MOVE WHAT IS ALREADY HERE!
+	// IMPORTANT: ONLY ADD AT THE END!!!! DO NOT MOVE WHAT IS ALREADY HERE!
+}
+
+type sample struct {
+	t Tag
+	v any
+	s tu.Snake
+}
+
+var samples = []sample {
+	/* int8     */ sample { t: LSI.WithCN(0), s: tu.S(0x45) },
+	/* int16    */ sample { t: LSI.WithCN(1), s: tu.S(0x45, 0x67) },
+	/* int32    */ sample { t: LSI.WithCN(3), s: tu.S(0x45, 0x67, 0x89, 0xAB) },
+	/* int64    */ sample { t: LSI.WithCN(7), s: tu.S(0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23) },
+	/* uint5    */ sample { t: SI.WithCN(12), s: tu.S() },
+	/* uint8    */ sample { t: LI.WithCN(0),  s: tu.S(0x45) },
+	/* uint16   */ sample { t: LI.WithCN(1),  s: tu.S(0x45, 0x67) },
+	/* uint32   */ sample { t: LI.WithCN(3),  s: tu.S(0x45, 0x67, 0x89, 0xAB) },
+	/* uint64   */ sample { t: LI.WithCN(7),  s: tu.S(0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23) },
+	/* bool     */ sample { t: SI.WithCN(0),  s: tu.S() },
+	/* bool     */ sample { t: SI.WithCN(1),  s: tu.S() },
+	/* string   */ sample { t: SBA.WithCN(7), s: tu.S('p', 'u', 'p', 'e', 'v', 'e', 'r') },
+	/* []byte   */ sample { t: SBA.WithCN(5), s: tu.S('b', 'l', 'a', 'r', 'g') },
+	/* []string */ sample {
+		t: OTA.WithCN(0),
+		s: tu.S(2, byte(LBA.WithCN(0)),
+			0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
+			0x05, 0x11, 0x11, 0x11, 0x11, 0x11),
+	},
+	/* map[uint16] any */ sample {
+		t: KTV.WithCN(0),
+		s: tu.S(2).AddVar(
+			[]byte { 0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67 },
+			[]byte { 0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB }),
+	},
+	/* map[uint16] any */ sample {
+		t: KTV.WithCN(0),
+		s: tu.S(3).AddVar(
+			[]byte { 0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00 },
+			[]byte { 0x00, 0x02, 0x82, 'h', 'i' },
+			[]byte { 0x00, 0x03, 0x21, 0x39, 0x92 }),
+	},
+}
+
+var sampleOptionValues []any
+
+func init() {
+	sampleOptionValues = make([]any, len(sampleValues))
+	for index, value := range sampleValues {
+		sampleOptionValues[index] = ucontainer.O(value)
+		samples[index].v = value
+	}
+	samplePayloads = make([][]byte, len(samples))
+	for index, sample := range samples {
+		item := append([]byte { byte(sample.t) }, sample.s.Flatten()...)
+		samplePayloads[index] = item
+	}
 }
 
 type userDefinedInteger int16
@@ -73,7 +117,9 @@ func TestEncodeAnyTable(test *testing.T) {
 		0x3456: userDefinedInteger(0x3921),
 		0x1F1F: float32(67.26),
 		0x0F0F: float64(5.3),
-	}, KTV.WithCN(0), tu.S(9).AddVar(
+		0xAAAA: false,
+		0xBBBB: true,
+	}, KTV.WithCN(0), tu.S(11).AddVar(
 		[]byte {
 			0xF3, 0xB9,
 			byte(LSI.WithCN(3)),
@@ -127,6 +173,14 @@ func TestEncodeAnyTable(test *testing.T) {
 			byte(FP.WithCN(7)),
 			0x40, 0x15, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33,
 		},
+		[]byte {
+			0xAA, 0xAA,
+			byte(SI.WithCN(0)),
+		},
+		[]byte {
+			0xBB, 0xBB,
+			byte(SI.WithCN(1)),
+		},
 	))
 	if err != nil { test.Fatal(err) }
 }
@@ -135,7 +189,7 @@ func TestDecodeWrongType(test *testing.T) {
 	for index, data := range samplePayloads {
 		test.Logf("data %2d %v [%s]", index, Tag(data[0]), tu.HexBytes(data[1:]))
 		// integers should only assign to other integers
-		if index > 8 {
+		if index > 10 {
 			cas := func(destination any) {
 				n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
 				if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
@@ -144,7 +198,7 @@ func TestDecodeWrongType(test *testing.T) {
 					if reflectValue.Int() != 0 {
 						test.Fatalf("destination not zero: %v", reflectValue.Elem().Interface())
 					}
-				} else {
+				} else if reflectValue.Kind() != reflect.Bool {
 					if reflectValue.Uint() != 0 {
 						test.Fatalf("destination not zero: %v", reflectValue.Elem().Interface())
 					}
@@ -169,6 +223,8 @@ func TestDecodeWrongType(test *testing.T) {
 			{ var dest uint32; cas(&dest) }
 			test.Log("- uint64")
 			{ var dest uint64; cas(&dest) }
+			test.Log("- bool")
+			{ var dest bool; cas(&dest) }
 		}
 		arrayCase := func(destination any) {
 			n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
@@ -183,19 +239,19 @@ func TestDecodeWrongType(test *testing.T) {
 			}
 		}
 		// SBA/LBA types should only assign to other SBA/LBA types
-		if index != 9 && index != 10 {
+		if index != 11 && index != 12 {
 			test.Log("- string")
 			{ var dest string; arrayCase(&dest) }
 			test.Log("- []byte")
 			{ var dest []byte; arrayCase(&dest) }
 		}
 		// arrays should only assign to other arrays
-		if index != 11 {
+		if index != 13 {
 			test.Log("- []string")
 			{ var dest []string; arrayCase(&dest) }
 		}
 		// tables should only assign to other tables
-		if index != 12 {
+		if index != 14 && index != 15 {
 			test.Log("- map[uint16] any")
 			{ var dest = map[uint16] any { }; arrayCase(&dest) }
 		}
@@ -220,6 +276,12 @@ func TestEncodeDecodeAnyTable(test *testing.T) {
 func TestEncodeDecodeAnyDestination(test *testing.T) {
 	var destination any
 	for index, data := range samplePayloads {
+		if _, isBool := sampleValues[index].(bool); isBool {
+			// test is invalid for bools because they are never
+			// created as a skeleton value
+			continue
+		}
+	
 		tag     := Tag(data[0])
 		payload := data[1:]
 		test.Logf("data %2d %v [%s]", index, tag, tu.HexBytes(payload))
@@ -238,6 +300,20 @@ func TestEncodeDecodeAnyDestination(test *testing.T) {
 	}
 }
 
+func TestEncodeOption(test *testing.T) {
+	for _, sample := range samples {
+		snake   := sample.s
+		tag     := sample.t
+		value   := sample.v
+		if _, ok := value.(bool); tag.Is(SI) && !ok {
+			// we will never encode an SI unless its a bool
+			continue
+		}
+		err := testEncodeAny(test, value, tag, snake)
+		if err != nil { test.Fatal(err) }
+	}
+}
+
 func TestPeekSlice(test *testing.T) {
 	buffer := bytes.NewBuffer([]byte {
 		2, byte(OTA.WithCN(3)),
@@ -296,3 +372,69 @@ func TestPeekSliceOnce(test *testing.T) {
 		test.Fatalf("wrong n: %d != %d", got, correct)
 	}
 }
+
+func TestTagAny(test *testing.T) {
+	cases := [][2]any {
+		[2]any { LSI.WithCN(3),  int(9)     },
+		[2]any { LSI.WithCN(0),  int8(9)    },
+		[2]any { LSI.WithCN(1),  int16(9)   },
+		[2]any { LSI.WithCN(3),  int32(9)   },
+		[2]any { LSI.WithCN(7),  int64(9)   },
+		[2]any { LI.WithCN(3),   uint(9)    },
+		[2]any { LI.WithCN(0),   uint8(9)   },
+		[2]any { LI.WithCN(1),   uint16(9)  },
+		[2]any { LI.WithCN(3),   uint32(9)  },
+		[2]any { LI.WithCN(7),   uint64(9)  },
+		[2]any { FP.WithCN(3),   float32(9) },
+		[2]any { FP.WithCN(7),   float64(9) },
+		[2]any { SBA.WithCN(12), "small string" },
+		[2]any { SBA.WithCN(12), []byte("small string") },
+		[2]any { LBA.WithCN(0),  "this is a very long string that is long" },
+		[2]any { LBA.WithCN(0),  []byte("this is a very long string that is long") },
+		[2]any { LBA.WithCN(1),  lipsum },
+		[2]any { OTA.WithCN(0),  []int { 1, 2, 3, 4, 5 } },
+		[2]any { OTA.WithCN(0),  []string { "1, 2, 3, 4, 5" } },
+		[2]any { KTV.WithCN(0),  map[uint16] any {
+			0:  1,
+			1:  "wow",
+			2:  10.238,
+			45: -9,
+			9:  map[uint16] any { },
+		}},
+	}
+	for _, cas := range cases {
+		test.Log(cas)
+		got, err := TagAny(cas[1])
+		if err != nil { test.Fatal(err) }
+		if correct := cas[0].(Tag); correct != got {
+			test.Fatalf("wrong tag: %v != %v", got, correct)
+		}
+	}
+}
+
+func TestDecodeAny(test *testing.T) {
+	for index, payload := range samplePayloads {
+		if _, isBool := sampleValues[index].(bool); isBool {
+			// test is invalid for bools because they are never
+			// created as a skeleton value
+			continue
+		}
+		
+		correctValue := sampleValues[index]
+		data := payload[1:]
+		decoder := NewDecoder(bytes.NewBuffer(data))
+		tag     := Tag(payload[0])
+		decoded, n, err := DecodeAny(decoder, tag)
+		test.Log("n:      ", n)
+		test.Log("tag:    ", tag)
+		test.Log("got:    ", tu.Describe(decoded))
+		test.Log("correct:", tu.Describe(correctValue))
+		if err != nil { test.Fatal(err) }
+		if !reflect.DeepEqual(decoded, correctValue) {
+			test.Fatal("values not equal")
+		}
+		if n != len(data) {
+			test.Fatalf("n not equal: %d != %d", n, len(data))
+		}
+	}
+}

tape/misc_test.go

@@ -32,7 +32,9 @@ func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.S
 	test.Log("n:      ", n)
 	test.Log("tag:    ", tag)
 	test.Log("got:    ", tu.HexBytes(bytes))
+	test.Log("   :    ", tu.HexChars(bytes))
 	test.Log("correct:", correctBytes)
+	test.Log("       :", correctBytes.CharsString())
 	if tag != correctTag {
 		return fmt.Errorf("tag not equal: %v != %v", tag, correctTag)
 	}
@@ -56,6 +58,7 @@ func testEncodeDecodeAny(test *testing.T, value, correctValue any) error {
 	test.Log("n:  ", n)
 	test.Log("tag:", tag)
 	test.Log("got:", tu.HexBytes(bytes))
+	test.Log("   :", tu.HexChars(bytes))
 	test.Log("decoding...", tag)
 	if n != len(bytes) {
 		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))

tape/strings.go

@@ -0,0 +1,11 @@
+package tape
+
+const lipsum = `Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.`

tape/tag.go

@@ -18,6 +18,17 @@ type Tag byte; const (
 	CNLimit Tag = 32     // All valid CNs are < CNLimit
 )
 
+// what the first nybble of a tag means:
+//
+//   0-1 : SI 
+//   2-3 : LI 
+//   4-5 : LSI
+//   6-7 : FP 
+//   8-9 : SBA
+//   A-B : LBA
+//   C-D : OTA
+//   E-F : KTV
+
 func (tag Tag) TN() int {
 	return int(tag >> 5)
 }
@@ -67,6 +78,6 @@ func bufferLenTag(length int) Tag {
 	if length < int(CNLimit) {
 		return SBA.WithCN(length)
 	} else {
-		return LBA.WithCN(IntBytes(uint64(length)))
+		return LBA.WithCN(IntBytes(uint64(length)) - 1)
 	}
 }