tape: Fix crash when decoding a bool sometimes

Reviewed-on: #21

.gitignore

@@ -1 +1,2 @@
 /generate/test
+/debug

cmd/hopp-generate/main.go

@@ -1,48 +1,66 @@
 package main
 
 import "os"
-import "fmt"
 import "strings"
 import "path/filepath"
+import "git.tebibyte.media/sashakoshka/go-cli"
 import "git.tebibyte.media/sashakoshka/goparse"
 import "git.tebibyte.media/sashakoshka/hopp/generate"
 
 func main() {
-	name := os.Args[0]
+	flagOutput      := cli.NewInputFlag('o', "output", "The output file", "", cli.ValString)
-	if len(os.Args) != 3 {
+	flagPackageName := cli.NewInputFlag('p', "package-name", "The package name of the file", "", cli.ValString)
-		fmt.Fprintf(os.Stderr, "Usage: %s SOURCE DESTINATION\n", name)
+	command := cli.New("Compile PDL files to program source code",
+		flagOutput,
+		flagPackageName)
+	command.Syntax = "FILE [OPTIONS]..."
+	command.ParseOrExit(os.Args)
+
+	if len(command.Args) != 1 {
+		command.Usage()
 		os.Exit(2)
 	}
-	source      := os.Args[1]
+	source      := command.Args[0]
-	destination := os.Args[2]
+	destination := flagOutput.Value
+	if destination == "" {
+		destination = "protocol.go"
+	}
 
 	input, err := os.Open(source)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 	defer input.Close()
 	protocol, err := generate.ParseReader(source, input)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 
-	absDestination, err := filepath.Abs(destination)
+	packageName := flagPackageName.Value
-	handleErr(1, err)
+	if packageName == "" {
-	packageName := cleanPackageName(strings.ReplaceAll(
+		absDestination, err := filepath.Abs(destination)
-		strings.ToLower(filepath.Base(absDestination)),
+		handleErr(command, 1, err)
-		" ", "_"))
+		base := filepath.Base(absDestination)
-	destination = filepath.Join(os.Args[2], "generated.go")
+		if scrounged, ok := scroungeForPackageName(base); ok {
+			packageName = scrounged
+		} else {
+			packageName = strings.ReplaceAll(
+				strings.ToLower(base),
+				" ", "_")
+		}
+	}
+	packageName = cleanPackageName(packageName)
 
 	output, err := os.Create(destination)
-	handleErr(1, err)
+	handleErr(command, 1, err)
 	generator := generate.Generator {
 		Output:      output,
 		PackageName: packageName,
 	}
 	_, err = generator.Generate(protocol)
-	handleErr(1, err)
+	handleErr(command, 1, err)
-	fmt.Fprintf(os.Stderr, "%s: OK\n", name)
+	command.Println(output, "OK")
 }
 
-func handleErr(code int, err error) {
+func handleErr(command *cli.Cli, code int, err error) {
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[0], parse.Format(err))
+		command.Errorln(parse.Format(err))
 		os.Exit(code)
 	}
 }
@@ -61,3 +79,32 @@ func cleanPackageName(str string) string {
 	}
 	return string(buffer[:j])
 }
+
+func scroungeForPackageName(dir string) (string, bool) {
+	entries, err := os.ReadDir(dir)
+	if err != nil { return "", false}
+	for _, entry := range entries {
+		if !entry.Type().IsRegular() { continue }
+		file, err := os.Open(filepath.Join(dir, entry.Name()))
+		if err != nil { continue }
+		defer file.Close()
+		// FIXME: it is entirely possible that the only file will have
+		// a shitload of doc comments preceeding the package name, and
+		// those comments are usually huge so this is bad
+		buffer := [512]byte { }
+		n, _ := file.Read(buffer[:])
+		text := string(buffer[:n])
+
+		packageIndex := strings.Index(text, "package")
+		if packageIndex < 0 { continue }
+		text = text[packageIndex:]
+		
+		newlineIndex := strings.Index(text, "\n")
+		if packageIndex > 0 { text = text[:newlineIndex] }
+
+		fields := strings.Fields(text)
+		if len(fields) < 2 { continue }
+		return fields[1], true
+	}
+	return "", false
+}

design/pdl-language.md

@@ -6,7 +6,7 @@ PDL allows defining a protocol using HOPP and TAPE.
 
 | Syntax     | TN      | CN | Description
 | ---------- | ------- | -: | -----------
-| I5         | SI      |    |   
+| I5         | SI      |    | 
 | I8         | LSI     |  0 | 
 | I16        | LSI     |  1 | 
 | I32        | LSI     |  3 | 
@@ -25,11 +25,13 @@ PDL allows defining a protocol using HOPP and TAPE.
 | F64        | FP      |  7 | 
 | F128[^2]   | FP      | 15 | 
 | F256[^2]   | FP      | 31 | 
+| Bool       | SI      |    | 
 | String     | SBA/LBA |  * | UTF-8 string
 | Buffer     | SBA/LBA |  * | Byte array
 | []\<TYPE\> | OTA     |  * | Array of any type[^1]
 | Table      | KTV     |  * | Table with undefined schema
 | {...}      | KTV     |  * | Table with defined schema
+| Any        | *       |  * | Value of an undefined type
 
 [^1]: Excluding SI and SBA. I5 and U5 cannot be used in an array, but String and
 Buffer are simply forced to use their "long" variant.
@@ -51,6 +53,7 @@ structures. They are separated by whitespace.
 | RBrace   | `}`                | A right curly brace.
 | LBracket | `[`                | A left square bracket.
 | RBracket | `]`                | A right square bracket.
+| Comment  | `\/\/.*$`          | A doc comment starting with a double-slash.
 
 ## Syntax
 
@@ -67,18 +70,27 @@ an Ident token respectively. A message consists of the method code (Method), the
 message name (Ident), and the message's root type. This is usually a table, but
 can be anything.
 
+Messages, types, and table fields can all have doc comments preceding them,
+which are used to generate documentation for the protocol. The syntax is the
+same as Go's (for now). Comments aren't allowed anywhere else.
+
 Here is an example of all that:
 
 ```
+// Connect is sent from the client to the server as the first message of an
+// authenticated transaction.
 M0000 Connect {
 	0000 Name String,
 	0001 Password String,
 }
 
+// UserList is sent from the server to the client in response to a Connect
+// message.
 M0001 UserList {
 	0000 Users []User,
 }
 
+// User holds profile information about a single user.
 User {
 	0000 Name      String,
 	0001 Bio       String,
@@ -98,7 +110,7 @@ Below is an EBNF description of the language.
 <field>   -> <key> <ident> <type>
 <type> -> <ident>
         | "[" "]" <type>
-        | "{" (<field> ",")* [<field>] "}"
+        | "{" (<comment>* <field> ",")* [<comment>* <field>] "}"
-<message> -> <method> <ident> <type>
+<message> -> <comment>* <method> <ident> <type>
-<typedef> -> <ident> <type>
+<typedef> -> <comment>* <ident> <type>
 ```

design/protocol.md

@@ -75,16 +75,16 @@ connection. Thus, the value may range from 0 to 31 if unsigned, and from -16 to
 17 if signed.
 
 #### Large Integer (LI)
-LI encodes an integer of up to 256 bits, which are stored in the payload. The CN
+LI encodes an integer of up to 256 bits, which are stored in the payload. The
-determine the length of the payload in bytes. The integer is big-endian. Whether
+length of the payload (in bytes) is CN + 1. The integer is big-endian. Whether
 the payload is interpreted as unsigned or as signed two's complement is semantic
 information and must be agreed upon by both sides of the connection. Thus, the
 value may range from 0 to 31 if unsigned, and from -16 to 17 if signed.
 
 #### Floating Point (FP)
 FP encodes an IEEE 754 floating point number of up to 256 bits, which are stored
-in the payload. The CN determines the length of the payload in bytes, and it may
+in the payload. The length of the payload (in bytes) is CN + 1. The only
-only be one of these values: 16, 32, 64, 128, or 256.
+supported bit widths for floats are as follows: 16, 32, 64, 128, and 256.
 
 #### Small Byte Array (SBA)
 SBA encodes an array of up to 32 bytes, which are stored in the paylod. The
@@ -98,15 +98,16 @@ in bytes is determined by the CN.
 #### One-Tag Array (OTA)
 OTA encodes an array of up to 2^256 items, which are stored in the payload after
 the length field and the item tag, where the length field comes first. Each item
-must be the same length, as they all share the same tag. The length of the data
+must be the same length, as they all share the same tag. The length of the
-length field in bytes is determined by the CN.
+length field (in bytes) is CN + 1.
 
 #### Key-Tag-Value Table (KTV)
 KTV encodes a table of up to 2^256 key/value pairs, which are stored in the
 payload after the length field. The pairs themselves consist of a 16-bit
 unsigned big-endian key followed by a tag and then the payload. Pair values can
 be of different types and sizes. The order of the pairs is not significant and
-should never be treated as such.
+should never be treated as such. The length of the length field (in bytes) is
+CN + 1.
 
 ## Transports
 A transport is a protocol that HOPP connections can run on top of. HOPP

generate/generate.go

@@ -24,7 +24,7 @@ const preamble = `
 
 const static = `
 // Table is a KTV table with an undefined schema.
-type Table map[uint16] any
+type Table = map[uint16] any
 
 // Message is any message that can be sent along this protocol.
 type Message interface {
@@ -47,6 +47,15 @@ func canAssign(destination, source tape.Tag) bool {
 	}
 	return false
 }
+
+// boolInt converts a bool to an integer.
+func boolInt(input bool) int {
+	if input {
+		return 1
+	} else {
+		return 0
+	}
+}
 `
 
 // Generator converts protocols into Go code.
@@ -112,13 +121,20 @@ func (this *Generator) Generate(protocol *Protocol) (n int, err error) {
 	return n, nil
 }
 
-func (this *Generator) generateTypedef(name string, typ Type) (n int, err error) {
+func (this *Generator) generateTypedef(name string, typedef Typedef) (n int, err error) {
+	typ := typedef.Type
+
 	// type definition
-	nn, err := this.iprintf(
+	if typedef.Doc == "" {
-		"\n// %s represents the protocol data type %s.\n",
+		nn, err := this.iprintf(
-		name, name)
+			"\n// %s represents the protocol data type %s.\n",
-	n += nn; if err != nil { return n, err }
+			name, name)
-	nn, err = this.iprintf("type %s ", name)
+		n += nn; if err != nil { return n, err }
+	} else {
+		nn, err := this.iprintf("\n%s\n", this.formatComment(typedef.Doc))
+		n += nn; if err != nil { return n, err }
+	}
+	nn, err := this.iprintf("type %s ", name)
 	n += nn; if err != nil { return n, err }
 	nn, err = this.generateType(typ)
 	n += nn; if err != nil { return n, err }
@@ -208,10 +224,16 @@ func (this *Generator) generateTypedef(name string, typ Type) (n int, err error)
 // generateMessage generates the structure, as well as encoding decoding
 // functions for the given message.
 func (this *Generator) generateMessage(method uint16, message Message) (n int, err error) {
-	nn, err := this.iprintf(
+	if message.Doc == "" {
-		"\n// %s represents the protocol message M%04X %s.\n",
+		nn, err := this.iprintf(
-		message.Name, method, message.Name)
+			"\n// %s represents the protocol message M%04X %s.\n",
-	nn, err = this.iprintf("type %s ", this.resolveMessageName(message.Name))
+			message.Name, method, message.Name)
+		n += nn; if err != nil { return n, err }
+	} else {
+		nn, err := this.iprintf("\n%s\n", this.formatComment(message.Doc))
+		n += nn; if err != nil { return n, err }
+	}
+	nn, err := this.iprintf("type %s ", this.resolveMessageName(message.Name))
 	n += nn; if err != nil { return n, err }
 	nn, err = this.generateType(message.Type)
 	n += nn; if err != nil { return n, err }
@@ -235,17 +257,13 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e
 		this.resolveMessageName(message.Name))
 	n += nn; if err != nil { return n, err }
 	this.push()
-		nn, err = this.iprintf("tag := ")
+		tagVar, nn, err := this.generateTag(message.Type, "(*this)")
 		n += nn; if err != nil { return n, err }
-		nn, err = this.generateTag(message.Type, "(*this)")
+		nn, err = this.iprintf("nn, err := encoder.WriteTag(%s)\n", tagVar)
-		n += nn; if err != nil { return n, err }
-		nn, err = this.println()
-		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("nn, err := encoder.WriteTag(tag)\n")
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
-		nn, err = this.generateEncodeValue(message.Type, "(*this)", "tag")
+		nn, err = this.generateEncodeValue(message.Type, "(*this)", tagVar)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf("return n, nil\n")
 		n += nn; if err != nil { return n, err }
@@ -306,6 +324,9 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e
 //   - nn int
 func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource string) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		// SI: (none)
+		// SI stores the value in the tag, so we write nothing here
 	case TypeInt:
 		// SI:     (none)
 		// LI/LSI: <value: IntN>
@@ -376,7 +397,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		nn, err = this.iprintf("}\n")
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf(
-			"nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN())\n",
+			"nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN() + 1)\n",
 			valueSource, tagSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -398,15 +419,11 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 			this.push()
 				nn, err = this.iprintf("_ = item\n")
 				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("tag := ")
+				tagVar, nn, err := this.generateTag(typ.Element, "item")
 				n += nn; if err != nil { return n, err }
-				nn, err = this.generateTag(typ.Element, "item")
+				nn, err = this.iprintf("if %s.Is(tape.SBA) { continue }\n", tagVar)
 				n += nn; if err != nil { return n, err }
-				nn, err = this.println()
+				nn, err = this.iprintf("if %s.CN() > itemTag.CN() { itemTag = %s }\n", tagVar, tagVar)
-				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("if tag.Is(tape.SBA) { continue }\n")
-				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("if tag.CN() > itemTag.CN() { itemTag = tag }\n")
 				n += nn; if err != nil { return n, err }
 			this.pop()
 			nn, err = this.iprintf("}\n")
@@ -415,8 +432,8 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 			n += nn; if err != nil { return n, err }
 			nn, err = this.iprintf("nn, err = encoder.WriteTag(itemTag)\n")
 			n += nn; if err != nil { return n, err }
-			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
+			n += nn; if err != nil { return n, err }
 			nn, err = this.iprintf("for _, item := range %s {\n", valueSource)
 			n += nn; if err != nil { return n, err }
 			this.push()
@@ -453,7 +470,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		nn, err = this.iprintf("}\n")
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf(
-			"nn, err = encoder.WriteUintN(%d, %s.CN())\n",
+			"nn, err = encoder.WriteUintN(%d, %s.CN() + 1)\n",
 			len(typ.Fields), tagSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -461,25 +478,19 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		nn, err = this.iprintf("{\n")
 		n += nn; if err != nil { return n, err }
 		this.push()
-			nn, err = this.iprintf("var tag tape.Tag\n")
-			n += nn; if err != nil { return n, err }
 			for key, field := range typ.Fields {
 				nn, err = this.iprintf("nn, err = encoder.WriteUint16(0x%04X)\n", key)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("tag = ")
-				n += nn; if err != nil { return n, err }
 				fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name)
-				nn, err = this.generateTag(field.Type, fieldSource)
+				tagVar, nn, err := this.generateTag(field.Type, fieldSource)
 				n += nn; if err != nil { return n, err }
-				nn, err = this.println()
+				nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(%s))\n", tagVar)
-				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(tag))\n")
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
-				nn, err = this.generateEncodeValue(field.Type, fieldSource, "tag")
+				nn, err = this.generateEncodeValue(field.Type, fieldSource, tagVar)
 				n += nn; if err != nil { return n, err }
 			}
 		this.pop()
@@ -491,6 +502,12 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
+	case TypeAny:
+		// WHATEVER: [WHATEVER]
+		nn, err := this.iprintf("nn, err = tape.EncodeAny(encoder, %s, %s)\n", valueSource, tagSource)
+		n += nn; if err != nil { return n, err }
+		nn, err = this.generateErrorCheck()
+		n += nn; if err != nil { return n, err }
 	default:
 		panic(fmt.Errorf("unknown type: %T", typ))
 	}
@@ -513,6 +530,11 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 // for [Generator.generateDecodeBranch].
 func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagSource string) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		// SI: (none)
+		// SI stores the value in the tag
+		nn, err := this.iprintf("*%s = %s.CN() > 0\n", valueSource, tagSource)
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		// SI:     (none)
 		// LI/LSI: <value: IntN>
@@ -627,7 +649,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 	case TypeTable:
 		// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
 		nn, err := this.iprintf(
-			"nn, err = tape.DecodeAny(decoder, %s, %s)\n",
+			"nn, err = tape.DecodeAnyInto(decoder, %s, %s)\n",
 			valueSource, tagSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -642,6 +664,12 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
+	case TypeAny:
+		// WHATEVER: [WHATEVER]
+		nn, err := this.iprintf("*%s, nn, err = tape.DecodeAny(decoder, %s)\n", valueSource, tagSource)
+		n += nn; if err != nil { return n, err }
+		nn, err = this.generateErrorCheck()
+		n += nn; if err != nil { return n, err }
 	default:
 		panic(fmt.Errorf("unknown type: %T", typ))
 	}
@@ -711,7 +739,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar)
+			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()) + 1)\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
 			n += nn; if err != nil { return n, err }
@@ -787,7 +815,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar)
+			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()) + 1)\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
 			n += nn; if err != nil { return n, err }
@@ -924,49 +952,63 @@ func (this *Generator) generateErrorCheck() (n int, err error) {
 	return this.iprintf("n += nn; if err != nil { return n, err }\n")
 }
 
+func (this *Generator) generateBareErrorCheck() (n int, err error) {
+	return this.iprintf("if err != nil { return n, err }\n")
+}
+
 // generateTag generates the preferred TN and CN for the given type and value.
-// The generated code is INLINE.
+// The generated code is a BLOCK.
-func (this *Generator) generateTag(typ Type, source string) (n int, err error) {
+func (this *Generator) generateTag(typ Type, source string) (tagVar string, n int, err error) {
+	tagVar = this.newTemporaryVar("tag")
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.iprintf("%s := tape.SI.WithCN(boolInt(bool(%s)))\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeInt:
 		if typ.Bits <= 5 {
-			nn, err := this.printf("tape.SI.WithCN(int(%s))", source)
+			nn, err := this.iprintf("%s := tape.SI.WithCN(int(%s))\n", tagVar, source)
-			n += nn; if err != nil { return n, err }
+			n += nn; if err != nil { return tagVar, n, err }
 		} else if typ.Signed {
-			nn, err := this.printf("tape.LSI.WithCN(%d)", bitsToCN(typ.Bits))
+			nn, err := this.iprintf("%s := tape.LSI.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits))
-			n += nn; if err != nil { return n, err }
+			n += nn; if err != nil { return tagVar, n, err }
 		} else {
-			nn, err := this.printf("tape.LI.WithCN(%d)", bitsToCN(typ.Bits))
+			nn, err := this.iprintf("%s := tape.LI.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits))
-			n += nn; if err != nil { return n, err }
+			n += nn; if err != nil { return tagVar, n, err }
 		}
 	case TypeFloat:
-		nn, err := this.printf("tape.FP.WithCN(%d)", bitsToCN(typ.Bits))
+		nn, err := this.iprintf("%s := tape.FP.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits))
-		n += nn; if err != nil { return n, err }
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeString:
-		nn, err := this.printf("tape.StringTag(string(%s))", source)
+		nn, err := this.iprintf("%s := tape.StringTag(string(%s))\n", tagVar, source)
-		n += nn; if err != nil { return n, err }
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeBuffer:
-		nn, err := this.printf("tape.BufferTag([]byte(%s))", source)
+		nn, err := this.iprintf("%s := tape.BufferTag([]byte(%s))\n", tagVar, source)
-		n += nn; if err != nil { return n, err }
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeArray:
-		nn, err := this.printf("tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))))", source)
+		nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))) - 1)\n", tagVar, source)
-		n += nn; if err != nil { return n, err }
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeTable:
-		nn, err := this.printf("tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))))", source)
+		nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))) - 1)\n", tagVar, source)
-		n += nn; if err != nil { return n, err }
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeTableDefined:
-		nn, err := this.printf("tape.KTV.WithCN(%d)", tape.IntBytes(uint64(len(typ.Fields))))
+		nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields))) - 1)
-		n += nn; if err != nil { return n, err }
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeNamed:
 		resolved, err := this.resolveTypeName(typ.Name)
-		if err != nil { return n, err }
+		if err != nil { return tagVar, n, err }
-		nn, err := this.generateTag(resolved, source)
+		subTagVar, nn, err := this.generateTag(resolved, source)
-		n += nn; if err != nil { return n, err }
+		n += nn; if err != nil { return tagVar, n, err }
+		tagVar = subTagVar
+	case TypeAny:
+		nn, err := this.iprintf("%s, err := tape.TagAny(%s)\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
+		nn, err = this.generateBareErrorCheck()
+		n += nn; if err != nil { return tagVar, n, err }
 	default:
 		panic(fmt.Errorf("unknown type: %T", typ))
 	}
 
-	return n, nil
+	return tagVar, n, nil
 }
 
 // generateTN generates the appropriate TN for the given type. The generated
@@ -975,6 +1017,9 @@ func (this *Generator) generateTag(typ Type, source string) (n int, err error) {
 // information is chosen.
 func (this *Generator) generateTN(typ Type) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.printf("tape.SI")
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		if typ.Bits <= 5 {
 			nn, err := this.printf("tape.SI")
@@ -1009,6 +1054,8 @@ func (this *Generator) generateTN(typ Type) (n int, err error) {
 		if err != nil { return n, err }
 		nn, err := this.generateTN(resolved)
 		n += nn; if err != nil { return n, err }
+	default:
+		panic(fmt.Errorf("unknown type: %T", typ))
 	}
 
 	return n, nil
@@ -1016,6 +1063,9 @@ func (this *Generator) generateTN(typ Type) (n int, err error) {
 
 func (this *Generator) generateType(typ Type) (n int, err error) {
 	switch typ := typ.(type) {
+	case TypeBool:
+		nn, err := this.printf("bool")
+		n += nn; if err != nil { return n, err }
 	case TypeInt:
 		if err := this.validateIntBitSize(typ.Bits); err != nil {
 			return n, err
@@ -1063,6 +1113,11 @@ func (this *Generator) generateType(typ Type) (n int, err error) {
 	case TypeNamed:
 		nn, err := this.print(typ.Name)
 		n += nn; if err != nil { return n, err }
+	case TypeAny:
+		nn, err := this.print("any")
+		n += nn; if err != nil { return n, err }
+	default:
+		panic(fmt.Errorf("unknown type: %T", typ))
 	}
 	return n, nil
 }
@@ -1074,7 +1129,9 @@ func (this *Generator) generateTypeTableDefined(typ TypeTableDefined) (n int, er
 
 	for _, key := range slices.Sorted(maps.Keys(typ.Fields)) {
 		field := typ.Fields[key]
-		nn, err := this.iprintf("%s ", field.Name)
+		nn, err := this.iprintf("%s\n", this.formatComment(field.Doc))
+		n += nn; if err != nil { return n, err }
+		nn, err = this.iprintf("%s ", field.Name)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateType(field.Type)
 		n += nn; if err != nil { return n, err }
@@ -1092,12 +1149,17 @@ func (this *Generator) generateTypeTableDefined(typ TypeTableDefined) (n int, er
 // by tagSource can be assigned to a Go destination generated from typ. The
 // generated code is INLINE.
 func (this *Generator) generateCanAssign(typ Type, tagSource string) (n int, err error) {
-	nn, err := this.printf("canAssign(")
+	if _, ok := typ.(TypeAny); ok {
-	n += nn; if err != nil { return n, err }
+		nn, err := this.printf("true")
-	nn, err = this.generateTN(typ)
+		n += nn; if err != nil { return n, err }
-	n += nn; if err != nil { return n, err }
+	} else {
-	nn, err = this.printf(", %s)", tagSource)
+		nn, err := this.printf("canAssign(")
-	n += nn; if err != nil { return n, err }
+		n += nn; if err != nil { return n, err }
+		nn, err = this.generateTN(typ)
+		n += nn; if err != nil { return n, err }
+		nn, err = this.printf(", %s)", tagSource)
+		n += nn; if err != nil { return n, err }
+	}
 	return n, nil
 }
 
@@ -1154,17 +1216,21 @@ func (this *Generator) iprintf(format string, args ...any) (n int, err error) {
 	return fmt.Fprintf(this.Output, this.indent() + format, args...)
 }
 
+func (this *Generator) formatComment(comment string) string {
+	return "// " + strings.ReplaceAll(comment, "\n", "\n" + this.indent() + "// ")
+}
+
 func (this *Generator) resolveMessageName(message string) string {
 	return "Message" + message
 }
 
 func (this *Generator) resolveTypeName(name string) (Type, error) {	
-	if typ, ok := this.protocol.Types[name]; ok {
+	if typedef, ok := this.protocol.Types[name]; ok {
-		if typ, ok := typ.(TypeNamed); ok {
+		if typ, ok := typedef.Type.(TypeNamed); ok {
 			return this.resolveTypeName(typ.Name)
 		}
 		
-		return typ, nil
+		return typedef.Type, nil
 	}
 	return nil, fmt.Errorf("no type exists called %s", name)
 }

generate/generate_test.go

@@ -59,14 +59,38 @@ func init() {
 				0x000C: Field { Name: "NI16",Type: TypeInt { Bits: 16, Signed: true } },
 				0x000D: Field { Name: "NI32",Type: TypeInt { Bits: 32, Signed: true } },
 				0x000E: Field { Name: "NI64",Type: TypeInt { Bits: 64, Signed: true } },
+				0x000F: Field { Name: "Bool",Type: TypeBool { } },
 			},
 		},
 	}
-	exampleProtocol.Types["User"] = TypeTableDefined {
+	exampleProtocol.Messages[0x0005] = Message {
-		Fields: map[uint16] Field {
+		Name: "Dynamic",
-			0x0000: Field { Name: "Name",      Type: TypeString { } },
+		Type: TypeTableDefined {
-			0x0001: Field { Name: "Bio",       Type: TypeString { } },
+			Fields: map[uint16] Field {
-			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+				0x0000: Field { Name: "AU8", Type: TypeAny { } },
+				0x0001: Field { Name: "AU16", Type: TypeAny { } },
+				0x0002: Field { Name: "AU32", Type: TypeAny { } },
+				0x0003: Field { Name: "AU64", Type: TypeAny { } },
+				0x0004: Field { Name: "AI8", Type: TypeAny { } },
+				0x0005: Field { Name: "AI16", Type: TypeAny { } },
+				0x0006: Field { Name: "AI32", Type: TypeAny { } },
+				0x0007: Field { Name: "AI64", Type: TypeAny { } },
+				0x0008: Field { Name: "AF32", Type: TypeAny { } },
+				0x0009: Field { Name: "AF64", Type: TypeAny { } },
+				0x000A: Field { Name: "AString", Type: TypeAny { } },
+				0x000B: Field { Name: "AArray", Type: TypeAny { } },
+				0x000C: Field { Name: "ATable", Type: TypeAny { } },
+				0x000D: Field { Name: "T0", Type: TypeTable { } },
+			},
+		},
+	}
+	exampleProtocol.Types["User"] = Typedef {
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Bio",       Type: TypeString { } },
+				0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+			},
 		},
 	}
 }
@@ -82,7 +106,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageConnect,
-			tu.S(0xE1, 0x02).AddVar(
+			tu.S(0xE0, 0x02).AddVar(
 				[]byte { 0x00, 0x00, 0x86, 'r', 'a', 'r', 'i', 't', 'y' },
 				[]byte { 0x00, 0x01, 0x84, 'g', 'e', 'm', 's' },
 				))
@@ -108,8 +132,8 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageUserList,
-			tu.S(0xE1, 0x01, 0x00, 0x00,
+			tu.S(0xE0, 0x01, 0x00, 0x00,
-				0xC1, 0x03, 0xE1,
+				0xC0, 0x03, 0xE0,
 					).Add(0x03).AddVar(
 						[]byte { 0x00, 0x00, 0x86, 'r', 'a', 'r', 'i', 't', 'y' },
 						[]byte { 0x00, 0x01, 0x87, 'a', 's', 'd', 'j', 'a', 'd', 's' },
@@ -134,7 +158,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messagePulse,
-			tu.S(0xE1, 0x05).AddVar(
+			tu.S(0xE0, 0x05).AddVar(
 				[]byte { 0x00, 0x00, 0x09 },
 				[]byte { 0x00, 0x01, 0x41, 0xCA, 0xDF },
 				[]byte { 0x00, 0x02, 0x61, 0x51, 0xAC },
@@ -155,7 +179,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageNestedArray,
-			tu.S(0xC1, 0x02, 0xC1,
+			tu.S(0xC0, 0x02, 0xC0,
 				0x06, 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6,
 				35,   0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6,
 				            0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC,
@@ -178,10 +202,11 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 			NI16: -0x34C9,
 			NI32: -0x10E134C9,
 			NI64: -0x639109BC10E134C9,
+			Bool: true,
 		}
 		testEncodeDecode(
 			&messageIntegers,
-			tu.S(0xE1, 13).AddVar(
+			tu.S(0xE0, 14).AddVar(
 				[]byte { 0x00, 0x00, 0x13 },
 				[]byte { 0x00, 0x01, 0x20, 0xC9 },
 				[]byte { 0x00, 0x02, 0x21, 0x34, 0xC9 },
@@ -195,6 +220,58 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 				[]byte { 0x00, 0x0C, 0x41, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0D, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0E, 0x47, 0x9C, 0x6E, 0xF6, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
+				[]byte { 0x00, 0x0F, 0x01 },
+				))
+		log.Println("MessageDynamic")
+		messageDynamic := MessageDynamic {
+			AU8:     uint8(0x23),
+			AU16:    uint16(0x3247),
+			AU32:    uint32(0x87324523),
+			AU64:    uint64(0x3284029034098234),
+			AI8:     int8(0x23),
+			AI16:    int16(0x3247),
+			AI32:    int32(0x57324523),
+			AI64:    int64(0x3284029034098234),
+			AF32:    float32(2342.2378),
+			AF64:    float64(324.8899992),
+			AString: "fox bed",
+			AArray:  []int16 { 0x7, 0x6, 0x5, 0x4 },
+			ATable: map[uint16] any {
+				0x0001: int8(0x8),
+				0x0002: float64(4.4),
+			},
+			T0: map[uint16] any {
+				0x0001: float32(489.5),
+				0x0002: "hi",
+				0x0003: uint16(0x3992),
+			},
+		}
+		testEncodeDecode(
+			&messageDynamic,
+			tu.S(0xE0, 14).AddVar(
+				[]byte { 0x00, 0x00, 0x20, 0x23 },
+				[]byte { 0x00, 0x01, 0x21, 0x32, 0x47 },
+				[]byte { 0x00, 0x02, 0x23, 0x87, 0x32, 0x45, 0x23 },
+				[]byte { 0x00, 0x03, 0x27, 0x32, 0x84, 0x02, 0x90, 0x34, 0x09, 0x82, 0x34 },
+				[]byte { 0x00, 0x04, 0x40, 0x23 },
+				[]byte { 0x00, 0x05, 0x41, 0x32, 0x47 },
+				[]byte { 0x00, 0x06, 0x43, 0x57, 0x32, 0x45, 0x23 },
+				[]byte { 0x00, 0x07, 0x47, 0x32, 0x84, 0x02, 0x90, 0x34, 0x09, 0x82, 0x34 },
+				[]byte { 0x00, 0x08, 0x63, 0x45, 0x12, 0x63, 0xCE },
+				[]byte { 0x00, 0x09, 0x67, 0x40, 0x74, 0x4E, 0x3D, 0x6F, 0xCD, 0x17, 0x75 },
+				[]byte { 0x00, 0x0A, 0x87, 'f', 'o', 'x', ' ', 'b', 'e', 'd' },
+				[]byte { 0x00, 0x0B, 0xC0, 0x04, 0x41,
+					0x00, 0x07,
+					0x00, 0x06,
+					0x00, 0x05,
+					0x00, 0x04 },
+				[]byte { 0x00, 0x0C, 0xE0, 0x02,
+					0x00, 0x01, 0x40, 0x08,
+					0x00, 0x02, 0x67, 0x40, 0x11, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A },
+				[]byte { 0x00, 0x0D, 0xE0, 0x03, // ERR
+					0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00,
+					0x00, 0x02, 0x82, 'h', 'i',
+					0x00, 0x03, 0x21, 0x39, 0x92 },
 				))
 	`)
 }

generate/lex.go

@@ -15,6 +15,7 @@ const (
 	TokenRBrace
 	TokenLBracket
 	TokenRBracket
+	TokenComment
 )
 
 var tokenNames = map[parse.TokenKind] string {
@@ -26,6 +27,7 @@ var tokenNames = map[parse.TokenKind] string {
 	TokenRBrace:   "RBrace",
 	TokenLBracket: "LBracket",
 	TokenRBracket: "RBracket",
+	TokenComment:  "Comment",
 }
 
 func Lex(fileName string, reader io.Reader) (parse.Lexer, error) {
@@ -81,6 +83,18 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 		}
 	}
 
+	unexpected := func() error {
+		if unicode.IsPrint(this.rune) {
+			return parse.Errorf (
+				this.pos(), "unexpected rune '%c'",
+				this.rune)
+		} else {
+			return parse.Errorf (
+				this.pos(), "unexpected rune %U",
+				this.rune)
+		}
+	}
+
 	defer func () {
 		newPos := this.pos()
 		newPos.End -- // TODO figure out why tf we have to do this
@@ -133,14 +147,21 @@ func (this *lexer) nextInternal() (token parse.Token, err error) {
 		token.Kind = TokenRBracket
 		appendRune()
 		if this.eof { err = nil; return }
-	case unicode.IsPrint(this.rune):
+	// Comment
-		err = parse.Errorf (
+	case this.rune == '/':
-			this.pos(), "unexpected rune '%c'",
+		token.Kind = TokenComment
-			this.rune)
+		appendRune()
+		if this.eof { return }
+		if this.rune != '/' {
+			err = unexpected()
+			return
+		}
+		for this.rune != '\n' {
+			appendRune()
+			if this.eof { err = nil; return }
+		}
 	default:
-		err = parse.Errorf (
+		err = unexpected()
-			this.pos(), "unexpected rune %U",
-			this.rune)
 	}
 
 	return

generate/lex_test.go

@@ -6,14 +6,21 @@ import "git.tebibyte.media/sashakoshka/goparse"
 
 func TestLex(test *testing.T) {
 	lexer, err := Lex("test.pdl", strings.NewReader(`
+	// User holds profile information about a single user.
 	M0001 User {
 		0000 Name      String,
+		// dog water comment
+		
+		// Users is asdkjsagkj why
+		// 
+		// wow
 		0001 Users []User,
 		0002 Followers U32,
 	}`))
 	if err != nil { test.Fatal(parse.Format(err)) }
 
 	correctTokens := []parse.Token {
+		tok(TokenComment,  "// User holds profile information about a single user."),
 		tok(TokenMethod,   "0001"),
 		tok(TokenIdent,    "User"),
 		tok(TokenLBrace,   "{"),
@@ -21,6 +28,10 @@ func TestLex(test *testing.T) {
 		tok(TokenIdent,    "Name"),
 		tok(TokenIdent,    "String"),
 		tok(TokenComma,    ","),
+		tok(TokenComment,  "// dog water comment"),
+		tok(TokenComment,  "// Users is asdkjsagkj why"),
+		tok(TokenComment,  "// "),
+		tok(TokenComment,  "// wow"),
 		tok(TokenKey,      "0001"),
 		tok(TokenIdent,    "Users"),
 		tok(TokenLBracket, "["),

generate/misc_test.go

@@ -100,12 +100,12 @@ func testGenerateRun(test *testing.T, protocol *Protocol, title, imports, testCa
 			log.Println("decoding:")
 			destination := reflect.New(reflect.ValueOf(message).Elem().Type()).Interface().(Message)
 			flat := data.Flatten()
-			log.Println("before: ", destination)
+			log.Println("before: ", tu.Describe(destination))
 			decoder := tape.NewDecoder(bytes.NewBuffer(flat))
 			n, err = destination.Decode(decoder)
 			if err != nil { log.Fatalf("at %d: %v\n", n, err) }
-			log.Println("got:    ", destination)
+			log.Println("got:    ", tu.Describe(destination))
-			log.Println("correct:", message)
+			log.Println("correct:", tu.Describe(message))
 			if n != len(flat) {
 				log.Fatalf("n incorrect: %d != %d\n", n, len(flat))
 			}

generate/parse.go

@@ -1,6 +1,7 @@
 package generate
 
 import "io"
+import "strings"
 import "strconv"
 import "git.tebibyte.media/sashakoshka/goparse"
 
@@ -21,7 +22,7 @@ func Parse(lx parse.Lexer) (*Protocol, error) {
 func defaultProtocol() Protocol {
 	return Protocol {
 		Messages: make(map[uint16] Message),
-		Types: map[string] Type { },
+		Types: map[string] Typedef { },
 	}
 }
 
@@ -47,18 +48,28 @@ func (this *parser) parse() error {
 }
 
 func (this *parser) parseTopLevel() error {
-	err := this.ExpectDesc("message or typedef", TokenMethod, TokenIdent)
+	doc := ""
-	if err != nil { return err }
+	for {
-	if this.EOF() { return nil }
+		err := this.ExpectDesc("message or typedef", TokenMethod, TokenIdent, TokenComment)
+		if err != nil { return err }
+		if this.EOF() { return nil }
+		if this.Kind() == TokenComment {
+			if doc != "" { doc += "\n" }
+			doc += this.parseComment(this.Value())
+			this.Next()
+		} else {
+			break
+		}
+	}
 
 	switch this.Kind() {
-	case TokenMethod: return this.parseMessage()
+	case TokenMethod: return this.parseMessage(doc)
-	case TokenIdent:  return this.parseTypedef()
+	case TokenIdent:  return this.parseTypedef(doc)
 	}
 	panic("bug")
 }
 
-func (this *parser) parseMessage() error {
+func (this *parser) parseMessage(doc string) error {
 	err := this.Expect(TokenMethod)
 	if err != nil { return err }
 	method, err := this.parseHexNumber(this.Value(), 0xFFFF)
@@ -72,12 +83,13 @@ func (this *parser) parseMessage() error {
 	if err != nil { return err }
 	this.protocol.Messages[uint16(method)] = Message {
 		Name: name,
+		Doc:  doc,
 		Type: typ,
 	}
 	return nil
 }
 
-func (this *parser) parseTypedef() error {
+func (this *parser) parseTypedef(doc string) error {
 	err := this.Expect(TokenIdent)
 	if err != nil { return err }
 	name := this.Value()
@@ -85,7 +97,10 @@ func (this *parser) parseTypedef() error {
 	if err != nil { return err }
 	typ, err := this.parseType()
 	if err != nil { return err }
-	this.protocol.Types[name] = typ
+	this.protocol.Types[name] = Typedef {
+		Doc:  doc,
+		Type: typ,
+	}
 	return nil
 }
 
@@ -116,6 +131,8 @@ func (this *parser) parseType() (Type, error) {
 		case "String": return TypeString { }, this.Next()
 		case "Buffer": return TypeBuffer { }, this.Next()
 		case "Table":  return TypeTable { }, this.Next()
+		case "Any":    return TypeAny { }, this.Next()
+		case "Bool":   return TypeBool { }, this.Next()
 		}
 		return this.parseTypeNamed()
 	case TokenLBracket:
@@ -156,12 +173,22 @@ func (this *parser) parseTypeTable() (TypeTableDefined, error) {
 		Fields: make(map[uint16] Field),
 	}
 	for {
-		err := this.ExpectDesc("table field", TokenKey, TokenRBrace)
+		doc := ""
-		if err != nil { return TypeTableDefined { }, err }
+		for {
+			err := this.ExpectDesc("table field", TokenKey, TokenRBrace, TokenComment)
+			if err != nil { return TypeTableDefined { }, err }
+			if this.Kind() == TokenComment {
+				if doc != "" { doc += "\n" }
+				doc += this.parseComment(this.Value())
+				this.Next()
+			} else {
+				break
+			}
+		}
 		if this.Is(TokenRBrace) {
 			break
 		}
-		key, field, err := this.parseField()
+		key, field, err := this.parseField(doc)
 		if err != nil { return TypeTableDefined { }, err }
 		typ.Fields[key] = field
 		err = this.Expect(TokenComma, TokenRBrace)
@@ -177,7 +204,7 @@ func (this *parser) parseTypeTable() (TypeTableDefined, error) {
 	return typ, nil
 }
 
-func (this *parser) parseField() (uint16, Field, error) {
+func (this *parser) parseField(doc string) (uint16, Field, error) {
 	err := this.Expect(TokenKey)
 	if err != nil { return 0, Field { }, err }
 	key, err := this.parseHexNumber(this.Value(), 0xFFFF)
@@ -191,6 +218,7 @@ func (this *parser) parseField() (uint16, Field, error) {
 	if err != nil { return 0, Field { }, err }
 	return uint16(key), Field {
 		Name: name,
+		Doc:  doc,
 		Type: typ,
 	}, nil
 }
@@ -205,3 +233,7 @@ func (this *parser) parseHexNumber(input string, maxValue int64) (int64, error)
 	}
 	return number, nil
 }
+
+func (this *parser) parseComment(input string) string {
+	return strings.TrimPrefix(strings.TrimPrefix(input, "//"), " ")
+}

generate/parse_test.go

@@ -9,6 +9,7 @@ func TestParse(test *testing.T) {
 	correct := defaultProtocol()
 	correct.Messages[0x0000] = Message {
 		Name: "Connect",
+		Doc:  "Connect is sent from the client to the server as the first message of an\nauthenticated transaction.",
 		Type: TypeTableDefined {
 			Fields: map[uint16] Field {
 				0x0000: Field { Name: "Name",      Type: TypeString { } },
@@ -18,36 +19,53 @@ func TestParse(test *testing.T) {
 	}
 	correct.Messages[0x0001] = Message {
 		Name: "UserList",
+		Doc:  "UserList is sent from the server to the client in response to a Connect\nmessage.",
 		Type: TypeTableDefined {
 			Fields: map[uint16] Field {
 				0x0000: Field { Name: "Users",  Type: TypeArray { Element: TypeNamed { Name: "User" } } },
 			},
 		},
 	}
-	correct.Types["User"] = TypeTableDefined {
+	correct.Types["User"] = Typedef {
-		Fields: map[uint16] Field {
+		Doc:  "User holds profile information about a single user.",
-			0x0000: Field { Name: "Name",      Type: TypeString { } },
+		Type: TypeTableDefined {
-			0x0001: Field { Name: "Bio",       Type: TypeString { } },
+			Fields: map[uint16] Field {
-			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Bio",       Type: TypeString { } },
+				0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+				0x0003: Field { Name: "Bouncy",    Type: TypeBool { } },
+			},
 		},
 	}
+	correct.Types["Anything"] = Typedef {
+		Type: TypeAny { },
+	}
 	test.Log("CORRECT:", &correct)
 	
 	got, err := ParseReader("test.pdl", strings.NewReader(`
+	// Connect is sent from the client to the server as the first message of an
+	// authenticated transaction.
 	M0000 Connect {
 		0000 Name String,
+		// Password is where you put your secrets, your shameful secrets
 		0001 Password String,
 	}
 	
+	// UserList is sent from the server to the client in response to a Connect
+	// message.
 	M0001 UserList {
 		0000 Users []User,
 	}
 	
+	// User holds profile information about a single user.
 	User {
 		0000 Name      String,
 		0001 Bio       String,
 		0002 Followers U32,
+		0003 Bouncy    Bool,
 	}
+
+	Anything Any
 	`))
 	if err != nil { test.Fatal(parse.Format(err)) }
 	test.Log("GOT:    ", got)

generate/protocol.go

@@ -7,11 +7,17 @@ import "crypto/md5"
 
 type Protocol struct {
 	Messages map[uint16] Message
-	Types    map[string] Type
+	Types    map[string] Typedef
 }
 
 type Message struct {
 	Name string
+	Doc  string
+	Type Type
+}
+
+type Typedef struct {
+	Doc  string
 	Type Type
 }
 
@@ -43,6 +49,12 @@ func (typ TypeFloat) String() string {
 	return fmt.Sprintf("F%d", typ.Bits)
 }
 
+type TypeBool struct { }
+
+func (TypeBool) String() string {
+	return "Bool"
+}
+
 type TypeString struct { }
 
 func (TypeString) String() string {
@@ -84,6 +96,7 @@ func (typ TypeTableDefined) String() string {
 
 type Field struct {
 	Name string
+	Doc  string
 	Type Type
 }
 
@@ -99,6 +112,12 @@ func (typ TypeNamed) String() string {
 	return typ.Name
 }
 
+type TypeAny struct { }
+
+func (typ TypeAny) String() string {
+	return "Any"
+}
+
 func HashType(typ Type) [16]byte {
 	// TODO: if we ever want to make the compiler more efficient, this would
 	// be a good place to start, complex string concatenation in a hot path

go.mod

@@ -3,6 +3,7 @@ module git.tebibyte.media/sashakoshka/hopp
 go 1.23.0
 
 require (
+	git.tebibyte.media/sashakoshka/go-cli v0.1.3
 	git.tebibyte.media/sashakoshka/go-util v0.9.1
 	git.tebibyte.media/sashakoshka/goparse v0.2.0
 )

go.sum

@@ -1,3 +1,5 @@
+git.tebibyte.media/sashakoshka/go-cli v0.1.3 h1:tSkWjyx2JrGu6KotbXWSTKSYGGS1D4O3qwCrRoZuwbs=
+git.tebibyte.media/sashakoshka/go-cli v0.1.3/go.mod h1:JFA3wSdRkXxa4iQJWHfe3DokiG7Dh2XUJBzPmuVlbuY=
 git.tebibyte.media/sashakoshka/go-util v0.9.1 h1:eGAbLwYhOlh4aq/0w+YnJcxT83yPhXtxnYMzz6K7xGo=
 git.tebibyte.media/sashakoshka/go-util v0.9.1/go.mod h1:0Q1t+PePdx6tFYkRuJNcpM1Mru7wE6X+it1kwuOH+6Y=
 git.tebibyte.media/sashakoshka/goparse v0.2.0 h1:uQmKvOCV2AOlCHEDjg9uclZCXQZzq2PxaXfZ1aIMiQI=

internal/testutil/testutil.go

@@ -2,8 +2,9 @@ package testutil
 
 import "fmt"
 import "slices"
-import "strings"
 import "reflect"
+import "strings"
+import "unicode"
 
 // Snake lets you compare blocks of data where the ordering of certain parts may
 // be swapped every which way. It is designed for comparing the encoding of
@@ -119,7 +120,15 @@ type describer struct {
 }
 
 func (this *describer) describe(value reflect.Value) {
+	if !value.IsValid() {
+		this.printf("<invalid>")
+		return
+	}
 	value = reflect.ValueOf(value.Interface())
+	if !value.IsValid() {
+		this.printf("<invalid>")
+		return
+	}
 	switch value.Kind() {
 	case reflect.Array, reflect.Slice:
 		this.printf("[\n")
@@ -137,12 +146,20 @@ func (this *describer) describe(value reflect.Value) {
 		typ := value.Type()
 		for index := range typ.NumField() {
 			indexBuffer := [1]int { index }
-			this.iprintf("%s: ", typ.Field(index).Name)
+			field := typ.Field(index)
-			this.describe(value.FieldByIndex(indexBuffer[:]))
+			this.iprintf("%s: ", field.Name)
+			for _, char := range field.Name {
+				if unicode.IsUpper(char) {
+					this.describe(value.FieldByIndex(indexBuffer[:]))
+				} else {
+					this.printf("<private>")
+				}
+				break
+			}
 			this.iprintf("\n")
 		}
 		this.indent -= 1
-		this.iprintf("}\n")
+		this.iprintf("}")
 	case reflect.Map:
 		this.printf("map {\n")
 		this.indent += 1
@@ -155,7 +172,7 @@ func (this *describer) describe(value reflect.Value) {
 			this.iprintf("\n")
 		}
 		this.indent -= 1
-		this.iprintf("}\n")
+		this.iprintf("}")
 	case reflect.Pointer:
 		this.printf("& ")
 		this.describe(value.Elem())

tape/dynamic.go

@@ -8,6 +8,17 @@ package tape
 
 // TODO: test all of these smaller functions individually
 
+// For an explanation as to why this package always treats LBA/SBA as strings,
+// refer to https://go.dev/blog/strings:
+// 
+//   It’s important to state right up front that a string holds arbitrary
+//   bytes. It is not required to hold Unicode text, UTF-8 text, or any other
+//   predefined format. As far as the content of a string is concerned, it is
+//   exactly equivalent to a slice of bytes.
+//
+// Arbitrary byte slices and blobs won't be as common of a use case as text
+// data, and if you need that anyway you can just cast it to a byte slice.
+
 import "fmt"
 import "reflect"
 
@@ -36,16 +47,19 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	// primitives
 	reflectValue := reflect.ValueOf(value)
 	switch reflectValue.Kind() {
-	case reflect.Int:    return encoder.WriteInt32(int32(reflectValue.Int()))
+	case reflect.Int:     return encoder.WriteInt32(int32(reflectValue.Int()))
-	case reflect.Uint:   return encoder.WriteUint32(uint32(reflectValue.Uint()))
+	case reflect.Uint:    return encoder.WriteUint32(uint32(reflectValue.Uint()))
-	case reflect.Int8:   return encoder.WriteInt8(int8(reflectValue.Int()))
+	case reflect.Int8:    return encoder.WriteInt8(int8(reflectValue.Int()))
-	case reflect.Uint8:  return encoder.WriteUint8(uint8(reflectValue.Uint()))
+	case reflect.Uint8:   return encoder.WriteUint8(uint8(reflectValue.Uint()))
-	case reflect.Int16:  return encoder.WriteInt16(int16(reflectValue.Int()))
+	case reflect.Int16:   return encoder.WriteInt16(int16(reflectValue.Int()))
-	case reflect.Uint16: return encoder.WriteUint16(uint16(reflectValue.Uint()))
+	case reflect.Uint16:  return encoder.WriteUint16(uint16(reflectValue.Uint()))
-	case reflect.Int32:  return encoder.WriteInt32(int32(reflectValue.Int()))
+	case reflect.Int32:   return encoder.WriteInt32(int32(reflectValue.Int()))
-	case reflect.Uint32: return encoder.WriteUint32(uint32(reflectValue.Uint()))
+	case reflect.Uint32:  return encoder.WriteUint32(uint32(reflectValue.Uint()))
-	case reflect.Int64:  return encoder.WriteInt64(int64(reflectValue.Int()))
+	case reflect.Int64:   return encoder.WriteInt64(int64(reflectValue.Int()))
-	case reflect.Uint64: return encoder.WriteUint64(uint64(reflectValue.Uint()))
+	case reflect.Uint64:  return encoder.WriteUint64(uint64(reflectValue.Uint()))
+	case reflect.Float32: return encoder.WriteFloat32(float32(reflectValue.Float()))
+	case reflect.Float64: return encoder.WriteFloat64(float64(reflectValue.Float()))
+	case reflect.Bool:    return // SI has no payload
 	case reflect.String:
 		if reflectValue.Len() > MaxStructureLength {
 			return 0, ErrTooLong
@@ -86,9 +100,10 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	return n, fmt.Errorf("cannot encode type %T", value)
 }
 
-// DecodeAny decodes data and places it into destination, which must be a
+// DecodeAnyInto decodes data and places it into destination, which must be a
 // pointer to a supported type. See [EncodeAny] for a list of supported types.
-func DecodeAny(decoder *Decoder, destination any, tag Tag) (n int, err error) {
+// The head of the decoder must be at the start of the payload.
+func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error) {
 	reflectDestination := reflect.ValueOf(destination)
 	if reflectDestination.Kind() != reflect.Pointer {
 		return n, fmt.Errorf("expected pointer destination, not %v", destination)
@@ -96,6 +111,17 @@ func DecodeAny(decoder *Decoder, destination any, tag Tag) (n int, err error) {
 	return decodeAny(decoder, reflectDestination.Elem(), tag)
 }
 
+// DecodeAny is like [DecodeAnyInto], but it automatically creates the
+// destination from the tag and data. The head of the decoder must be at the
+// start of the payload.
+func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) {
+	destination, err := skeletonPointer(decoder, tag)
+	if err != nil { return nil, n, err }
+	nn, err := decodeAny(decoder, destination.Elem(), tag)
+	n += nn; if err != nil { return nil, n, err }
+	return destination.Elem().Interface(), n, err
+}
+
 // unknownSlicePlaceholder is inserted by skeletonValue and informs the program
 // that the destination for the slice needs to be generated based on the item
 // tag in the OTA.
@@ -105,7 +131,8 @@ var unknownSlicePlaceholderType = reflect.TypeOf(unknownSlicePlaceholder { })
 // decodeAny is internal to [DecodeAny]. It takes in an addressable
 // [reflect.Value] as the destination. If the decoded value cannot fit in the
 // destination, it skims over the payload, leaves the destination empty, and
-// returns without an error.
+// returns without an error. The head of the decoder must be at the start of the
+// payload.
 func decodeAny(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err error) {
 	n, err = decodeAnyOrError(decoder, destination, tag)
 	if _, ok := err.(errCantAssign); ok {
@@ -122,7 +149,7 @@ func decodeAny(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err
 // destination, it decodes nothing and returns an error of type errCantAssign,
 // except for the case of a mismatched OTA element tag, wherein it will skim
 // over the rest of the payload, leave the destination empty, and return without
-// an error.
+// an error. The head of the decoder must be at the start of the payload.
 func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err error) {
 	err = canSet(destination.Type(), tag)
 	if err != nil { return n, err }
@@ -130,7 +157,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 	switch tag.WithoutCN() {
 	case SI:
 		// SI: (none)
-		setInt(destination, uint64(tag.CN()))
+		setUint(destination, uint64(tag.CN()), 1)
 	case LI:
 		// LI: <value: IntN>
 		nn, err := decodeAndSetUint(decoder, destination, tag.CN() + 1)
@@ -152,7 +179,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 		buffer := make([]byte, length)
 		nn, err := decoder.Read(buffer)
 		n += nn; if err != nil { return n, err }
-		setByteArray(destination, buffer)
+		setString(destination, string(buffer))
 	case LBA:
 		// LBA: <length: UN> <data: U8>*
 		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
@@ -163,9 +190,16 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 		buffer := make([]byte, length)
 		nn, err = decoder.Read(buffer)
 		n += nn; if err != nil { return n, err }
-		setByteArray(destination, buffer)
+		setString(destination, string(buffer))
 	case OTA:
 		// OTA: <length: UN> <elementTag: tape.Tag> <values>*
+		oldDestination := destination
+		if isTypeAny(destination.Type()) {
+			// need a skeleton value if we are assigning to any.
+			value, err := skeletonValue(decoder, tag)
+			if err != nil { return n, err }
+			destination = value
+		}
 		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
 		n += nn; if err != nil { return n, err }
 		if length > uint64(MaxStructureLength) {
@@ -199,6 +233,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 				return n, err
 			}
 		}
+		oldDestination.Set(destination)
 	case KTV:
 		// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
 		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
@@ -206,13 +241,31 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 		if length > uint64(MaxStructureLength) {
 			return 0, ErrTooLong
 		}
+		lengthCast, err := Uint64ToIntSafe(length)
+		if err != nil { return n, err }
+
+		// im fucking so done dude. im so fucking done. this was
+		// supposed to only run when we need it but i guess it runs all
+		// the time, because when we get a map destination (a valid,
+		// allocated one) we break apart on SetMapIndex because of a nil
+		// map. yeah thats right. a fucking nil map panic. on the map we
+		// just allocated. but running this unconditionally (whether or
+		// not we receive an empty any value) actually makes it fucking
+		// work. go figure().
+		//
+		// (the map allocation functionality in skeletonPointer was
+		// removed after this comment was written)
+		value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast)
+		destination.Set(value)
+		destination = value
+		
 		destination.Clear()
-		for _ = range length {
+		for _ = range lengthCast {
 			key, nn, err := decoder.ReadUint16()
 			n += nn; if err != nil { return n, err }
 			itemTag, nn, err := decoder.ReadTag()
 			n += nn; if err != nil { return n, err }
-			value, err := skeletonValue(decoder, itemTag)
+			value, err := skeletonPointer(decoder, itemTag)
 			if err != nil { return n, err }
 			nn, err = decodeAny(decoder, value.Elem(), itemTag)
 			n += nn; if err != nil { return n, err }
@@ -234,17 +287,25 @@ func TagAny(value any) (Tag, error) {
 func tagAny(reflectValue reflect.Value) (Tag, error) {
 	// primitives
 	switch reflectValue.Kind() {
-	case reflect.Int:    return LSI.WithCN(3), nil
+	case reflect.Int:     return LSI.WithCN(3), nil
-	case reflect.Int8:   return LSI.WithCN(0), nil
+	case reflect.Int8:    return LSI.WithCN(0), nil
-	case reflect.Int16:  return LSI.WithCN(1), nil
+	case reflect.Int16:   return LSI.WithCN(1), nil
-	case reflect.Int32:  return LSI.WithCN(3), nil
+	case reflect.Int32:   return LSI.WithCN(3), nil
-	case reflect.Int64:  return LSI.WithCN(7), nil
+	case reflect.Int64:   return LSI.WithCN(7), nil
-	case reflect.Uint:   return LI.WithCN(3), nil
+	case reflect.Uint:    return LI.WithCN(3), nil
-	case reflect.Uint8:  return LI.WithCN(0), nil
+	case reflect.Uint8:   return LI.WithCN(0), nil
-	case reflect.Uint16: return LI.WithCN(1), nil
+	case reflect.Uint16:  return LI.WithCN(1), nil
-	case reflect.Uint32: return LI.WithCN(3), nil
+	case reflect.Uint32:  return LI.WithCN(3), nil
-	case reflect.Uint64: return LI.WithCN(7), nil
+	case reflect.Uint64:  return LI.WithCN(7), nil
-	case reflect.String: return bufferLenTag(reflectValue.Len()), nil
+	case reflect.Float32: return FP.WithCN(3), nil
+	case reflect.Float64: return FP.WithCN(7), nil
+	case reflect.Bool:
+		if reflectValue.Bool() {
+			return SI.WithCN(1), nil
+		} else {
+			return SI.WithCN(0), nil
+		}
+	case reflect.String:  return bufferLenTag(reflectValue.Len()), nil
 	}
 	if reflectValue.CanConvert(reflect.TypeOf(dummyBuffer)) {
 		return bufferLenTag(reflectValue.Len()), nil
@@ -311,12 +372,17 @@ func encodeAnyMap(encoder *Encoder, value any, tag Tag) (n int, err error) {
 }
 
 func canSet(destination reflect.Type, tag Tag) error {
+	// anything can be assigned to `any`
+	if isTypeAny(destination) {
+		return nil
+	}
 	switch tag.WithoutCN() {
 	case SI, LI, LSI:
 		switch destination.Kind() {
 		case
 			reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
-			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
+			reflect.Bool:
 		default:
 			return errCantAssignf("cannot assign integer to %v", destination)
 		}
@@ -327,6 +393,7 @@ func canSet(destination reflect.Type, tag Tag) error {
 			return errCantAssignf("cannot assign float to %v", destination)
 		}
 	case SBA, LBA:
+		if destination.Kind() == reflect.String { return nil }
 		if destination.Kind() != reflect.Slice {
 			return errCantAssignf("cannot assign byte array to %v", destination)
 		}
@@ -338,7 +405,11 @@ func canSet(destination reflect.Type, tag Tag) error {
 			return errCantAssignf("cannot assign array to %v", destination)
 		}
 	case KTV:
-		if destination != reflect.TypeOf(dummyMap) {
+		cantAssign :=
+			destination.Kind() != reflect.Map ||
+			destination.Key().Kind() != reflect.Uint16 ||
+			!isTypeAny(destination.Elem())
+		if cantAssign {
 			return errCantAssignf("cannot assign table to %v", destination)
 		}
 	default:
@@ -348,17 +419,45 @@ func canSet(destination reflect.Type, tag Tag) error {
 }
 
 // setInt expects a settable destination.
-func setInt[T int64 | uint64](destination reflect.Value, value T) {
+func setInt(destination reflect.Value, value int64, bytes int) {
 	switch {
 	case destination.CanInt():
 		destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type()))
 	case destination.CanUint():
 		destination.Set(reflect.ValueOf(value).Convert(destination.Type()))
+	case isTypeAny(destination.Type()):
+		switch {
+		case bytes > 4: destination.Set(reflect.ValueOf(int64(value)))
+		case bytes > 2: destination.Set(reflect.ValueOf(int32(value)))
+		case bytes > 1: destination.Set(reflect.ValueOf(int16(value)))
+		default:        destination.Set(reflect.ValueOf(int8(value)))
+		}
 	default:
 		panic("setInt called on an unsupported type")
 	}
 }
 
+// setUint expects a settable destination.
+func setUint(destination reflect.Value, value uint64, bytes int) {
+	switch {
+	case destination.Kind() == reflect.Bool:
+		destination.Set(reflect.ValueOf(value > 0))
+	case destination.CanInt():
+		destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type()))
+	case destination.CanUint():
+		destination.Set(reflect.ValueOf(value).Convert(destination.Type()))
+	case isTypeAny(destination.Type()):
+		switch {
+		case bytes > 4: destination.Set(reflect.ValueOf(uint64(value)))
+		case bytes > 2: destination.Set(reflect.ValueOf(uint32(value)))
+		case bytes > 1: destination.Set(reflect.ValueOf(uint16(value)))
+		default:        destination.Set(reflect.ValueOf(uint8(value)))
+		}
+	default:
+		panic("setUint called on an unsupported type")
+	}
+}
+
 // setFloat expects a settable destination.
 func setFloat(destination reflect.Value, value float64) {
 	destination.Set(reflect.ValueOf(value).Convert(destination.Type()))
@@ -369,11 +468,16 @@ func setByteArray(destination reflect.Value, value []byte) {
 	destination.Set(reflect.ValueOf(value))
 }
 
+// setString exepctes a settable destination
+func setString(destination reflect.Value, value string) {
+	destination.Set(reflect.ValueOf(value))
+}
+
 // decodeAndSetInt expects a settable destination.
 func decodeAndSetInt(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) {
 	value, nn, err := decoder.ReadIntN(bytes)
 	n += nn; if err != nil { return n, err }
-	setInt(destination, value)
+	setInt(destination, value, bytes)
 	return n, nil
 }
 
@@ -381,7 +485,7 @@ func decodeAndSetInt(decoder *Decoder, destination reflect.Value, bytes int) (n
 func decodeAndSetUint(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) {
 	value, nn, err := decoder.ReadUintN(bytes)
 	n += nn; if err != nil { return n, err }
-	setInt(destination, value)
+	setUint(destination, value, bytes)
 	return n, nil
 }
 
@@ -402,16 +506,27 @@ func decodeAndSetFloat(decoder *Decoder, destination reflect.Value, bytes int) (
 	return n, errCantAssignf("unsupported bit width float%d", bytes * 8)
 }
 
-// skeletonValue returns a pointer value. In order for it to be set, it must be
+// skeletonValue returns an addressable value. It can be set directly. The head
-// dereferenced using Elem().
+// of the decoder must be at the start of the payload when calling.
 func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) {
+	ptr, err := skeletonPointer(decoder, tag)
+	if err != nil { return reflect.Value { }, err }
+	return ptr.Elem(), nil
+}
+
+// skeletonPointer returns a pointer value. In order for it to be set, it must
+// be dereferenced using Elem(). The head of the decoder must be at the start of
+// the payload when calling.
+func skeletonPointer(decoder *Decoder, tag Tag) (reflect.Value, error) {
 	typ, err := typeOf(decoder, tag)
 	if err != nil { return reflect.Value { }, err }
-	return reflect.New(typ), nil
+	value := reflect.New(typ)
+	return value, nil
 }
 
 // typeOf returns the type of the current tag being decoded. It does not use up
-// the decoder, it only peeks.
+// the decoder, it only peeks. The head of the decoder must be at the start of
+// the payload when calling.
 func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) {
 	switch tag.WithoutCN() {
 	case SI:
@@ -438,8 +553,8 @@ func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) {
 		case 7: return reflect.TypeOf(float64(0)), nil
 		}
 		return nil, fmt.Errorf("unknown CN %d for FP", tag.CN())
-	case SBA: return reflect.SliceOf(reflect.TypeOf(byte(0))), nil
+	case SBA: return reflect.TypeOf(""), nil
-	case LBA: return reflect.SliceOf(reflect.TypeOf(byte(0))), nil
+	case LBA: return reflect.TypeOf(""), nil
 	case OTA:
 		elemTag, dimension, err := peekSlice(decoder, tag)
 		if err != nil { return nil, err }
@@ -455,6 +570,12 @@ func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) {
 	return nil, fmt.Errorf("unknown TN %d", tag.TN())
 }
 
+// isTypeAny returns whether the given reflect.Type is an interface with no
+// methods.
+func isTypeAny(typ reflect.Type) bool {
+	return typ.Kind() == reflect.Interface && typ.NumMethod() == 0
+}
+
 // peekSlice returns the element tag and dimension count of the OTA currently
 // being decoded. It does not use up the decoder, it only peeks.
 func peekSlice(decoder *Decoder, tag Tag) (Tag, int, error) {

tape/dynamic_test.go

@@ -1,11 +1,71 @@
 package tape
 
-import "fmt"
 import "bytes"
 import "testing"
 import "reflect"
 import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
 
+var samplePayloads = [][]byte {
+	/* int8     */ []byte { byte(LSI.WithCN(0)), 0x45 },
+	/* int16    */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 },
+	/* int32    */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
+	/* int64    */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
+	/* uint5    */ []byte { byte(SI.WithCN(12)) },
+	/* uint8    */ []byte { byte(LI.WithCN(0)), 0x45 },
+	/* uint16   */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 },
+	/* uint32   */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
+	/* uint64   */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
+	/* bool     */ []byte { byte(SI.WithCN(0)) },
+	/* bool     */ []byte { byte(SI.WithCN(1)) },
+	/* string   */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' },
+	/* []byte   */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' },
+	/* []string */ []byte {
+		byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)),
+		0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
+		0x05, 0x11, 0x11, 0x11, 0x11, 0x11,
+	},
+	/* map[uint16] any */ []byte {
+		byte(KTV.WithCN(0)), 2,
+		0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67,
+		0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB,
+	},
+	/* map[uint16] any */ []byte {
+		byte(KTV.WithCN(0)), 3,
+		0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00,
+		0x00, 0x02, 0x82, 'h', 'i',
+		0x00, 0x03, 0x21, 0x39, 0x92,
+	},
+}
+
+var sampleValues = []any {
+	/* int8     */ int8(0x45),
+	/* int16    */ int16(0x4567),
+	/* int32    */ int32(0x456789AB),
+	/* int64    */ int64(0x456789ABCDEF0123),
+	/* uint5    */ uint8(12),
+	/* uint8    */ uint8(0x45),
+	/* uint16   */ uint16(0x4567),
+	/* uint32   */ uint32(0x456789AB),
+	/* uint64   */ uint64(0x456789ABCDEF0123),
+	/* bool     */ false,
+	/* bool     */ true,
+	/* string   */ "pupever",
+	/* []byte   */ "blarg",
+	/* []string */ []string {
+		"\x45\x67\x89\xAB\xCD\xEF\x01\x23",
+		"\x11\x11\x11\x11\x11",
+	},
+	/* map[uint16] any */ map[uint16] any {
+		0x0223: int16(0x4567),
+		0x0224: uint32(0x456789AB),
+	},
+	/* map[uint16] any */ map[uint16] any {
+		0x0001: float32(489.5),
+		0x0002: "hi",
+		0x0003: uint16(0x3992),
+	},
+}
+
 type userDefinedInteger int16
 
 func TestEncodeAnyInt(test *testing.T) {
@@ -26,7 +86,11 @@ func TestEncodeAnyTable(test *testing.T) {
 		0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} },
 		0x2345: [][]int16 { []int16 { 0x5 }, []int16 { 0x17, -0xAAA } },
 		0x3456: userDefinedInteger(0x3921),
-	}, KTV.WithCN(0), tu.S(7).AddVar(
+		0x1F1F: float32(67.26),
+		0x0F0F: float64(5.3),
+		0xAAAA: false,
+		0xBBBB: true,
+	}, KTV.WithCN(0), tu.S(11).AddVar(
 		[]byte {
 			0xF3, 0xB9,
 			byte(LSI.WithCN(3)),
@@ -70,48 +134,42 @@ func TestEncodeAnyTable(test *testing.T) {
 			byte(LSI.WithCN(1)),
 			0x39, 0x21,
 		},
+		[]byte {
+			0x1F, 0x1F,
+			byte(FP.WithCN(3)),
+			0x42, 0x86, 0x85, 0x1F,
+		},
+		[]byte {
+			0x0F, 0x0F,
+			byte(FP.WithCN(7)),
+			0x40, 0x15, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33,
+		},
+		[]byte {
+			0xAA, 0xAA,
+			byte(SI.WithCN(0)),
+		},
+		[]byte {
+			0xBB, 0xBB,
+			byte(SI.WithCN(1)),
+		},
 	))
 	if err != nil { test.Fatal(err) }
 }
 
 func TestDecodeWrongType(test *testing.T) {
-	datas := [][]byte {
+	for index, data := range samplePayloads {
-		/* int8     */ []byte { byte(LSI.WithCN(0)), 0x45 },
-		/* int16    */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 },
-		/* int32    */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-		/* int64    */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-		/* uint5    */ []byte { byte(SI.WithCN(12)) },
-		/* uint8    */ []byte { byte(LI.WithCN(0)), 0x45 },
-		/* uint16   */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 },
-		/* uint32   */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-		/* uint64   */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-		/* string   */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' },
-		/* []byte   */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' },
-		/* []string */ []byte {
-			byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)),
-			0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
-			0x05, 0x11, 0x11, 0x11, 0x11, 0x11,
-		},
-		/* map[uint16] any */ []byte {
-			byte(KTV.WithCN(0)), 2,
-			0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67,
-			0x02, 0x23, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB,
-		},
-	}
-
-	for index, data := range datas {
 		test.Logf("data %2d %v [%s]", index, Tag(data[0]), tu.HexBytes(data[1:]))
 		// integers should only assign to other integers
-		if index > 8 {
+		if index > 10 {
 			cas := func(destination any) {
-				n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
+				n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
 				if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
 				reflectValue := reflect.ValueOf(destination).Elem()
 				if reflectValue.CanInt() {
 					if reflectValue.Int() != 0 {
 						test.Fatalf("destination not zero: %v", reflectValue.Elem().Interface())
 					}
-				} else {
+				} else if reflectValue.Kind() != reflect.Bool {
 					if reflectValue.Uint() != 0 {
 						test.Fatalf("destination not zero: %v", reflectValue.Elem().Interface())
 					}
@@ -136,9 +194,11 @@ func TestDecodeWrongType(test *testing.T) {
 			{ var dest uint32; cas(&dest) }
 			test.Log("- uint64")
 			{ var dest uint64; cas(&dest) }
+			test.Log("- bool")
+			{ var dest bool; cas(&dest) }
 		}
 		arrayCase := func(destination any) {
-			n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
+			n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
 			if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
 			reflectDestination := reflect.ValueOf(destination)
 			reflectValue := reflectDestination.Elem()
@@ -150,19 +210,19 @@ func TestDecodeWrongType(test *testing.T) {
 			}
 		}
 		// SBA/LBA types should only assign to other SBA/LBA types
-		if index != 9 && index != 10 {
+		if index != 11 && index != 12 {
 			test.Log("- string")
 			{ var dest string; arrayCase(&dest) }
 			test.Log("- []byte")
 			{ var dest []byte; arrayCase(&dest) }
 		}
 		// arrays should only assign to other arrays
-		if index != 11 {
+		if index != 13 {
 			test.Log("- []string")
 			{ var dest []string; arrayCase(&dest) }
 		}
 		// tables should only assign to other tables
-		if index != 12 {
+		if index != 14 && index != 15 {
 			test.Log("- map[uint16] any")
 			{ var dest = map[uint16] any { }; arrayCase(&dest) }
 		}
@@ -175,13 +235,42 @@ func TestEncodeDecodeAnyTable(test *testing.T) {
 		0x0102: uint32(2),
 		0x0103: int64(23432),
 		0x0104: int64(-88777),
-		0x0000: []byte("hi!"),
+		0x0000: "hi!",
 		0xFFFF: []uint16 { 0xBEE5, 0x7777 },
 		0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} },
+		0x1F1F: float32(67.26),
+		0x0F0F: float64(5.3),
 	}, nil)
 	if err != nil { test.Fatal(err) }
 }
 
+func TestEncodeDecodeAnyDestination(test *testing.T) {
+	var destination any
+	for index, data := range samplePayloads {
+		if _, isBool := sampleValues[index].(bool); isBool {
+			// test is invalid for bools because they are never
+			// created as a skeleton value
+			continue
+		}
+	
+		tag     := Tag(data[0])
+		payload := data[1:]
+		test.Logf("data %2d %v [%s]", index, tag, tu.HexBytes(payload))
+		n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(payload)), &destination, tag)
+		if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
+		got     := destination
+		correct := sampleValues[index]
+		test.Log("got:    ", tu.Describe(got))
+		test.Log("correct:", tu.Describe(correct))
+		if !reflect.DeepEqual(got, correct) {
+			test.Fatalf("values not equal")
+		}
+		if n != len(payload) {
+			test.Fatalf("n not equal: %d != %d", n, len(payload))
+		}
+	}
+}
+
 func TestPeekSlice(test *testing.T) {
 	buffer := bytes.NewBuffer([]byte {
 		2, byte(OTA.WithCN(3)),
@@ -241,70 +330,68 @@ func TestPeekSliceOnce(test *testing.T) {
 	}
 }
 
-func encAny(value any) ([]byte, Tag, int, error) {
+func TestTagAny(test *testing.T) {
-	tag, err := TagAny(value)
+	cases := [][2]any {
-	if err != nil { return nil, 0, 0, err }
+		[2]any { LSI.WithCN(3),  int(9)     },
-	buffer := bytes.Buffer { }
+		[2]any { LSI.WithCN(0),  int8(9)    },
-	encoder := NewEncoder(&buffer)
+		[2]any { LSI.WithCN(1),  int16(9)   },
-	n, err := EncodeAny(encoder, value, tag)
+		[2]any { LSI.WithCN(3),  int32(9)   },
-	if err != nil { return nil, 0, n, err }
+		[2]any { LSI.WithCN(7),  int64(9)   },
-	encoder.Flush()
+		[2]any { LI.WithCN(3),   uint(9)    },
-	return buffer.Bytes(), tag, n, nil
+		[2]any { LI.WithCN(0),   uint8(9)   },
+		[2]any { LI.WithCN(1),   uint16(9)  },
+		[2]any { LI.WithCN(3),   uint32(9)  },
+		[2]any { LI.WithCN(7),   uint64(9)  },
+		[2]any { FP.WithCN(3),   float32(9) },
+		[2]any { FP.WithCN(7),   float64(9) },
+		[2]any { SBA.WithCN(12), "small string" },
+		[2]any { SBA.WithCN(12), []byte("small string") },
+		[2]any { LBA.WithCN(0),  "this is a very long string that is long" },
+		[2]any { LBA.WithCN(0),  []byte("this is a very long string that is long") },
+		[2]any { LBA.WithCN(1),  lipsum },
+		[2]any { OTA.WithCN(0),  []int { 1, 2, 3, 4, 5 } },
+		[2]any { OTA.WithCN(0),  []string { "1, 2, 3, 4, 5" } },
+		[2]any { KTV.WithCN(0),  map[uint16] any {
+			0:  1,
+			1:  "wow",
+			2:  10.238,
+			45: -9,
+			9:  map[uint16] any { },
+		}},
+	}
+	for _, cas := range cases {
+		test.Log(cas)
+		got, err := TagAny(cas[1])
+		if err != nil { test.Fatal(err) }
+		if correct := cas[0].(Tag); correct != got {
+			test.Fatalf("wrong tag: %v != %v", got, correct)
+		}
+	}
 }
 
-func decAny(data []byte) (Tag, any, int, error) {
+func TestDecodeAny(test *testing.T) {
-	destination := map[uint16] any { }
+	for index, payload := range samplePayloads {
-	tag, err := TagAny(destination)
+		if _, isBool := sampleValues[index].(bool); isBool {
-	if err != nil { return 0, nil, 0, err }
+			// test is invalid for bools because they are never
-	n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data)), &destination, tag)
+			// created as a skeleton value
-	if err != nil { return 0, nil, n, err }
+			continue
-	return tag, destination, n, nil
+		}
-}
+		
-
+		correctValue := sampleValues[index]
-func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.Snake) error {
+		data := payload[1:]
-	bytes, tag, n, err := encAny(value)
+		decoder := NewDecoder(bytes.NewBuffer(data))
-	if err != nil { return err }
+		tag     := Tag(payload[0])
-	test.Log("n:      ", n)
+		decoded, n, err := DecodeAny(decoder, tag)
-	test.Log("tag:    ", tag)
+		test.Log("n:      ", n)
-	test.Log("got:    ", tu.HexBytes(bytes))
+		test.Log("tag:    ", tag)
-	test.Log("correct:", correctBytes)
+		test.Log("got:    ", tu.Describe(decoded))
-	if tag != correctTag {
+		test.Log("correct:", tu.Describe(correctValue))
-		return fmt.Errorf("tag not equal: %v != %v", tag, correctTag)
+		if err != nil { test.Fatal(err) }
-	}
+		if !reflect.DeepEqual(decoded, correctValue) {
-	if ok, n := correctBytes.Check(bytes); !ok {
+			test.Fatal("values not equal")
-		return fmt.Errorf("bytes not equal at index %d", n)
+		}
-	}
+		if n != len(data) {
-	if n != len(bytes) {
+			test.Fatalf("n not equal: %d != %d", n, len(data))
-		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+		}
-	}
+	}
-	return nil
-}
-
-func testEncodeDecodeAny(test *testing.T, value, correctValue any) error {
-	if correctValue == nil { 
-		correctValue = value
-	}
-
-	test.Log("encoding...")
-	bytes, tag, n, err := encAny(value)
-	if err != nil { return err }
-	test.Log("n:  ", n)
-	test.Log("tag:", tag)
-	test.Log("got:", tu.HexBytes(bytes))
-	test.Log("decoding...", tag)
-	if n != len(bytes) {
-		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
-	}
-	
-	_, decoded, n, err := decAny(bytes)
-	if err != nil { return err }
-	test.Log("got:    ", tu.Describe(decoded))
-	test.Log("correct:", tu.Describe(correctValue))
-	if !reflect.DeepEqual(decoded, correctValue) {
-		return fmt.Errorf("values not equal")
-	}
-	if n != len(bytes) {
-		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
-	}
-	return nil
 }

tape/misc_test.go

@@ -0,0 +1,75 @@
+package tape
+
+import "fmt"
+import "bytes"
+import "testing"
+import "reflect"
+import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
+
+func encAny(value any) ([]byte, Tag, int, error) {
+	tag, err := TagAny(value)
+	if err != nil { return nil, 0, 0, err }
+	buffer := bytes.Buffer { }
+	encoder := NewEncoder(&buffer)
+	n, err := EncodeAny(encoder, value, tag)
+	if err != nil { return nil, 0, n, err }
+	encoder.Flush()
+	return buffer.Bytes(), tag, n, nil
+}
+
+func decAny(data []byte) (Tag, any, int, error) {
+	destination := map[uint16] any { }
+	tag, err := TagAny(destination)
+	if err != nil { return 0, nil, 0, err }
+	n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data)), &destination, tag)
+	if err != nil { return 0, nil, n, err }
+	return tag, destination, n, nil
+}
+
+func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.Snake) error {
+	bytes, tag, n, err := encAny(value)
+	if err != nil { return err }
+	test.Log("n:      ", n)
+	test.Log("tag:    ", tag)
+	test.Log("got:    ", tu.HexBytes(bytes))
+	test.Log("correct:", correctBytes)
+	if tag != correctTag {
+		return fmt.Errorf("tag not equal: %v != %v", tag, correctTag)
+	}
+	if ok, n := correctBytes.Check(bytes); !ok {
+		return fmt.Errorf("bytes not equal at index %d", n)
+	}
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	return nil
+}
+
+func testEncodeDecodeAny(test *testing.T, value, correctValue any) error {
+	if correctValue == nil { 
+		correctValue = value
+	}
+
+	test.Log("encoding...")
+	bytes, tag, n, err := encAny(value)
+	if err != nil { return err }
+	test.Log("n:  ", n)
+	test.Log("tag:", tag)
+	test.Log("got:", tu.HexBytes(bytes))
+	test.Log("decoding...", tag)
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	
+	_, decoded, n, err := decAny(bytes)
+	if err != nil { return err }
+	test.Log("got:    ", tu.Describe(decoded))
+	test.Log("correct:", tu.Describe(correctValue))
+	if !reflect.DeepEqual(decoded, correctValue) {
+		return fmt.Errorf("values not equal")
+	}
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	return nil
+}

tape/strings.go

@@ -0,0 +1,11 @@
+package tape
+
+const lipsum = `Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.`

tape/tag.go

@@ -18,6 +18,17 @@ type Tag byte; const (
 	CNLimit Tag = 32     // All valid CNs are < CNLimit
 )
 
+// what the first nybble of a tag means:
+//
+//   0-1 : SI 
+//   2-3 : LI 
+//   4-5 : LSI
+//   6-7 : FP 
+//   8-9 : SBA
+//   A-B : LBA
+//   C-D : OTA
+//   E-F : KTV
+
 func (tag Tag) TN() int {
 	return int(tag >> 5)
 }
@@ -67,6 +78,6 @@ func bufferLenTag(length int) Tag {
 	if length < int(CNLimit) {
 		return SBA.WithCN(length)
 	} else {
-		return LBA.WithCN(IntBytes(uint64(length)))
+		return LBA.WithCN(IntBytes(uint64(length)) - 1)
 	}
 }