tape: Fix TestEncodeDecodeAnyTable

Reviewed-on: #3

.gitignore

@@ -1 +1,2 @@
 /generate/test
+/debug

cmd/hopp-generate/main.go

@@ -4,6 +4,7 @@ import "os"
 import "fmt"
 import "strings"
 import "path/filepath"
+import "git.tebibyte.media/sashakoshka/goparse"
 import "git.tebibyte.media/sashakoshka/hopp/generate"
 
 func main() {
@@ -18,7 +19,7 @@ func main() {
 	input, err := os.Open(source)
 	handleErr(1, err)
 	defer input.Close()
-	protocol, err := generate.ParseReader(input)
+	protocol, err := generate.ParseReader(source, input)
 	handleErr(1, err)
 
 	absDestination, err := filepath.Abs(destination)
@@ -30,14 +31,18 @@ func main() {
 
 	output, err := os.Create(destination)
 	handleErr(1, err)
-	err = protocol.Generate(output, packageName)
+	generator := generate.Generator {
+		Output:      output,
+		PackageName: packageName,
+	}
+	_, err = generator.Generate(protocol)
 	handleErr(1, err)
 	fmt.Fprintf(os.Stderr, "%s: OK\n", name)
 }
 
 func handleErr(code int, err error) {
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[0], err)
+		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[0], parse.Format(err))
 		os.Exit(code)
 	}
 }

connection.go

@@ -2,7 +2,7 @@ package hopp
 
 import "io"
 import "net"
-// import "time"
+import "time"
 
 const defaultSizeLimit int64 = 1024 * 1024 // 1 megabyte
 
@@ -23,8 +23,13 @@ type Conn interface {
 	// be called in a loop to avoid the connection locking up.
 	AcceptTrans() (Trans, error)
 
+	// SetDeadline operates is [net.Conn.SetDeadline] but for OpenTrans
+	// and AcceptTrans calls.
+	SetDeadline(t time.Time) error	
 	// SetSizeLimit sets a limit (in bytes) for how large messages can be.
-	// By default, this limit is 1 megabyte.
+	// By default, this limit is 1 megabyte. Note that this is only
+	// enforced when sending and receiving byte slices, and it does not
+	// apply to [Trans.SendWriter] or [Trans.ReceiveReader].
 	SetSizeLimit(limit int64)
 }
 
@@ -39,8 +44,6 @@ type Trans interface {
 	// ID returns the transaction ID. This must not change, and it must be
 	// unique within the connection. This method is safe for concurrent use.
 	ID() int64
-
-	// TODO: add methods for setting send and receive deadlines
 		
 	// Send sends a message. This method is not safe for concurrent use.
 	Send(method uint16, data []byte) error
@@ -57,4 +60,12 @@ type Trans interface {
 	// previously opened through this function will be discarded. This
 	// method is not safe for concurrent use, and neither is its result.
 	ReceiveReader() (method uint16, data io.Reader, err error)
+
+	// See the documentation for [net.Conn.SetDeadline].
+	SetDeadline(time.Time) error
+	// TODO
+	// // See the documentation for [net.Conn.SetReadDeadline].
+	// SetReadDeadline(t time.Time) error
+	// // See the documentation for [net.Conn.SetWriteDeadline].
+	// SetWriteDeadline(t time.Time) error
 }

design/pdl-language.md

@@ -30,6 +30,7 @@ PDL allows defining a protocol using HOPP and TAPE.
 | []\<TYPE\> | OTA     |  * | Array of any type[^1]
 | Table      | KTV     |  * | Table with undefined schema
 | {...}      | KTV     |  * | Table with defined schema
+| Any        | *       |  * | Value of an undefined type
 
 [^1]: Excluding SI and SBA. I5 and U5 cannot be used in an array, but String and
 Buffer are simply forced to use their "long" variant.

generate/generate.go

@@ -235,17 +235,13 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e
 		this.resolveMessageName(message.Name))
 	n += nn; if err != nil { return n, err }
 	this.push()
-		nn, err = this.iprintf("tag := ")
+		tagVar, nn, err := this.generateTag(message.Type, "(*this)")
 		n += nn; if err != nil { return n, err }
-		nn, err = this.generateTag(message.Type, "(*this)")
-		n += nn; if err != nil { return n, err }
-		nn, err = this.println()
-		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("nn, err := encoder.WriteTag(tag)\n")
+		nn, err = this.iprintf("nn, err := encoder.WriteTag(%s)\n", tagVar)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
-		nn, err = this.generateEncodeValue(message.Type, "(*this)", "tag")
+		nn, err = this.generateEncodeValue(message.Type, "(*this)", tagVar)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf("return n, nil\n")
 		n += nn; if err != nil { return n, err }
@@ -332,7 +328,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateType(typ)
 		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("%s)\n", valueSource)
+		nn, err = this.printf("(%s))\n", valueSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
@@ -343,7 +339,14 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 	case TypeBuffer:
 		// SBA: <data: U8>*
 		// LBA: <length: UN> <data: U8>*
-		nn, err := this.iprintf("if %s.Is(tape.LBA) {\n", tagSource)
+		nn, err := this.iprintf("if len(%s) > tape.MaxStructureLength {\n", valueSource)
+		n += nn; if err != nil { return n, err }
+		this.push()
+			nn, err = this.iprintf("return n, tape.ErrTooLong\n")
+		this.pop()
+		nn, err = this.iprintf("}\n")
+		n += nn; if err != nil { return n, err }
+		nn, err = this.iprintf("if %s.Is(tape.LBA) {\n", tagSource)
 		n += nn; if err != nil { return n, err }
 		this.push()
 			nn, err = this.iprintf(
@@ -355,14 +358,20 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		this.pop()
 		nn, err = this.iprintf("}\n")
 		n += nn; if err != nil { return n, err }
-		
 		nn, err = this.iprintf("nn, err = encoder.Write([]byte(%s))\n", valueSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
 	case TypeArray:
 		// OTA: <length: UN> <elementTag: tape.Tag> <values>*
-		nn, err := this.iprintf(
+		nn, err := this.iprintf("if len(%s) > tape.MaxStructureLength {\n", valueSource)
+		n += nn; if err != nil { return n, err }
+		this.push()
+			nn, err = this.iprintf("return n, tape.ErrTooLong\n")
+		this.pop()
+		nn, err = this.iprintf("}\n")
+		n += nn; if err != nil { return n, err }
+		nn, err = this.iprintf(
 			"nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN())\n",
 			valueSource, tagSource)
 		n += nn; if err != nil { return n, err }
@@ -385,15 +394,11 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 			this.push()
 				nn, err = this.iprintf("_ = item\n")
 				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("tag := ")
+				tagVar, nn, err := this.generateTag(typ.Element, "item")
 				n += nn; if err != nil { return n, err }
-				nn, err = this.generateTag(typ.Element, "item")
+				nn, err = this.iprintf("if %s.Is(tape.SBA) { continue }\n", tagVar)
 				n += nn; if err != nil { return n, err }
-				nn, err = this.println()
-				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("if tag.Is(tape.SBA) { continue }\n")
-				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("if tag.CN() > itemTag.CN() { itemTag = tag }\n")
+				nn, err = this.iprintf("if %s.CN() > itemTag.CN() { itemTag = %s }\n", tagVar, tagVar)
 				n += nn; if err != nil { return n, err }
 			this.pop()
 			nn, err = this.iprintf("}\n")
@@ -417,7 +422,14 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		n += nn; if err != nil { return n, err }
 	case TypeTable:
 		// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
-		nn, err := this.iprintf(
+		nn, err := this.iprintf("if len(%s) > tape.MaxStructureLength {\n", valueSource)
+		n += nn; if err != nil { return n, err }
+		this.push()
+			nn, err = this.iprintf("return n, tape.ErrTooLong\n")
+		this.pop()
+		nn, err = this.iprintf("}\n")
+		n += nn; if err != nil { return n, err }
+		nn, err = this.iprintf(
 			"nn, err = tape.EncodeAny(encoder, %s, %s)\n",
 			valueSource, tagSource)
 		n += nn; if err != nil { return n, err }
@@ -425,7 +437,14 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		n += nn; if err != nil { return n, err }
 	case TypeTableDefined:
 		// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
-		nn, err := this.iprintf(
+		nn, err := this.iprintf("if %d > tape.MaxStructureLength {\n", len(typ.Fields))
+		n += nn; if err != nil { return n, err }
+		this.push()
+			nn, err = this.iprintf("return n, tape.ErrTooLong\n")
+		this.pop()
+		nn, err = this.iprintf("}\n")
+		n += nn; if err != nil { return n, err }
+		nn, err = this.iprintf(
 			"nn, err = encoder.WriteUintN(%d, %s.CN())\n",
 			len(typ.Fields), tagSource)
 		n += nn; if err != nil { return n, err }
@@ -434,25 +453,19 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		nn, err = this.iprintf("{\n")
 		n += nn; if err != nil { return n, err }
 		this.push()
-			nn, err = this.iprintf("var tag tape.Tag\n")
-			n += nn; if err != nil { return n, err }
 			for key, field := range typ.Fields {
 				nn, err = this.iprintf("nn, err = encoder.WriteUint16(0x%04X)\n", key)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("tag = ")
-				n += nn; if err != nil { return n, err }
 				fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name)
-				nn, err = this.generateTag(field.Type, fieldSource)
+				tagVar, nn, err := this.generateTag(field.Type, fieldSource)
 				n += nn; if err != nil { return n, err }
-				nn, err = this.println()
-				n += nn; if err != nil { return n, err }
-				nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(tag))\n")
+				nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(%s))\n", tagVar)
 				n += nn; if err != nil { return n, err }
 				nn, err = this.generateErrorCheck()
 				n += nn; if err != nil { return n, err }
-				nn, err = this.generateEncodeValue(field.Type, fieldSource, "tag")
+				nn, err = this.generateEncodeValue(field.Type, fieldSource, tagVar)
 				n += nn; if err != nil { return n, err }
 			}
 		this.pop()
@@ -464,6 +477,12 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
+	case TypeAny:
+		// WHATEVER: [WHATEVER]
+		nn, err := this.iprintf("nn, err = tape.EncodeAny(encoder, %s, %s)\n", valueSource, tagSource)
+		n += nn; if err != nil { return n, err }
+		nn, err = this.generateErrorCheck()
+		n += nn; if err != nil { return n, err }
 	default:
 		panic(fmt.Errorf("unknown type: %T", typ))
 	}
@@ -527,6 +546,10 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 		destinationVar := this.newTemporaryVar("destination")
 		nn, err := this.iprintf("var %s ", destinationVar)
 		n += nn; if err != nil { return n, err }
+		nn, err = this.generateType(typ)
+		n += nn; if err != nil { return n, err }
+		nn, err = this.print("\n")
+		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf("%s, nn, err = decoder.ReadFloat%d()\n", destinationVar, typ.Bits)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -564,7 +587,14 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 		this.pop()
 		nn, err = this.iprintf("}\n")
 		n += nn; if err != nil { return n, err }
-		nn, err = this.iprintf("buffer := make([]byte, int(%s))\n", lengthVar)
+		nn, err = this.iprintf("if %s > uint64(tape.MaxStructureLength) {\n", lengthVar)
+		n += nn; if err != nil { return n, err }
+		this.push()
+			nn, err = this.iprintf("return n, tape.ErrTooLong\n")
+		this.pop()
+		nn, err = this.iprintf("}\n")
+		n += nn; if err != nil { return n, err }
+		nn, err = this.iprintf("buffer := make([]byte, %s)\n", lengthVar)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf("nn, err = decoder.Read(buffer)\n")
 		n += nn; if err != nil { return n, err }
@@ -589,7 +619,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 	case TypeTable:
 		// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
 		nn, err := this.iprintf(
-			"nn, err = tape.DecodeAny(decoder, %s, %s)\n",
+			"nn, err = tape.DecodeAnyInto(decoder, %s, %s)\n",
 			valueSource, tagSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -604,6 +634,12 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
 		n += nn; if err != nil { return n, err }
+	case TypeAny:
+		// WHATEVER: [WHATEVER]
+		nn, err := this.iprintf("*%s, nn, err = tape.DecodeAny(decoder, %s)\n", valueSource, tagSource)
+		n += nn; if err != nil { return n, err }
+		nn, err = this.generateErrorCheck()
+		n += nn; if err != nil { return n, err }
 	default:
 		panic(fmt.Errorf("unknown type: %T", typ))
 	}
@@ -666,6 +702,13 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			lengthVar := this.newTemporaryVar("length")
 			nn, err := this.iprintf("var %s uint64\n", lengthVar)
 			n += nn; if err != nil { return n, err }
+			nn, err = this.iprintf("if %s > uint64(tape.MaxStructureLength) {\n", lengthVar)
+			n += nn; if err != nil { return n, err }
+			this.push()
+				nn, err = this.iprintf("return n, tape.ErrTooLong\n")
+			this.pop()
+			nn, err = this.iprintf("}\n")
+			n += nn; if err != nil { return n, err }
 			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
@@ -735,6 +778,13 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			lengthVar := this.newTemporaryVar("length")
 			nn, err := this.iprintf("var %s uint64\n", lengthVar)
 			n += nn; if err != nil { return n, err }
+			nn, err = this.iprintf("if %s > uint64(tape.MaxStructureLength) {\n", lengthVar)
+			n += nn; if err != nil { return n, err }
+			this.push()
+				nn, err = this.iprintf("return n, tape.ErrTooLong\n")
+			this.pop()
+			nn, err = this.iprintf("}\n")
+			n += nn; if err != nil { return n, err }
 			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
@@ -747,7 +797,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			// problems
 
 			// read fields
-			nn, err = this.iprintf("for _ = range int(%s) {\n", lengthVar)
+			nn, err = this.iprintf("for _ = range %s {\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			this.push()
 				// read field header
@@ -872,49 +922,60 @@ func (this *Generator) generateErrorCheck() (n int, err error) {
 	return this.iprintf("n += nn; if err != nil { return n, err }\n")
 }
 
+func (this *Generator) generateBareErrorCheck() (n int, err error) {
+	return this.iprintf("if err != nil { return n, err }\n")
+}
+
 // generateTag generates the preferred TN and CN for the given type and value.
-// The generated code is INLINE.
-func (this *Generator) generateTag(typ Type, source string) (n int, err error) {
+// The generated code is a BLOCK.
+func (this *Generator) generateTag(typ Type, source string) (tagVar string, n int, err error) {
+	tagVar = this.newTemporaryVar("tag")
 	switch typ := typ.(type) {
 	case TypeInt:
 		if typ.Bits <= 5 {
-			nn, err := this.printf("tape.SI.WithCN(int(%s))", source)
-			n += nn; if err != nil { return n, err }
+			nn, err := this.iprintf("%s := tape.SI.WithCN(int(%s))\n", tagVar, source)
+			n += nn; if err != nil { return tagVar, n, err }
 		} else if typ.Signed {
-			nn, err := this.printf("tape.LSI.WithCN(%d)", bitsToCN(typ.Bits))
-			n += nn; if err != nil { return n, err }
+			nn, err := this.iprintf("%s := tape.LSI.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits))
+			n += nn; if err != nil { return tagVar, n, err }
 		} else {
-			nn, err := this.printf("tape.LI.WithCN(%d)", bitsToCN(typ.Bits))
-			n += nn; if err != nil { return n, err }
+			nn, err := this.iprintf("%s := tape.LI.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits))
+			n += nn; if err != nil { return tagVar, n, err }
 		}
 	case TypeFloat:
-		nn, err := this.printf("tape.FP.WithCN(%d)", bitsToCN(typ.Bits))
-		n += nn; if err != nil { return n, err }
+		nn, err := this.iprintf("%s := tape.FP.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits))
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeString:
-		nn, err := this.printf("tape.StringTag(string(%s))", source)
-		n += nn; if err != nil { return n, err }
+		nn, err := this.iprintf("%s := tape.StringTag(string(%s))\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeBuffer:
-		nn, err := this.printf("tape.BufferTag([]byte(%s))", source)
-		n += nn; if err != nil { return n, err }
+		nn, err := this.iprintf("%s := tape.BufferTag([]byte(%s))\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeArray:
-		nn, err := this.printf("tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))))", source)
-		n += nn; if err != nil { return n, err }
+		nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeTable:
-		nn, err := this.printf("tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))))", source)
-		n += nn; if err != nil { return n, err }
+		nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeTableDefined:
-		nn, err := this.printf("tape.KTV.WithCN(%d)", tape.IntBytes(uint64(len(typ.Fields))))
-		n += nn; if err != nil { return n, err }
+		nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields))))
+		n += nn; if err != nil { return tagVar, n, err }
 	case TypeNamed:
 		resolved, err := this.resolveTypeName(typ.Name)
-		if err != nil { return n, err }
-		nn, err := this.generateTag(resolved, source)
-		n += nn; if err != nil { return n, err }
+		if err != nil { return tagVar, n, err }
+		subTagVar, nn, err := this.generateTag(resolved, source)
+		n += nn; if err != nil { return tagVar, n, err }
+		tagVar = subTagVar
+	case TypeAny:
+		nn, err := this.iprintf("%s, err := tape.TagAny(%s)\n", tagVar, source)
+		n += nn; if err != nil { return tagVar, n, err }
+		nn, err = this.generateBareErrorCheck()
+		n += nn; if err != nil { return tagVar, n, err }
 	default:
 		panic(fmt.Errorf("unknown type: %T", typ))
 	}
 
-	return n, nil
+	return tagVar, n, nil
 }
 
 // generateTN generates the appropriate TN for the given type. The generated
@@ -957,6 +1018,8 @@ func (this *Generator) generateTN(typ Type) (n int, err error) {
 		if err != nil { return n, err }
 		nn, err := this.generateTN(resolved)
 		n += nn; if err != nil { return n, err }
+	default:
+		panic(fmt.Errorf("unknown type: %T", typ))
 	}
 
 	return n, nil
@@ -1011,6 +1074,11 @@ func (this *Generator) generateType(typ Type) (n int, err error) {
 	case TypeNamed:
 		nn, err := this.print(typ.Name)
 		n += nn; if err != nil { return n, err }
+	case TypeAny:
+		nn, err := this.print("any")
+		n += nn; if err != nil { return n, err }
+	default:
+		panic(fmt.Errorf("unknown type: %T", typ))
 	}
 	return n, nil
 }
@@ -1040,12 +1108,17 @@ func (this *Generator) generateTypeTableDefined(typ TypeTableDefined) (n int, er
 // by tagSource can be assigned to a Go destination generated from typ. The
 // generated code is INLINE.
 func (this *Generator) generateCanAssign(typ Type, tagSource string) (n int, err error) {
-	nn, err := this.printf("canAssign(")
-	n += nn; if err != nil { return n, err }
-	nn, err = this.generateTN(typ)
-	n += nn; if err != nil { return n, err }
-	nn, err = this.printf(", %s)", tagSource)
-	n += nn; if err != nil { return n, err }
+	if _, ok := typ.(TypeAny); ok {
+		nn, err := this.printf("true")
+		n += nn; if err != nil { return n, err }
+	} else {
+		nn, err := this.printf("canAssign(")
+		n += nn; if err != nil { return n, err }
+		nn, err = this.generateTN(typ)
+		n += nn; if err != nil { return n, err }
+		nn, err = this.printf(", %s)", tagSource)
+		n += nn; if err != nil { return n, err }
+	}
 	return n, nil
 }
 

generate/generate_test.go

@@ -62,6 +62,27 @@ func init() {
 			},
 		},
 	}
+	exampleProtocol.Messages[0x0005] = Message {
+		Name: "Dynamic",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "AU8", Type: TypeAny { } },
+				0x0001: Field { Name: "AU16", Type: TypeAny { } },
+				0x0002: Field { Name: "AU32", Type: TypeAny { } },
+				0x0003: Field { Name: "AU64", Type: TypeAny { } },
+				0x0004: Field { Name: "AI8", Type: TypeAny { } },
+				0x0005: Field { Name: "AI16", Type: TypeAny { } },
+				0x0006: Field { Name: "AI32", Type: TypeAny { } },
+				0x0007: Field { Name: "AI64", Type: TypeAny { } },
+				0x0008: Field { Name: "AF32", Type: TypeAny { } },
+				0x0009: Field { Name: "AF64", Type: TypeAny { } },
+				0x000A: Field { Name: "AString", Type: TypeAny { } },
+				0x000B: Field { Name: "AArray", Type: TypeAny { } },
+				0x000C: Field { Name: "ATable", Type: TypeAny { } },
+				0x000D: Field { Name: "T0", Type: TypeTable { } },
+			},
+		},
+	}
 	exampleProtocol.Types["User"] = TypeTableDefined {
 		Fields: map[uint16] Field {
 			0x0000: Field { Name: "Name",      Type: TypeString { } },
@@ -196,6 +217,53 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 				[]byte { 0x00, 0x0D, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
 				[]byte { 0x00, 0x0E, 0x47, 0x9C, 0x6E, 0xF6, 0x43, 0xEF, 0x1E, 0xCB, 0x37 },
 				))
+		log.Println("MessageDynamic")
+		messageDynamic := MessageDynamic {
+			AU8:     uint8(0x23),
+			AU16:    uint16(0x3247),
+			AU32:    uint32(0x87324523),
+			AU64:    uint64(0x3284029034098234),
+			AI8:     int8(0x23),
+			AI16:    int16(0x3247),
+			AI32:    int32(0x57324523),
+			AI64:    int64(0x3284029034098234),
+			AF32:    float32(2342.2378),
+			AF64:    float64(324.8899992),
+			AString: "fox bed",
+			AArray:  []int16 { 0x7, 0x6, 0x5, 0x4 },
+			ATable: map[uint16] any {
+				0x0001: int8(0x8),
+				0x0002: float64(4.4),
+			},
+			T0: map[uint16] any {
+				0x0001: float32(489.5),
+				0x0002: "hi",
+				0x0003: uint16(0x3992),
+			},
+		}
+		testEncodeDecode(
+			&messageDynamic,
+			tu.S(0xE1, 14).AddVar(
+				[]byte { 0x00, 0x00, 0x20, 0x23 },
+				[]byte { 0x00, 0x01, 0x21, 0x32, 0x47 },
+				[]byte { 0x00, 0x02, 0x23, 0x87, 0x32, 0x45, 0x23 },
+				[]byte { 0x00, 0x03, 0x27, 0x32, 0x84, 0x02, 0x90, 0x34, 0x09, 0x82, 0x34 },
+				[]byte { 0x00, 0x04, 0x40, 0x23 },
+				[]byte { 0x00, 0x05, 0x41, 0x32, 0x47 },
+				[]byte { 0x00, 0x06, 0x43, 0x57, 0x32, 0x45, 0x23 },
+				[]byte { 0x00, 0x07, 0x47, 0x32, 0x84, 0x02, 0x90, 0x34, 0x09, 0x82, 0x34 },
+				[]byte { 0x00, 0x08, 0x63, 0x45, 0x12, 0x63, 0xCE },
+				[]byte { 0x00, 0x09, 0x67, 0x40, 0x74, 0x4E, 0x3D, 0x6F, 0xCD, 0x17, 0x75 },
+				[]byte { 0x00, 0x0A, 0x87, 'f', 'o', 'x', ' ', 'b', 'e', 'd' },
+				[]byte { 0x00, 0x0B, 0xC4, 0x00, 0x07, 0x00, 0x06, 0x00, 0x05, 0x00, 0x04 },
+				[]byte { 0x00, 0x0C, 0xE1, 0x02,
+					0x00, 0x01, 0x20, 0x08,
+					0x00, 0x02, 0x67, 0x40, 0x11, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A },
+				[]byte { 0x00, 0x0D, 0xE1, 0x03,
+					0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00,
+					0x00, 0x02, 0x82, 'h', 'i',
+					0x00, 0x03, 0x21, 0x39, 0x92 },
+				))
 	`)
 }
 

generate/parse.go

@@ -25,8 +25,8 @@ func defaultProtocol() Protocol {
 	}
 }
 
-func ParseReader(reader io.Reader) (*Protocol, error) {
-	lx, err := Lex("test.pdl", reader)
+func ParseReader(fileName string, reader io.Reader) (*Protocol, error) {
+	lx, err := Lex(fileName, reader)
 	if err != nil { return nil, err }
 	return Parse(lx)
 }
@@ -116,6 +116,7 @@ func (this *parser) parseType() (Type, error) {
 		case "String": return TypeString { }, this.Next()
 		case "Buffer": return TypeBuffer { }, this.Next()
 		case "Table":  return TypeTable { }, this.Next()
+		case "Any":    return TypeAny { }, this.Next()
 		}
 		return this.parseTypeNamed()
 	case TokenLBracket:

generate/parse_test.go

@@ -31,9 +31,10 @@ func TestParse(test *testing.T) {
 			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
 		},
 	}
+	correct.Types["Anything"] = TypeAny { }
 	test.Log("CORRECT:", &correct)
 	
-	got, err := ParseReader(strings.NewReader(`
+	got, err := ParseReader("test.pdl", strings.NewReader(`
 	M0000 Connect {
 		0000 Name String,
 		0001 Password String,
@@ -48,6 +49,8 @@ func TestParse(test *testing.T) {
 		0001 Bio       String,
 		0002 Followers U32,
 	}
+
+	Anything Any
 	`))
 	if err != nil { test.Fatal(parse.Format(err)) }
 	test.Log("GOT:    ", got)

generate/protocol.go

@@ -99,6 +99,12 @@ func (typ TypeNamed) String() string {
 	return typ.Name
 }
 
+type TypeAny struct { }
+
+func (typ TypeAny) String() string {
+	return "Any"
+}
+
 func HashType(typ Type) [16]byte {
 	// TODO: if we ever want to make the compiler more efficient, this would
 	// be a good place to start, complex string concatenation in a hot path

internal/testutil/testutil.go

@@ -119,6 +119,10 @@ type describer struct {
 }
 
 func (this *describer) describe(value reflect.Value) {
+	if !value.IsValid() {
+		this.printf("<invalid>")
+		return
+	}
 	value = reflect.ValueOf(value.Interface())
 	switch value.Kind() {
 	case reflect.Array, reflect.Slice:

metadapta.go

@@ -1,9 +1,11 @@
 package hopp
 
 import "io"
+import "os"
 import "fmt"
 import "net"
 import "sync"
+import "time"
 import "sync/atomic"
 import "git.tebibyte.media/sashakoshka/go-util/sync"
 
@@ -108,6 +110,10 @@ func (this *a) AcceptTrans() (Trans, error) {
 	}
 }
 
+func (this *a) SetDeadline(t time.Time) error {
+	return this.underlying.SetDeadline(t)
+}
+
 func (this *a) SetSizeLimit(limit int64) {
 	this.sizeLimit = limit
 }
@@ -212,6 +218,10 @@ type transA struct {
 	currentWriter io.Closer
 	writeBuffer   []byte
 	closed        atomic.Bool
+	closeErr      error
+	
+	deadline      *time.Timer
+	deadlineLock  sync.Mutex
 }
 
 func (this *transA) Close() error {
@@ -221,6 +231,11 @@ func (this *transA) Close() error {
 	return err
 }
 
+func (this *transA) closeWithError(err error) error {
+	this.closeErr = err
+	return this.Close()
+}
+
 func (this *transA) closeDontUnlist() (err error) {
 	// MUST be goroutine safe
 	this.incoming.Close()
@@ -269,9 +284,9 @@ func (this *transA) Receive() (method uint16, data []byte, err error) {
 }
 
 func (this *transA) ReceiveReader() (uint16, io.Reader, error) {
-	// if the transaction has been closed, return an io.EOF
-	if this.closed.Load() {
-		return 0, nil, io.EOF
+	// if the transaction has been closed, return an appropriate error.
+	if err := this.errIfClosed(); err != nil {
+		return 0, nil, err
 	}
 
 	// drain previous reader if necessary
@@ -289,6 +304,54 @@ func (this *transA) ReceiveReader() (uint16, io.Reader, error) {
 	return method, reader, nil
 }
 
+func (this *transA) SetDeadline(t time.Time) error {
+	this.deadlineLock.Lock()
+	defer this.deadlineLock.Unlock()
+
+	if t == (time.Time { }) {
+		if this.deadline != nil {
+			this.deadline.Stop()
+		}
+		return nil
+	}
+	
+	until := time.Until(t)
+	if this.deadline == nil {
+		this.deadline.Reset(until)
+		return nil
+	}
+	this.deadline = time.AfterFunc(until, func () {
+		this.closeWithError(os.ErrDeadlineExceeded)
+	})
+	return nil
+}
+
+// TODO
+// func (this *transA) SetReadDeadline(t time.Time) error {
+// 	// TODO
+// }
+// 
+// func (this *transA) SetWriteDeadline(t time.Time) error {
+// 	// TODO
+// }
+
+func (this *transA) errIfClosed() error {
+	if !this.closed.Load() {
+		return nil
+	}
+	return this.bestErr()
+}
+
+func (this *transA) bestErr() error {
+	if this.parent.err != nil {
+		return this.parent.err
+	}
+	if this.closeErr != nil {
+		return this.closeErr
+	}
+	return io.EOF
+}
+
 type readerA struct {
 	parent   *transA
 	leftover []byte
@@ -319,11 +382,7 @@ func (this *readerA) pull() (uint16, error) {
 	// close and return error on failure
 	this.eof = true
 	this.parent.Close()
-	if this.parent.parent.err == nil {
-		return 0, fmt.Errorf("could not receive message: %w", io.EOF)
-	} else {
-		return 0, this.parent.parent.err
-	}
+	return 0, fmt.Errorf("could not receive message: %w", this.parent.bestErr())
 }
 
 func (this *readerA) Read(buffer []byte) (int, error) {

metadaptb.go

@@ -2,6 +2,7 @@ package hopp
 
 import "io"
 import "net"
+import "time"
 import "bytes"
 import "errors"
 import "context"
@@ -50,6 +51,10 @@ func (this *b) SetSizeLimit(limit int64) {
 	this.sizeLimit = limit
 }
 
+func (this *b) SetDeadline(t time.Time) error {
+	return this.underlying.SetDeadline(t)
+}
+
 func (this *b) newTrans(underlying Stream) *transB {
 	return &transB {
 		sizeLimit: this.sizeLimit,
@@ -124,6 +129,10 @@ func (this *transB) receiveReader() (uint16, int64, io.Reader, error) {
 	return method, size, data, nil
 }
 
+func (this *transB) SetDeadline(t time.Time) error {
+	return this.underlying.SetDeadline(t)
+}
+
 type writerB struct {
 	parent *transB
 	buffer bytes.Buffer
@@ -149,12 +158,16 @@ type MultiConn interface {
 	AcceptStream(context.Context) (Stream, error)
 	// OpenStream opens a new stream.
 	OpenStream() (Stream, error)
+	// See the documentation for [net.Conn.SetDeadline].
+	SetDeadline(time.Time) error
 }
 
 // Stream represents a single stream returned by a [MultiConn].
 type Stream interface {
 	// See documentation for [net.Conn].
 	io.ReadWriteCloser
+	// See the documentation for [net.Conn.SetDeadline].
+	SetDeadline(time.Time) error
 	// ID returns the stream ID
 	ID() int64
 }

tape/dynamic.go

@@ -8,6 +8,17 @@ package tape
 
 // TODO: test all of these smaller functions individually
 
+// For an explanation as to why this package always treats LBA/SBA as strings,
+// refer to https://go.dev/blog/strings:
+// 
+//   It’s important to state right up front that a string holds arbitrary
+//   bytes. It is not required to hold Unicode text, UTF-8 text, or any other
+//   predefined format. As far as the content of a string is concerned, it is
+//   exactly equivalent to a slice of bytes.
+//
+// Arbitrary byte slices and blobs won't be as common of a use case as text
+// data, and if you need that anyway you can just cast it to a byte slice.
+
 import "fmt"
 import "reflect"
 
@@ -36,19 +47,28 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	// primitives
 	reflectValue := reflect.ValueOf(value)
 	switch reflectValue.Kind() {
-	case reflect.Int:    return encoder.WriteInt32(int32(reflectValue.Int()))
-	case reflect.Uint:   return encoder.WriteUint32(uint32(reflectValue.Uint()))
-	case reflect.Int8:   return encoder.WriteInt8(int8(reflectValue.Int()))
-	case reflect.Uint8:  return encoder.WriteUint8(uint8(reflectValue.Uint()))
-	case reflect.Int16:  return encoder.WriteInt16(int16(reflectValue.Int()))
-	case reflect.Uint16: return encoder.WriteUint16(uint16(reflectValue.Uint()))
-	case reflect.Int32:  return encoder.WriteInt32(int32(reflectValue.Int()))
-	case reflect.Uint32: return encoder.WriteUint32(uint32(reflectValue.Uint()))
-	case reflect.Int64:  return encoder.WriteInt64(int64(reflectValue.Int()))
-	case reflect.Uint64: return encoder.WriteUint64(uint64(reflectValue.Uint()))
-	case reflect.String: return EncodeAny(encoder, []byte(reflectValue.String()), tag)
+	case reflect.Int:     return encoder.WriteInt32(int32(reflectValue.Int()))
+	case reflect.Uint:    return encoder.WriteUint32(uint32(reflectValue.Uint()))
+	case reflect.Int8:    return encoder.WriteInt8(int8(reflectValue.Int()))
+	case reflect.Uint8:   return encoder.WriteUint8(uint8(reflectValue.Uint()))
+	case reflect.Int16:   return encoder.WriteInt16(int16(reflectValue.Int()))
+	case reflect.Uint16:  return encoder.WriteUint16(uint16(reflectValue.Uint()))
+	case reflect.Int32:   return encoder.WriteInt32(int32(reflectValue.Int()))
+	case reflect.Uint32:  return encoder.WriteUint32(uint32(reflectValue.Uint()))
+	case reflect.Int64:   return encoder.WriteInt64(int64(reflectValue.Int()))
+	case reflect.Uint64:  return encoder.WriteUint64(uint64(reflectValue.Uint()))
+	case reflect.Float32: return encoder.WriteFloat32(float32(reflectValue.Float()))
+	case reflect.Float64: return encoder.WriteFloat64(float64(reflectValue.Float()))
+	case reflect.String:
+		if reflectValue.Len() > MaxStructureLength {
+			return 0, ErrTooLong
+		}
+		return EncodeAny(encoder, []byte(reflectValue.String()), tag)
 	}
 	if reflectValue.CanConvert(reflect.TypeOf(dummyBuffer)) {
+		if reflectValue.Len() > MaxStructureLength {
+			return 0, ErrTooLong
+		}
 		if tag.Is(LBA) {
 			nn, err := encoder.WriteUintN(uint64(reflectValue.Len()), tag.CN() + 1)
 			n += nn; if err != nil { return n, err }
@@ -64,8 +84,13 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	case reflect.Slice:
 		return encodeAnySlice(encoder, value, tag)
 	// case reflect.Array:
+		// TODO: we can encode arrays. but can we decode into them?
+		// that's the fucken question. maybe we just do the first
 		// return encodeAnySlice(encoder, reflect.ValueOf(value).Slice(0, reflectType.Len()).Interface(), tag)
 	case reflect.Map:
+		if reflectValue.Len() > MaxStructureLength {
+			return 0, ErrTooLong
+		}
 		if reflectType.Key() == reflect.TypeOf(uint16(0)) {
 			return encodeAnyMap(encoder, value, tag)
 		}
@@ -74,9 +99,10 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	return n, fmt.Errorf("cannot encode type %T", value)
 }
 
-// DecodeAny decodes data and places it into destination, which must be a
+// DecodeAnyInto decodes data and places it into destination, which must be a
 // pointer to a supported type. See [EncodeAny] for a list of supported types.
-func DecodeAny(decoder *Decoder, destination any, tag Tag) (n int, err error) {
+// The head of the decoder must be at the start of the payload.
+func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error) {
 	reflectDestination := reflect.ValueOf(destination)
 	if reflectDestination.Kind() != reflect.Pointer {
 		return n, fmt.Errorf("expected pointer destination, not %v", destination)
@@ -84,6 +110,17 @@ func DecodeAny(decoder *Decoder, destination any, tag Tag) (n int, err error) {
 	return decodeAny(decoder, reflectDestination.Elem(), tag)
 }
 
+// DecodeAny is like [DecodeAnyInto], but it automatically creates the
+// destination from the tag and data. The head of the decoder must be at the
+// start of the payload.
+func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) {
+	destination, err := skeletonPointer(decoder, tag)
+	if err != nil { return nil, n, err }
+	nn, err := DecodeAnyInto(decoder, destination, tag)
+	n += nn; if err != nil { return nil, n, err }
+	return destination, n, err
+}
+
 // unknownSlicePlaceholder is inserted by skeletonValue and informs the program
 // that the destination for the slice needs to be generated based on the item
 // tag in the OTA.
@@ -93,7 +130,8 @@ var unknownSlicePlaceholderType = reflect.TypeOf(unknownSlicePlaceholder { })
 // decodeAny is internal to [DecodeAny]. It takes in an addressable
 // [reflect.Value] as the destination. If the decoded value cannot fit in the
 // destination, it skims over the payload, leaves the destination empty, and
-// returns without an error.
+// returns without an error. The head of the decoder must be at the start of the
+// payload.
 func decodeAny(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err error) {
 	n, err = decodeAnyOrError(decoder, destination, tag)
 	if _, ok := err.(errCantAssign); ok {
@@ -110,7 +148,7 @@ func decodeAny(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err
 // destination, it decodes nothing and returns an error of type errCantAssign,
 // except for the case of a mismatched OTA element tag, wherein it will skim
 // over the rest of the payload, leave the destination empty, and return without
-// an error.
+// an error. The head of the decoder must be at the start of the payload.
 func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err error) {
 	err = canSet(destination.Type(), tag)
 	if err != nil { return n, err }
@@ -118,7 +156,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 	switch tag.WithoutCN() {
 	case SI:
 		// SI: (none)
-		setInt(destination, uint64(tag.CN()))
+		setUint(destination, uint64(tag.CN()), 1)
 	case LI:
 		// LI: <value: IntN>
 		nn, err := decodeAndSetUint(decoder, destination, tag.CN() + 1)
@@ -133,26 +171,45 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 		n += nn; if err != nil { return n, err }
 	case SBA:
 		// SBA: <data: U8>*
-		buffer := make([]byte, tag.CN())
+		length := tag.CN()
+		if length > MaxStructureLength {
+			return 0, ErrTooLong
+		}
+		buffer := make([]byte, length)
 		nn, err := decoder.Read(buffer)
 		n += nn; if err != nil { return n, err }
-		setByteArray(destination, buffer)
+		setString(destination, string(buffer))
 	case LBA:
 		// LBA: <length: UN> <data: U8>*
 		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
 		n += nn; if err != nil { return n, err }
+		if length > uint64(MaxStructureLength) {
+			return 0, ErrTooLong
+		}
 		buffer := make([]byte, length)
 		nn, err = decoder.Read(buffer)
 		n += nn; if err != nil { return n, err }
-		setByteArray(destination, buffer)
+		setString(destination, string(buffer))
 	case OTA:
 		// OTA: <length: UN> <elementTag: tape.Tag> <values>*
+		oldDestination := destination
+		if isTypeAny(destination.Type()) {
+			// need a skeleton value if we are assigning to any.
+			value, err := skeletonValue(decoder, tag)
+			if err != nil { return n, err }
+			destination = value
+		}
 		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
 		n += nn; if err != nil { return n, err }
+		if length > uint64(MaxStructureLength) {
+			return 0, ErrTooLong
+		}
+		lengthCast, err := Uint64ToIntSafe(length)
+		if err != nil { return n, err }
 		oneTag, nn, err := decoder.ReadTag()
 		n += nn; if err != nil { return n, err }
-		if destination.Cap() < int(length) {
-			destination.Grow(int(length) - destination.Cap())
+		if destination.Cap() < lengthCast {
+			destination.Grow(lengthCast - destination.Cap())
 		}
 		// skip the rest of the array if the one tag doesn't
 		// match up with the destination
@@ -165,7 +222,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 			break
 		}
 		if err != nil { return n, err }
-		destination.SetLen(int(length))
+		destination.SetLen(lengthCast)
 		for index := range length {
 			nn, err := decodeAny(decoder, destination.Index(int(index)), oneTag)
 			n += nn
@@ -175,17 +232,29 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 				return n, err
 			}
 		}
+		oldDestination.Set(destination)
 	case KTV:
 		// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
 		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
 		n += nn; if err != nil { return n, err }
+		if length > uint64(MaxStructureLength) {
+			return 0, ErrTooLong
+		}
+		lengthCast, err := Uint64ToIntSafe(length)
+		if err != nil { return n, err }
+		if isTypeAny(destination.Type()) {
+			// need a skeleton value if we are assigning to any.
+			value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast)
+			destination.Set(value)
+			destination = value
+		}
 		destination.Clear()
-		for _ = range length {
+		for _ = range lengthCast {
 			key, nn, err := decoder.ReadUint16()
 			n += nn; if err != nil { return n, err }
 			itemTag, nn, err := decoder.ReadTag()
 			n += nn; if err != nil { return n, err }
-			value, err := skeletonValue(decoder, itemTag)
+			value, err := skeletonPointer(decoder, itemTag)
 			if err != nil { return n, err }
 			nn, err = decodeAny(decoder, value.Elem(), itemTag)
 			n += nn; if err != nil { return n, err }
@@ -207,17 +276,19 @@ func TagAny(value any) (Tag, error) {
 func tagAny(reflectValue reflect.Value) (Tag, error) {
 	// primitives
 	switch reflectValue.Kind() {
-	case reflect.Int:    return LSI.WithCN(3), nil
-	case reflect.Int8:   return LSI.WithCN(0), nil
-	case reflect.Int16:  return LSI.WithCN(1), nil
-	case reflect.Int32:  return LSI.WithCN(3), nil
-	case reflect.Int64:  return LSI.WithCN(7), nil
-	case reflect.Uint:   return LI.WithCN(3), nil
-	case reflect.Uint8:  return LI.WithCN(0), nil
-	case reflect.Uint16: return LI.WithCN(1), nil
-	case reflect.Uint32: return LI.WithCN(3), nil
-	case reflect.Uint64: return LI.WithCN(7), nil
-	case reflect.String: return bufferLenTag(reflectValue.Len()), nil
+	case reflect.Int:     return LSI.WithCN(3), nil
+	case reflect.Int8:    return LSI.WithCN(0), nil
+	case reflect.Int16:   return LSI.WithCN(1), nil
+	case reflect.Int32:   return LSI.WithCN(3), nil
+	case reflect.Int64:   return LSI.WithCN(7), nil
+	case reflect.Uint:    return LI.WithCN(3), nil
+	case reflect.Uint8:   return LI.WithCN(0), nil
+	case reflect.Uint16:  return LI.WithCN(1), nil
+	case reflect.Uint32:  return LI.WithCN(3), nil
+	case reflect.Uint64:  return LI.WithCN(7), nil
+	case reflect.Float32: return FP.WithCN(3), nil
+	case reflect.Float64: return FP.WithCN(7), nil
+	case reflect.String:  return bufferLenTag(reflectValue.Len()), nil
 	}
 	if reflectValue.CanConvert(reflect.TypeOf(dummyBuffer)) {
 		return bufferLenTag(reflectValue.Len()), nil
@@ -284,6 +355,10 @@ func encodeAnyMap(encoder *Encoder, value any, tag Tag) (n int, err error) {
 }
 
 func canSet(destination reflect.Type, tag Tag) error {
+	// anything can be assigned to `any`
+	if isTypeAny(destination) {
+		return nil
+	}
 	switch tag.WithoutCN() {
 	case SI, LI, LSI:
 		switch destination.Kind() {
@@ -300,6 +375,7 @@ func canSet(destination reflect.Type, tag Tag) error {
 			return errCantAssignf("cannot assign float to %v", destination)
 		}
 	case SBA, LBA:
+		if destination.Kind() == reflect.String { return nil }
 		if destination.Kind() != reflect.Slice {
 			return errCantAssignf("cannot assign byte array to %v", destination)
 		}
@@ -321,17 +397,43 @@ func canSet(destination reflect.Type, tag Tag) error {
 }
 
 // setInt expects a settable destination.
-func setInt[T int64 | uint64](destination reflect.Value, value T) {
+func setInt(destination reflect.Value, value int64, bytes int) {
 	switch {
 	case destination.CanInt():
 		destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type()))
 	case destination.CanUint():
 		destination.Set(reflect.ValueOf(value).Convert(destination.Type()))
+	case isTypeAny(destination.Type()):
+		switch {
+		case bytes > 4: destination.Set(reflect.ValueOf(int64(value)))
+		case bytes > 2: destination.Set(reflect.ValueOf(int32(value)))
+		case bytes > 1: destination.Set(reflect.ValueOf(int16(value)))
+		default:        destination.Set(reflect.ValueOf(int8(value)))
+		}
 	default:
 		panic("setInt called on an unsupported type")
 	}
 }
 
+// setUint expects a settable destination.
+func setUint(destination reflect.Value, value uint64, bytes int) {
+	switch {
+	case destination.CanInt():
+		destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type()))
+	case destination.CanUint():
+		destination.Set(reflect.ValueOf(value).Convert(destination.Type()))
+	case isTypeAny(destination.Type()):
+		switch {
+		case bytes > 4: destination.Set(reflect.ValueOf(uint64(value)))
+		case bytes > 2: destination.Set(reflect.ValueOf(uint32(value)))
+		case bytes > 1: destination.Set(reflect.ValueOf(uint16(value)))
+		default:        destination.Set(reflect.ValueOf(uint8(value)))
+		}
+	default:
+		panic("setUint called on an unsupported type")
+	}
+}
+
 // setFloat expects a settable destination.
 func setFloat(destination reflect.Value, value float64) {
 	destination.Set(reflect.ValueOf(value).Convert(destination.Type()))
@@ -342,11 +444,16 @@ func setByteArray(destination reflect.Value, value []byte) {
 	destination.Set(reflect.ValueOf(value))
 }
 
+// setString exepctes a settable destination
+func setString(destination reflect.Value, value string) {
+	destination.Set(reflect.ValueOf(value))
+}
+
 // decodeAndSetInt expects a settable destination.
 func decodeAndSetInt(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) {
 	value, nn, err := decoder.ReadIntN(bytes)
 	n += nn; if err != nil { return n, err }
-	setInt(destination, value)
+	setInt(destination, value, bytes)
 	return n, nil
 }
 
@@ -354,7 +461,7 @@ func decodeAndSetInt(decoder *Decoder, destination reflect.Value, bytes int) (n
 func decodeAndSetUint(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) {
 	value, nn, err := decoder.ReadUintN(bytes)
 	n += nn; if err != nil { return n, err }
-	setInt(destination, value)
+	setUint(destination, value, bytes)
 	return n, nil
 }
 
@@ -375,16 +482,26 @@ func decodeAndSetFloat(decoder *Decoder, destination reflect.Value, bytes int) (
 	return n, errCantAssignf("unsupported bit width float%d", bytes * 8)
 }
 
-// skeletonValue returns a pointer value. In order for it to be set, it must be
-// dereferenced using Elem().
+// skeletonValue returns an addressable value. It can be set directly. The head
+// of the decoder must be at the start of the payload when calling.
 func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) {
+	ptr, err := skeletonPointer(decoder, tag)
+	if err != nil { return reflect.Value { }, err }
+	return ptr.Elem(), nil
+}
+
+// skeletonPointer returns a pointer value. In order for it to be set, it must
+// be dereferenced using Elem(). The head of the decoder must be at the start of
+// the payload when calling.
+func skeletonPointer(decoder *Decoder, tag Tag) (reflect.Value, error) {
 	typ, err := typeOf(decoder, tag)
 	if err != nil { return reflect.Value { }, err }
 	return reflect.New(typ), nil
 }
 
 // typeOf returns the type of the current tag being decoded. It does not use up
-// the decoder, it only peeks.
+// the decoder, it only peeks. The head of the decoder must be at the start of
+// the payload when calling.
 func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) {
 	switch tag.WithoutCN() {
 	case SI:
@@ -411,8 +528,8 @@ func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) {
 		case 7: return reflect.TypeOf(float64(0)), nil
 		}
 		return nil, fmt.Errorf("unknown CN %d for FP", tag.CN())
-	case SBA: return reflect.SliceOf(reflect.TypeOf(byte(0))), nil
-	case LBA: return reflect.SliceOf(reflect.TypeOf(byte(0))), nil
+	case SBA: return reflect.TypeOf(""), nil
+	case LBA: return reflect.TypeOf(""), nil
 	case OTA:
 		elemTag, dimension, err := peekSlice(decoder, tag)
 		if err != nil { return nil, err }
@@ -428,6 +545,12 @@ func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) {
 	return nil, fmt.Errorf("unknown TN %d", tag.TN())
 }
 
+// isTypeAny returns whether the given reflect.Type is an interface with no
+// methods.
+func isTypeAny(typ reflect.Type) bool {
+	return typ.Kind() == reflect.Interface && typ.NumMethod() == 0
+}
+
 // peekSlice returns the element tag and dimension count of the OTA currently
 // being decoded. It does not use up the decoder, it only peeks.
 func peekSlice(decoder *Decoder, tag Tag) (Tag, int, error) {

tape/dynamic_test.go

@@ -1,11 +1,56 @@
 package tape
 
-import "fmt"
 import "bytes"
 import "testing"
 import "reflect"
 import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
 
+var samplePayloads = [][]byte {
+	/* int8     */ []byte { byte(LSI.WithCN(0)), 0x45 },
+	/* int16    */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 },
+	/* int32    */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
+	/* int64    */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
+	/* uint5    */ []byte { byte(SI.WithCN(12)) },
+	/* uint8    */ []byte { byte(LI.WithCN(0)), 0x45 },
+	/* uint16   */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 },
+	/* uint32   */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
+	/* uint64   */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
+	/* string   */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' },
+	/* []byte   */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' },
+	/* []string */ []byte {
+		byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)),
+		0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
+		0x05, 0x11, 0x11, 0x11, 0x11, 0x11,
+	},
+	/* map[uint16] any */ []byte {
+		byte(KTV.WithCN(0)), 2,
+		0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67,
+		0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB,
+	},
+}
+
+var sampleValues = []any {
+	/* int8     */ int8(0x45),
+	/* int16    */ int16(0x4567),
+	/* int32    */ int32(0x456789AB),
+	/* int64    */ int64(0x456789ABCDEF0123),
+	/* uint5    */ uint8(12),
+	/* uint8    */ uint8(0x45),
+	/* uint16   */ uint16(0x4567),
+	/* uint32   */ uint32(0x456789AB),
+	/* uint64   */ uint64(0x456789ABCDEF0123),
+	/* string   */ "pupever",
+	/* []byte   */ "blarg",
+	/* []string */ []string {
+		"\x45\x67\x89\xAB\xCD\xEF\x01\x23",
+		"\x11\x11\x11\x11\x11",
+	},
+	/* map[uint16] any */ map[uint16] any {
+		0x0223: int16(0x4567),
+		0x0224: uint32(0x456789AB),
+	},
+}
+
 type userDefinedInteger int16
 
 func TestEncodeAnyInt(test *testing.T) {
@@ -26,7 +71,9 @@ func TestEncodeAnyTable(test *testing.T) {
 		0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} },
 		0x2345: [][]int16 { []int16 { 0x5 }, []int16 { 0x17, -0xAAA } },
 		0x3456: userDefinedInteger(0x3921),
-	}, KTV.WithCN(0), tu.S(7).AddVar(
+		0x1F1F: float32(67.26),
+		0x0F0F: float64(5.3),
+	}, KTV.WithCN(0), tu.S(9).AddVar(
 		[]byte {
 			0xF3, 0xB9,
 			byte(LSI.WithCN(3)),
@@ -70,41 +117,27 @@ func TestEncodeAnyTable(test *testing.T) {
 			byte(LSI.WithCN(1)),
 			0x39, 0x21,
 		},
+		[]byte {
+			0x1F, 0x1F,
+			byte(FP.WithCN(3)),
+			0x42, 0x86, 0x85, 0x1F,
+		},
+		[]byte {
+			0x0F, 0x0F,
+			byte(FP.WithCN(7)),
+			0x40, 0x15, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33,
+		},
 	))
 	if err != nil { test.Fatal(err) }
 }
 
 func TestDecodeWrongType(test *testing.T) {
-	datas := [][]byte {
-		/* int8     */ []byte { byte(LSI.WithCN(0)), 0x45 },
-		/* int16    */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 },
-		/* int32    */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-		/* int64    */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-		/* uint5    */ []byte { byte(SI.WithCN(12)) },
-		/* uint8    */ []byte { byte(LI.WithCN(0)), 0x45 },
-		/* uint16   */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 },
-		/* uint32   */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB },
-		/* uint64   */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 },
-		/* string   */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' },
-		/* []byte   */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' },
-		/* []string */ []byte {
-			byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)),
-			0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23,
-			0x05, 0x11, 0x11, 0x11, 0x11, 0x11,
-		},
-		/* map[uint16] any */ []byte {
-			byte(KTV.WithCN(0)), 2,
-			0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67,
-			0x02, 0x23, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB,
-		},
-	}
-
-	for index, data := range datas {
+	for index, data := range samplePayloads {
 		test.Logf("data %2d %v [%s]", index, Tag(data[0]), tu.HexBytes(data[1:]))
 		// integers should only assign to other integers
 		if index > 8 {
 			cas := func(destination any) {
-				n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
+				n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
 				if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
 				reflectValue := reflect.ValueOf(destination).Elem()
 				if reflectValue.CanInt() {
@@ -138,7 +171,7 @@ func TestDecodeWrongType(test *testing.T) {
 			{ var dest uint64; cas(&dest) }
 		}
 		arrayCase := func(destination any) {
-			n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
+			n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0]))
 			if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
 			reflectDestination := reflect.ValueOf(destination)
 			reflectValue := reflectDestination.Elem()
@@ -175,13 +208,36 @@ func TestEncodeDecodeAnyTable(test *testing.T) {
 		0x0102: uint32(2),
 		0x0103: int64(23432),
 		0x0104: int64(-88777),
-		0x0000: []byte("hi!"),
+		0x0000: "hi!",
 		0xFFFF: []uint16 { 0xBEE5, 0x7777 },
 		0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} },
+		0x1F1F: float32(67.26),
+		0x0F0F: float64(5.3),
 	}, nil)
 	if err != nil { test.Fatal(err) }
 }
 
+func TestEncodeDecodeAnyDestination(test *testing.T) {
+	var destination any
+	for index, data := range samplePayloads {
+		tag     := Tag(data[0])
+		payload := data[1:]
+		test.Logf("data %2d %v [%s]", index, tag, tu.HexBytes(payload))
+		n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(payload)), &destination, tag)
+		if err != nil { test.Fatalf("error: %v | n: %d", err, n) }
+		got     := destination
+		correct := sampleValues[index]
+		test.Log("got:    ", tu.Describe(got))
+		test.Log("correct:", tu.Describe(correct))
+		if !reflect.DeepEqual(got, correct) {
+			test.Fatalf("values not equal")
+		}
+		if n != len(payload) {
+			test.Fatalf("n not equal: %d != %d", n, len(payload))
+		}
+	}
+}
+
 func TestPeekSlice(test *testing.T) {
 	buffer := bytes.NewBuffer([]byte {
 		2, byte(OTA.WithCN(3)),
@@ -240,71 +296,3 @@ func TestPeekSliceOnce(test *testing.T) {
 		test.Fatalf("wrong n: %d != %d", got, correct)
 	}
 }
-
-func encAny(value any) ([]byte, Tag, int, error) {
-	tag, err := TagAny(value)
-	if err != nil { return nil, 0, 0, err }
-	buffer := bytes.Buffer { }
-	encoder := NewEncoder(&buffer)
-	n, err := EncodeAny(encoder, value, tag)
-	if err != nil { return nil, 0, n, err }
-	encoder.Flush()
-	return buffer.Bytes(), tag, n, nil
-}
-
-func decAny(data []byte) (Tag, any, int, error) {
-	destination := map[uint16] any { }
-	tag, err := TagAny(destination)
-	if err != nil { return 0, nil, 0, err }
-	n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data)), &destination, tag)
-	if err != nil { return 0, nil, n, err }
-	return tag, destination, n, nil
-}
-
-func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.Snake) error {
-	bytes, tag, n, err := encAny(value)
-	if err != nil { return err }
-	test.Log("n:      ", n)
-	test.Log("tag:    ", tag)
-	test.Log("got:    ", tu.HexBytes(bytes))
-	test.Log("correct:", correctBytes)
-	if tag != correctTag {
-		return fmt.Errorf("tag not equal: %v != %v", tag, correctTag)
-	}
-	if ok, n := correctBytes.Check(bytes); !ok {
-		return fmt.Errorf("bytes not equal at index %d", n)
-	}
-	if n != len(bytes) {
-		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
-	}
-	return nil
-}
-
-func testEncodeDecodeAny(test *testing.T, value, correctValue any) error {
-	if correctValue == nil { 
-		correctValue = value
-	}
-
-	test.Log("encoding...")
-	bytes, tag, n, err := encAny(value)
-	if err != nil { return err }
-	test.Log("n:  ", n)
-	test.Log("tag:", tag)
-	test.Log("got:", tu.HexBytes(bytes))
-	test.Log("decoding...", tag)
-	if n != len(bytes) {
-		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
-	}
-	
-	_, decoded, n, err := decAny(bytes)
-	if err != nil { return err }
-	test.Log("got:    ", tu.Describe(decoded))
-	test.Log("correct:", tu.Describe(correctValue))
-	if !reflect.DeepEqual(decoded, correctValue) {
-		return fmt.Errorf("values not equal")
-	}
-	if n != len(bytes) {
-		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
-	}
-	return nil
-}

tape/error.go

@@ -0,0 +1,12 @@
+package tape
+
+// Error enumerates common errors in this package.
+type Error string; const (
+	ErrTooLong  Error = "data structure too long"
+	ErrTooLarge Error = "number too large"
+)
+
+// Error implements the error interface.
+func (err Error) Error() string {
+	return string(err)
+}

tape/limits.go

@@ -0,0 +1,26 @@
+package tape
+
+// MaxStructureLength determines how long a TAPE data structure can be. This
+// applies to:
+//
+//  - OTA
+//  - SBA/LBA
+//  - KTV
+//
+// By default it is set at 2^20 (about a million).
+// You shouldn't need to change this. If you do, it should only be set once at
+// the start of the program.
+var MaxStructureLength = 1024 * 1024
+
+// MaxInt is the maximum value an int can hold. This varies depending on the
+// system.
+const MaxInt int = int(^uint(0) >> 1)
+
+// Uint64ToIntSafe casts the input to an int if it can be done without overflow,
+// or returns an error otherwise.
+func Uint64ToIntSafe(input uint64) (int, error) {
+	if input > uint64(MaxInt) {
+		return 0, ErrTooLarge
+	}
+	return int(input), nil
+}

tape/misc_test.go

@@ -0,0 +1,75 @@
+package tape
+
+import "fmt"
+import "bytes"
+import "testing"
+import "reflect"
+import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
+
+func encAny(value any) ([]byte, Tag, int, error) {
+	tag, err := TagAny(value)
+	if err != nil { return nil, 0, 0, err }
+	buffer := bytes.Buffer { }
+	encoder := NewEncoder(&buffer)
+	n, err := EncodeAny(encoder, value, tag)
+	if err != nil { return nil, 0, n, err }
+	encoder.Flush()
+	return buffer.Bytes(), tag, n, nil
+}
+
+func decAny(data []byte) (Tag, any, int, error) {
+	destination := map[uint16] any { }
+	tag, err := TagAny(destination)
+	if err != nil { return 0, nil, 0, err }
+	n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data)), &destination, tag)
+	if err != nil { return 0, nil, n, err }
+	return tag, destination, n, nil
+}
+
+func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.Snake) error {
+	bytes, tag, n, err := encAny(value)
+	if err != nil { return err }
+	test.Log("n:      ", n)
+	test.Log("tag:    ", tag)
+	test.Log("got:    ", tu.HexBytes(bytes))
+	test.Log("correct:", correctBytes)
+	if tag != correctTag {
+		return fmt.Errorf("tag not equal: %v != %v", tag, correctTag)
+	}
+	if ok, n := correctBytes.Check(bytes); !ok {
+		return fmt.Errorf("bytes not equal at index %d", n)
+	}
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	return nil
+}
+
+func testEncodeDecodeAny(test *testing.T, value, correctValue any) error {
+	if correctValue == nil { 
+		correctValue = value
+	}
+
+	test.Log("encoding...")
+	bytes, tag, n, err := encAny(value)
+	if err != nil { return err }
+	test.Log("n:  ", n)
+	test.Log("tag:", tag)
+	test.Log("got:", tu.HexBytes(bytes))
+	test.Log("decoding...", tag)
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	
+	_, decoded, n, err := decAny(bytes)
+	if err != nil { return err }
+	test.Log("got:    ", tu.Describe(decoded))
+	test.Log("correct:", tu.Describe(correctValue))
+	if !reflect.DeepEqual(decoded, correctValue) {
+		return fmt.Errorf("values not equal")
+	}
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	return nil
+}