generate: Generator compiles

This is a stupid fucking restriction

.gitignore

@@ -0,0 +1 @@
+/generate/test

codec/decode.go

@@ -1,108 +0,0 @@
-package codec
-
-import "io"
-
-// Decodable is any type that can decode itself from a decoder.
-type Decodable interface {
-	// Decode reads data from decoder, replacing the data of the object. It
-	// returns the amount of bytes written, and an error if the write
-	// stopped early.
-	Decode(decoder *Decoder) (n int, err error)
-}
-
-// Decoder wraps an [io.Reader] and decodes data from it.
-type Decoder struct {
-	io.Reader
-}
-
-// ReadFull calls [io.ReadFull] on the reader.
-func (this *Decoder) ReadFull(buffer []byte) (n int, err error) {
-	return io.ReadFull(this, buffer)
-}
-
-// ReadByte decodes a single byte from the input reader.
-func (this *Decoder) ReadByte() (value byte, n int, err error) {
-	uncasted, n, err := this.ReadUint8()
-	return byte(uncasted), n, err
-}
-
-// ReadInt8 decodes an 8-bit signed integer from the input reader.
-func (this *Decoder) ReadInt8() (value int8, n int, err error) {
-	uncasted, n, err := this.ReadUint8()
-	return int8(uncasted), n, err
-}
-
-// ReadUint8 decodes an 8-bit unsigned integer from the input reader.
-func (this *Decoder) ReadUint8() (value uint8, n int, err error) {
-	buffer := [1]byte { }
-	n, err = this.ReadFull(buffer[:])
-	return uint8(buffer[0]), n, err
-}
-
-// ReadInt16 decodes an 16-bit signed integer from the input reader.
-func (this *Decoder) ReadInt16() (value int16, n int, err error) {
-	uncasted, n, err := this.ReadUint16()
-	return int16(uncasted), n, err
-}
-
-// ReadUint16 decodes an 16-bit unsigned integer from the input reader.
-func (this *Decoder) ReadUint16() (value uint16, n int, err error) {
-	buffer := [2]byte { }
-	n, err = this.ReadFull(buffer[:])
-	return uint16(buffer[0]) << 8 |
-		uint16(buffer[1]), n, err
-}
-
-// ReadInt32 decodes an 32-bit signed integer from the input reader.
-func (this *Decoder) ReadInt32() (value int32, n int, err error) {
-	uncasted, n, err := this.ReadUint32()
-	return int32(uncasted), n, err
-}
-
-// ReadUint32 decodes an 32-bit unsigned integer from the input reader.
-func (this *Decoder) ReadUint32() (value uint32, n int, err error) {
-	buffer := [4]byte { }
-	n, err = this.ReadFull(buffer[:])
-	return uint32(buffer[0]) << 24 |
-		uint32(buffer[1]) << 16 |
-		uint32(buffer[2]) << 8 |
-		uint32(buffer[3]), n, err
-}
-
-// ReadInt64 decodes an 64-bit signed integer from the input reader.
-func (this *Decoder) ReadInt64() (value int64, n int, err error) {
-	uncasted, n, err := this.ReadUint64()
-	return int64(uncasted), n, err
-}
-
-// ReadUint64 decodes an 64-bit unsigned integer from the input reader.
-func (this *Decoder) ReadUint64() (value uint64, n int, err error) {
-	buffer := [8]byte { }
-	n, err = this.ReadFull(buffer[:])
-	return uint64(buffer[0]) << 56 |
-		uint64(buffer[1]) << 48 |
-		uint64(buffer[2]) << 48 |
-		uint64(buffer[3]) << 32 |
-		uint64(buffer[4]) << 24 |
-		uint64(buffer[5]) << 16 |
-		uint64(buffer[6]) << 8 |
-		uint64(buffer[7]), n, err
-}
-
-// ReadGBEU decodes a growing unsigned integer of up to 64 bits from the input
-// reader.
-func (this *Decoder) ReadGBEU() (value uint64, n int, err error) {
-	var fullValue uint64
-	for {
-		chunk, nn, err := this.ReadByte()
-		if err != nil { return 0, n, err }
-		n += nn
-		
-		fullValue *= 0x80
-		fullValue += uint64(chunk & 0x7F)
-		ccb := chunk >> 7
-		if ccb == 0 {
-			return fullValue, n, nil
-		}
-	}
-}

codec/encode.go

@@ -1,102 +0,0 @@
-package codec
-
-import "io"
-
-// Encodable is any type that can write itself to an encoder.
-type Encodable interface {
-	// Encode sends data to encoder. It returns the amount of bytes written,
-	// and an error if the write stopped early.
-	Encode(encoder *Encoder) (n int, err error)
-}
-	
-// Encoder wraps an [io.Writer] and encodes data to it.
-type Encoder struct {
-	io.Writer
-}
-
-// WriteByte encodes a single byte to the output writer.
-func (this *Encoder) WriteByte(value byte) (n int, err error) {
-	return this.WriteByte(uint8(value))
-}
-
-// WriteInt8 encodes an 8-bit signed integer to the output writer.
-func (this *Encoder) WriteInt8(value int8) (n int, err error) {
-	return this.WriteUint8(uint8(value))
-}
-
-// WriteUint8 encodes an 8-bit unsigned integer to the output writer.
-func (this *Encoder) WriteUint8(value uint8) (n int, err error) {
-	return this.Write([]byte { byte(value) })
-}
-
-// WriteInt16 encodes an 16-bit signed integer to the output writer.
-func (this *Encoder) WriteInt16(value int16) (n int, err error) {
-	return this.WriteUint16(uint16(value))
-}
-
-// WriteUint16 encodes an 16-bit unsigned integer to the output writer.
-func (this *Encoder) WriteUint16(value uint16) (n int, err error) {
-	return this.Write([]byte {
-		byte(value >> 8),
-		byte(value),
-	})
-}
-
-// WriteInt32 encodes an 32-bit signed integer to the output writer.
-func (this *Encoder) WriteInt32(value int32) (n int, err error) {
-	return this.WriteUint32(uint32(value))
-}
-
-// WriteUint32 encodes an 32-bit unsigned integer to the output writer.
-func (this *Encoder) WriteUint32(value uint32) (n int, err error) {
-	return this.Write([]byte {
-		byte(value >> 24),
-		byte(value >> 16),
-		byte(value >> 8),
-		byte(value),
-	})
-}
-
-// WriteInt64 encodes an 64-bit signed integer to the output writer.
-func (this *Encoder) WriteInt64(value int64) (n int, err error) {
-	return this.WriteUint64(uint64(value))
-}
-
-// WriteUint64 encodes an 64-bit unsigned integer to the output writer.
-func (this *Encoder) WriteUint64(value uint64) (n int, err error) {
-	return this.Write([]byte {
-		byte(value >> 56),
-		byte(value >> 48),
-		byte(value >> 40),
-		byte(value >> 32),
-		byte(value >> 24),
-		byte(value >> 16),
-		byte(value >> 8),
-		byte(value),
-	})
-}
-
-// EncodeGBEU encodes a growing unsigned integer of up to 64 bits to the output
-// writer.
-func (this *Encoder) EncodeGBEU(value uint64) (n int, err error) {
-	// increase if go somehow gets support for over 64 bit integers. we
-	// could also make an expanding int type in goutil to use here, or maybe
-	// there is one in the stdlib. keep this int64 version as well though
-	// because its ergonomic.
-	buffer := [16]byte { }
-	
-	window := (GBEUSize(value) - 1) * 7
-	index := 0
-	for window >= 0 {
-		chunk := uint8(value >> window) & 0x7F
-		if window > 0 {
-			chunk |= 0x80
-		}
-		buffer[index] = chunk
-		
-		index += 1
-		window -= 7
-	}
-	
-	return this.Write(buffer[:])
-}

codec/measure.go

@@ -1,11 +0,0 @@
-package codec
-
-// GBEUSize returns the size (in octets) of a GBEU integer.
-func GBEUSize(value uint64) int {
-	length := 0
-	for {
-		value >>= 7
-		length ++
-		if value == 0 { return length }
-	}
-}

design/branched-generated-encoder.md

@@ -0,0 +1,128 @@
+# Branched Generated Decoder
+
+Pasted here because Tebitea is down
+
+## The problem
+
+TAPE is designed so that the decoder can gloss over data it does not understand.
+Technically the protocol allows for this, but I completely forgot to implement
+this in the generated decoder, oops. This would be trivial if TAPE messages were
+still flat tables, but they aren't, because those aren't useful enough. So,
+let's analyze the problem.
+
+## When it happens
+
+There are two reasons something might not match up with the expected data:
+
+The first and most obvious is unrecognized keys. If the key is not in the set of
+recognized keys for a KTV, it should leave the corresponding struct field blank.
+Once #6 has been implemented, throw an error if the data was not optional.
+
+The second is wrong types. If we are expecting KTV and get SBA, we should leave
+the data as empty. The aforementioned concern about #6 also applies here. We
+don't need to worry about special cases at the structure root, because it would
+be technically possible to make the structure root an option, so it really is
+just a normal value. Until #6, we will leave that blank too.
+
+## Preliminary ideas
+
+The first is going to be pretty simple. All we need to do is have a skimmer
+function that skims over TAPE data very, and then call that on the KTV value
+each time we run into a mystery key. It should only return an error if the
+structure of the data is malformed in such a way that it cannot continue to the
+next one. This should be stored in the tape package alongside the dynamic
+decoding functions, because they will essentially function the same way and
+could probably share lots of code.
+
+The second is a bit more complicated because of the existence of KTV and OTA
+because they are aggregate types. Go types work a bit differently, as if you
+have an array of an array of an array of ints, that information is represented
+in one place, whereas TAPE doesn't really do that. All of that information is
+sort of buried within the data structure, so we don't know what we will be
+decoding before we actually do it. Whenever we encounter a type we don't expect,
+we would need to abort decoding of the entire data structure, and then skim over
+whatever detritus is left, which would literally be in a half-decoded state. The
+fact that the code is generated flat and thus cannot use return or defer
+statements contributes to the complexity of this problem. We need to go up, but
+we can't. There is no up, only forward.
+
+Of course, the dynamic decoder does not have this problem in the first place
+because it doesn't expect anything, and constructs the destination to fit
+whatever it sees in the TAPE structure as it is decoding it. KTVs are completely
+dynamic because they are implemented as maps, so the only time it needs to
+completely comprehend a type is with OTAs. There is a function called typeOf
+that gets the type of the current tag and returns it as a reflect.Type, which
+necessitates recursion and peeking at OTAs and their elements.
+
+We could try to do the same thing in the generated decoder, comparing the
+determined type against the expected type to try to figure out whether we should
+decode an array or a table, etc. This is immediately problematic as it requires
+memory to be allocated, both for the peek buffer and the resulting tree of type
+information. If we end up with some crazy way to keep track of the types, that's
+only one half of the allocation problem and we would still be spending extra
+cycles going over all of that twice.
+
+## Performance constraints
+
+The generated decoder is supposed to blaze through data, and it can't do that if
+it does all the singing and dancing that the dynamic decoder does. It's time for
+some performance constraints:
+
+- No allocations, except as required to build the destination for the data
+- No redundant work
+- So, no freaking peeking
+- It should take well under 500 lines of generated code to decode one message of
+reasonable size (i.e. be careful not to bloat the binary)
+
+I'm not really going to do my usual thing here of making a slow version and
+speeding it up over time based on evidence and experimentation because these
+constraints inform the design so much it would be impossible to continue without
+them. I am 99% confident that these constraints will allow for an acceptable
+baseline of performance (for generated code) and we can still profile and
+micro-optimize later. This is good enough for me.
+Heavy solution
+
+There is a solution that might work very well which involves completely redoing
+the generated decoding code. We could create a function for every source type to
+destination type mapping that exists in protocol, and then compose them all
+together. The decoding methods for each message or type would be wrappers around
+the correct function for their root TAPE -> Go type mapping. The main benefit of
+this is it would make this problem a lot more manageable because the interface
+points between the data would be represented by function boundaries. This would
+allow the use of return and defer statements, and would allow more code sharing,
+producing a smaller binary. Go would probably inline these where needed.
+
+Would this work? Probably. More investigation is required to make sure. I want
+to stop re-writing things I don't need to. On the other hand, it is just the
+decoder.
+
+## Light solution
+
+TODO: find a solution that satisfies the performance constraints, keeps the same
+identical interface, and works off the same code. I am convinced this is doable,
+and it might even allow us to extract more data from an unexpected structure.
+However, continuing this way might introduce unmanageable complexity. It is
+already a little unmanageable and I am just one pony (kind of).
+
+## Implementation
+
+Heavy solution is going to work here, applied to only the points of
+`Generator.generateDecodeValue` where it decodes an aggregate data structure.
+That way, only minimal amounts of code need to be redone.
+
+Whenever a branch needs to happen, a call shall be generated, a deferred
+implementation request shall be added to a special FIFO queue within the
+generator. After generating data structures and their root decoding functions,
+the generator shall pick away at this queue until no requests remain. The
+generator shall accept new items during this process, so that recursion is
+possible. This is all to ensure it is only ever writing one function at a time
+
+The functions shall take a pointer to a type that accepts any type like (~) the
+destination's base type. We should also probably just call
+`Generator.generateDecodeValue` directly on user defined types this way, keeping
+their public `Decode` methods just for convenience.
+
+The tape package shall contain a skimming function that takes a decoder and a
+tag, and recursively consumes the decoder given the context of the tag. This
+shall be utilized by the decoder functions to skip over values if their tags
+or keys do not match up with what is expected.

design/pdl-compiler.md

@@ -56,20 +56,27 @@ static section.
 
 For each defined type, the compiler shall generate a Go type with the same name
 as written in its definition. The Go type shall be encodable, and shall have
-`Encode` and `Decode` methods as described below.
+`EncodeValue`, `DecodeValue`, and `Tag` methods as described below.
 
 ## Encoding and Decoding Methods
 
-Each encodable type shall be given an `Encode` method and a `Decode` method,
+Each message shall be given an `Encode` method and a `Decode` method,
-which will take in a `codec.Encoder` and a `codec.Decoder` respectively. Both
+which shall take in a `codec.Encoder` and a `codec.Decoder` respectively. Both
 return an `(int, error)` pair describing the amount of bytes written and an
-error if the write stopped early. `Encode` will encode the data within the
+error if the write stopped early. `Encode` shall encode the data within the
-message to the given encoder, and `Decode` will decode data from the given
+message to the given encoder, and `Decode` shall decode data from the given
 decoder and place it in the type's value. The methods shall not retain or close
 any encoders or decoders they are given. Both methods shall have pointer
-receivers. In effect, these methods will satisfy `codec.Encodable` and
+receivers. In effect, these methods shall satisfy `codec.Encodable` and
 `codec.Decodable`.
 
+Each defined type shall be given an `EncodeValue` method and a `DecodeValue`
+method, which shall both take in a `tape.Tag`, then a `codec.Encoder` and a
+`codec.Decoder` respectively. These methods shall encode and decode the value
+according to the CN given by the tag. The TN shall be ignored. The message shall
+also have a method `Tag` that takes no arguments and returns the preferred tag
+of the type including the TN and CN.
+
 ## Connection
 
 The compiler shall generate a `Conn` struct which embeds a `hopp.Conn`, which

design/pdl-language.md

@@ -43,7 +43,6 @@ structures. They are separated by whitespace.
 
 | Name     | Syntax             | Description
 | -------- | ------------------ | -----------
-| Magic    | `PDL/0`            | Must appear at the very start of the file.
 | Method   | `M[0-9A-Fa-f]{4}`  | A 16-bit hexadecimal method code.
 | Key      | `[0-9A-Fa-f]{4}`   | A 16-bit hexadecimal table key.
 | Ident    | `[A-Z][A-Za-z0-9]` | An identifier.
@@ -55,8 +54,6 @@ structures. They are separated by whitespace.
 
 ## Syntax
 
-All files must begin with a Magic token.
-
 Types are expressed with an Ident. A table can be used by either writing the
 name of the type (Table), or by defining a schema with curly braces. Arrays must
 be expressed using two matching square brackets before their element type.
@@ -73,8 +70,6 @@ can be anything.
 Here is an example of all that:
 
 ```
-PDL/0
-
 M0000 Connect {
 	0000 Name String,
 	0001 Password String,
@@ -96,8 +91,7 @@ User {
 Below is an EBNF description of the language.
 
 ```
-<file>    -> <magic> (<message> | <typedef)*
+<file>    -> (<message> | <typedef)*
-<magic>   -> "PDL/0"
 <method>  -> /M[0-9A-Fa-f]{4}/
 <key>     -> /[0-9A-Fa-f]{4}/
 <ident>   -> /[A-Z][A-Za-z0-9]/

generate/generate_test.go

@@ -0,0 +1,327 @@
+package generate
+
+// import "fmt"
+import "strings"
+import "testing"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+var testGenerateCorrect =
+`package protocol
+
+/* # Do not edit this package by hand!
+ * 
+ * This file was automatically generated by the Holanet PDL compiler. The
+ * source file is located at input.pdl
+ * Please edit that file instead, and re-compile it to this location.
+ *
+ * HOPP, TAPE, METADAPT, PDL/0 (c) 2025 holanet.xyz
+ */
+
+import "git.tebibyte.media/sashakoshka/hopp/tape"
+
+// Table is a KTV table with an undefined schema.
+type Table map[uint16] any
+
+// Message is any message that can be sent along this protocol.
+type Message interface {
+	tape.Encodable
+	tape.Decodable
+
+	// Method returns the method code of the message.
+	Method() uint16
+}
+
+// User represents the protocol data type User.
+type User struct {
+	Name string
+	Bio string
+	Followers uint32
+}
+
+// EncodeValue encodes the value of this type without the tag. The value is
+// encoded according to the parameters specified by the tag, if possible.
+func (this *User) EncodeValue(encoder *tape.Encoder) (n int, err error) {
+	nn, err := tape.WriteTableHeader(2)
+	n += nn; if err != nil { return n, err }
+	nn, err := encoder.WriteUint16(0x0000)
+	n += nn; if err != nil { return n, err }
+	nn, err := tape.WriteString(encoder, this.Name)
+	n += nn; if err != nil { return n, err }
+	nn, err := encoder.WriteUint16(0x0001)
+	n += nn; if err != nil { return n, err }
+	nn, err := tape.WriteString(encoder, this.Bio)
+	n += nn; if err != nil { return n, err }
+	return n, nil
+}
+
+// Decode replaces the data in this User with information from the decoder.
+func (this *User) Decode(decoder *tape.Decoder) (n int, err error) {
+	pull, nn, err := tape.ReadTableHeader(decoder)
+	n += nn; if err != nil { return n, err }
+
+	for {
+		key, tag, end, nn, err := pull()
+		n += nn; if err != nil { return n, err }
+		if end { break }
+
+		switch key {
+		case 0x0000:
+			value, nn, err := tape.ReadString(decoder)
+			n += nn; if err != nil { return n, err }
+			this.Name = value
+		case 0x0001:
+			value, nn, err := tape.ReadString(decoder)
+			n += nn; if err != nil { return n, err }
+			this.Bio = value 
+		}
+	}
+	
+	return n, nil
+}
+
+// MessageConnect represents the protocol message M0000 Connect.
+type MessageConnect struct {
+	Name string
+	Password string
+}
+
+// Method returns the method code, M0000.
+func (this *MessageConnect) Method() uint16 {
+	return 0x0000
+}
+
+// Encode encodes the message to the encoder.
+func (this *MessageConnect) Encode(encoder *tape.Encoder) (n int, err error) {
+	nn, err := tape.WriteTableHeader(2)
+	n += nn; if err != nil { return n, err }
+	nn, err := encoder.WriteUint16(0x0000)
+	n += nn; if err != nil { return n, err }
+	nn, err := tape.WriteString(encoder, this.Name)
+	n += nn; if err != nil { return n, err }
+	nn, err := encoder.WriteUint16(0x0001)
+	n += nn; if err != nil { return n, err }
+	nn, err := tape.WriteString(encoder, this.Password)
+	n += nn; if err != nil { return n, err }
+	return n, nil
+}
+
+// Decode replaces the data in this message with information from the decoder.
+func (this *MessageConnect) Decode(decoder *tape.Decoder) (n int, err error) {
+	pull, nn, err := tape.ReadTableHeader(decoder)
+	n += nn; if err != nil { return n, err }
+
+	for {
+		key, tag, end, nn, err := pull()
+		n += nn; if err != nil { return n, err }
+		if end { break }
+
+		switch key {
+		case 0x0000:
+			value, nn, err := tape.ReadString(decoder)
+			n += nn; if err != nil { return n, err }
+			this.Name = value
+		case 0x0001:
+			value, nn, err := tape.ReadString(decoder)
+			n += nn; if err != nil { return n, err }
+			this.Password = value 
+		}
+	}
+	
+	return n, nil
+}
+
+// MessageUserList represents the protocol message M0001 UserList.
+type MessageUserList struct {
+	Users []User
+}
+
+// Method returns the method code, M0001.
+func (this *MessageUserList) Method() uint16 {
+	return 0x0001
+}
+
+// TODO methods
+`
+
+func TestGenerate(test *testing.T) {
+	protocol := defaultProtocol()
+	protocol.Messages[0x0000] = Message {
+		Name: "Connect",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Password",  Type: TypeString { } },
+			},
+		},
+	}
+	protocol.Messages[0x0001] = Message {
+		Name: "UserList",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Users",  Type: TypeArray { Element: TypeNamed { Name: "User" } } },
+			},
+		},
+	}
+	protocol.Types["User"] = TypeTableDefined {
+		Fields: map[uint16] Field {
+			0x0000: Field { Name: "Name",      Type: TypeString { } },
+			0x0001: Field { Name: "Bio",       Type: TypeString { } },
+			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+		},
+	}
+
+	correct := testGenerateCorrect
+
+	builder := strings.Builder { }
+	generator := Generator { Output: &builder }
+	/* TODO test n: */ _, err := generator.Generate(&protocol)
+	if err != nil { test.Fatal(parse.Format(err)) }
+	got := builder.String()
+
+	test.Log("CORRECT:")
+	test.Log(correct)
+	test.Log("GOT:")
+	test.Log(got)
+
+	if correct != got {
+		test.Error("not equal")
+		for index := range min(len(correct), len(got)) {
+			if correct[index] == got[index] { continue }
+			test.Log("C:", correct[max(0, index - 8):min(len(correct), index + 8)])
+			test.Log("G:", got[max(0, index - 8):min(len(got), index + 8)])
+			break
+		}
+		test.FailNow()
+	}
+}
+
+func TestGenerateRun(test *testing.T) {
+	protocol := defaultProtocol()
+	protocol.Messages[0x0000] = Message {
+		Name: "Connect",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Password",  Type: TypeString { } },
+			},
+		},
+	}
+	protocol.Messages[0x0001] = Message {
+		Name: "UserList",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Users",  Type: TypeArray { Element: TypeNamed { Name: "User" } } },
+			},
+		},
+	}
+	protocol.Messages[0x0002] = Message {
+		Name: "Pulse",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Index",  Type: TypeInt { Bits: 5 } },
+				0x0001: Field { Name: "Offset", Type: TypeInt { Bits: 16, Signed: true }},
+				0x0002: Field { Name: "X",      Type: TypeFloat { Bits: 16 }},
+				0x0003: Field { Name: "Y",      Type: TypeFloat { Bits: 32 }},
+				0x0004: Field { Name: "Z",      Type: TypeFloat { Bits: 64 }},
+			},
+		},
+	}
+	protocol.Messages[0x0003] = Message {
+		Name: "NestedArray",
+		Type: TypeArray { Element: TypeArray { Element: TypeInt { Bits: 8 } } },
+	}
+	protocol.Types["User"] = TypeTableDefined {
+		Fields: map[uint16] Field {
+			0x0000: Field { Name: "Name",      Type: TypeString { } },
+			0x0001: Field { Name: "Bio",       Type: TypeString { } },
+			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+		},
+	}
+	testGenerateRun(test, &protocol, `
+		// imports
+	`, `
+		// test case
+		log.Println("MessageConnect")
+		messageConnect := MessageConnect {
+			Name: "rarity",
+			Password: "gems",
+		}
+		testEncode(
+			&messageConnect,
+			tu.S(0xC1, 0x02).AddVar(
+				[]byte { 0x00, 0x00, 0x66, 'r', 'a', 'r', 'i', 't', 'y' },
+				[]byte { 0x00, 0x01, 0x64, 'g', 'e', 'm', 's' },
+				))
+		log.Println("MessageUserList")
+		messageUserList := MessageUserList {
+			Users: []User {
+				User {
+					Name: "rarity",
+					Bio: "asdjads",
+					Followers: 0x324,
+				},
+				User {
+					Name: "deez nuts",
+					Bio: "logy",
+					Followers: 0x8000,
+				},
+				User {
+					Name: "creekflow",
+					Bio: "im creekflow",
+					Followers: 0x3894,
+				},
+			},
+		}
+		testEncode(
+			&messageUserList,
+			tu.S(0xC1, 0x01, 0x00, 0x00,
+				0xA1, 0x03, 0xC1,
+					).Add(0x03).AddVar(
+						[]byte { 0x00, 0x00, 0x66, 'r', 'a', 'r', 'i', 't', 'y' },
+						[]byte { 0x00, 0x01, 0x67, 'a', 's', 'd', 'j', 'a', 'd', 's' },
+						[]byte { 0x00, 0x02, 0x23, 0x00, 0x00, 0x03, 0x24 },
+					).Add(0x03).AddVar(
+						[]byte { 0x00, 0x00, 0x69, 'd', 'e', 'e', 'z', ' ', 'n', 'u', 't', 's' },
+						[]byte { 0x00, 0x01, 0x64, 'l', 'o', 'g', 'y' },
+						[]byte { 0x00, 0x02, 0x23, 0x00, 0x00, 0x80, 0x00 },
+					).Add(0x03).AddVar(
+						[]byte { 0x00, 0x00, 0x69, 'c', 'r', 'e', 'e', 'k', 'f', 'l', 'o', 'w' },
+						[]byte { 0x00, 0x01, 0x6C, 'i', 'm', ' ', 'c', 'r', 'e', 'e', 'k', 'f',
+							'l', 'o', 'w' },
+						[]byte { 0x00, 0x02, 0x23, 0x00, 0x00, 0x38, 0x94 },
+				))
+		log.Println("MessagePulse")
+		messagePulse := MessagePulse {
+			Index: 9,
+			Offset: -0x3521,
+			X: 45.389,
+			Y: 294.1,
+			Z: 384729384.234892034,
+		}
+		testEncode(
+			&messagePulse,
+			tu.S(0xC1, 0x05).AddVar(
+				[]byte { 0x00, 0x00, 0x09 },
+				[]byte { 0x00, 0x01, 0x21, 0xCA, 0xDF },
+				[]byte { 0x00, 0x02, 0x41, 0x51, 0xAC },
+				[]byte { 0x00, 0x03, 0x43, 0x43, 0x93, 0x0C, 0xCD },
+				[]byte { 0x00, 0x04, 0x47, 0x41, 0xB6, 0xEE, 0x81, 0x28, 0x3C, 0x21, 0xE2 },
+				))
+		log.Println("MessageNestedArray")
+		uint8s := func(n int) []uint8 {
+			array := make([]uint8, n)
+			for index := range array {
+				array[index] = uint8(index + 1) | 0xF0
+			}
+			return array
+		}
+		messageNestedArray := MessageNestedArray {
+			uint8s(6),
+			uint8s(35),
+		}
+		testEncode(
+			&messageNestedArray,
+			tu.S(0xA1, // TODO
+				))
+	`)
+}

generate/lex.go

@@ -0,0 +1,230 @@
+package generate
+
+import "io"
+import "bufio"
+import "unicode"
+import "unicode/utf8"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+const (
+	TokenMethod parse.TokenKind = iota
+	TokenKey
+	TokenIdent
+	TokenComma
+	TokenLBrace
+	TokenRBrace
+	TokenLBracket
+	TokenRBracket
+)
+
+var tokenNames = map[parse.TokenKind] string {
+	TokenMethod:   "Method",
+	TokenKey:      "Key",
+	TokenIdent:    "Ident",
+	TokenComma:    "Comma",
+	TokenLBrace:   "LBrace",
+	TokenRBrace:   "RBrace",
+	TokenLBracket: "LBracket",
+	TokenRBracket: "RBracket",
+}
+
+func Lex(fileName string, reader io.Reader) (parse.Lexer, error) {
+	lex := &lexer {
+		fileName: fileName,
+		lineScanner: bufio.NewScanner(reader),
+	}
+	lex.nextRune()
+	return lex, nil
+}
+
+type lexer struct {
+	fileName    string
+	lineScanner *bufio.Scanner
+	rune        rune
+	line        string
+	lineFood    string
+	
+	offset   int
+	row      int
+	column   int
+
+	eof bool
+
+}
+
+func (this *lexer) Next() (parse.Token, error) {
+	token, err := this.nextInternal()
+	if err == io.EOF { err = this.errUnexpectedEOF() }
+	return token, err
+}
+
+func (this *lexer) nextInternal() (token parse.Token, err error) {
+	err = this.skipWhitespace()
+	token.Position = this.pos()
+	if this.eof {
+		token.Kind = parse.EOF
+		err = nil
+		return
+	}
+	if err != nil { return }
+
+	appendRune := func () {
+		token.Value += string(this.rune)
+		err = this.nextRune()
+	}
+
+	doNumber := func () {
+		for isDigit(this.rune) {
+			appendRune()
+			if this.eof { err = nil; return }
+			if err != nil { return }
+		}
+	}
+
+	defer func () {
+		newPos := this.pos()
+		newPos.End -- // TODO figure out why tf we have to do this
+		token.Position = token.Position.Union(newPos)
+	} ()
+
+	switch {
+	// Method
+	case this.rune == 'M':
+		token.Kind = TokenMethod
+		err = this.nextRune()
+		if err != nil { return }
+		doNumber()
+		if this.eof { err = nil; return }
+	// Key
+	case isDigit(this.rune):
+		token.Kind = TokenKey
+		doNumber()
+		if this.eof { err = nil; return }
+	// Ident
+	case unicode.IsUpper(this.rune):
+		token.Kind = TokenIdent
+		for unicode.IsLetter(this.rune) || isDigit(this.rune) {
+			appendRune()
+			if this.eof { err = nil; return }
+			if err != nil { return }
+		}
+	// Comma
+	case this.rune == ',':
+		token.Kind = TokenComma
+		appendRune()
+		if this.eof { err = nil; return }
+	// LBrace
+	case this.rune == '{':
+		token.Kind = TokenLBrace
+		appendRune()
+		if this.eof { err = nil; return }
+	// RBrace
+	case this.rune == '}':
+		token.Kind = TokenRBrace
+		appendRune()
+		if this.eof { err = nil; return }
+	// LBracket
+	case this.rune == '[':
+		token.Kind = TokenLBracket
+		appendRune()
+		if this.eof { err = nil; return }
+	// RBracket
+	case this.rune == ']':
+		token.Kind = TokenRBracket
+		appendRune()
+		if this.eof { err = nil; return }
+	case unicode.IsPrint(this.rune):
+		err = parse.Errorf (
+			this.pos(), "unexpected rune '%c'",
+			this.rune)
+	default:
+		err = parse.Errorf (
+			this.pos(), "unexpected rune %U",
+			this.rune)
+	}
+
+	return
+}
+
+func (this *lexer) nextRune() error {
+	if this.lineFood == "" {
+		ok := this.lineScanner.Scan()
+		if ok {
+			this.line     = this.lineScanner.Text()
+			this.lineFood = this.line
+			this.rune     = '\n'
+			this.column   = 0
+			this.row      ++
+		} else {
+			err := this.lineScanner.Err()
+			if err == nil {
+				this.eof = true
+				return io.EOF
+			} else {
+				return err
+			}
+		}
+	} else {
+		var ch   rune
+		var size int
+		for ch == 0 && this.lineFood != "" {
+			ch, size = utf8.DecodeRuneInString(this.lineFood)
+			this.lineFood = this.lineFood[size:]
+		}
+		this.rune = ch
+		this.column ++
+	}
+
+	return nil
+}
+
+func (this *lexer) skipWhitespace() error {
+	err := this.skipComment()
+	if err != nil { return err }
+	for isWhitespace(this.rune) {
+		err := this.nextRune()
+		if err != nil { return err }
+		err = this.skipComment()
+		if err != nil { return err }
+	}
+	return nil
+}
+
+func (this *lexer) skipComment() error {
+	if this.rune == ';' {
+		for this.rune != '\n' {
+			err := this.nextRune()
+			if err != nil { return err }
+		}
+	}
+	return nil
+}
+
+func (this *lexer) pos() parse.Position {
+	return parse.Position {
+		File:   this.fileName,
+		Line:   this.lineScanner.Text(),
+		Row:    this.row    - 1,
+		Start:  this.column - 1,
+		End:    this.column,
+	}
+}
+
+func (this *lexer) errUnexpectedEOF() error {
+	return parse.Errorf(this.pos(), "unexpected EOF")
+}
+
+func isWhitespace(char rune) bool {
+	switch char {
+	case ' ', '\t', '\r', '\n': return true
+	default: return false
+	}
+}
+
+func isDigit(char rune) bool {
+	return char >= '0' && char <= '9'
+}
+
+func isHexDigit(char rune) bool {
+	return isDigit(char) || char >= 'a' && char <= 'f' || char >= 'A' && char <= 'F'
+}

generate/lex_test.go

@@ -0,0 +1,54 @@
+package generate
+
+import "strings"
+import "testing"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+func TestLex(test *testing.T) {
+	lexer, err := Lex("test.pdl", strings.NewReader(`
+	M0001 User {
+		0000 Name      String,
+		0001 Users []User,
+		0002 Followers U32,
+	}`))
+	if err != nil { test.Fatal(parse.Format(err)) }
+
+	correctTokens := []parse.Token {
+		tok(TokenMethod,   "0001"),
+		tok(TokenIdent,    "User"),
+		tok(TokenLBrace,   "{"),
+		tok(TokenKey,      "0000"),
+		tok(TokenIdent,    "Name"),
+		tok(TokenIdent,    "String"),
+		tok(TokenComma,    ","),
+		tok(TokenKey,      "0001"),
+		tok(TokenIdent,    "Users"),
+		tok(TokenLBracket, "["),
+		tok(TokenRBracket, "]"),
+		tok(TokenIdent,    "User"),
+		tok(TokenComma,    ","),
+		tok(TokenKey,      "0002"),
+		tok(TokenIdent,    "Followers"),
+		tok(TokenIdent,    "U32"),
+		tok(TokenComma,    ","),
+		tok(TokenRBrace,   "}"),
+		tok(parse.EOF,     ""),
+	}
+
+	for index, correct := range correctTokens {
+		got, err := lexer.Next()
+		if err != nil { test.Fatal(parse.Format(err)) }
+		if got.Kind != correct.Kind || got.Value != correct.Value {
+			test.Logf("token %d mismatch", index)
+			test.Log("GOT:", tokenNames[got.Kind], got.Value)
+			test.Fatal("CORRECT:", tokenNames[correct.Kind], correct.Value)
+		}
+	}
+}
+
+func tok(kind parse.TokenKind, value string) parse.Token {
+	return parse.Token {
+		Kind:  kind,
+		Value: value,
+	}
+}

generate/misc_test.go

@@ -0,0 +1,73 @@
+package generate
+
+import "os"
+import "fmt"
+import "os/exec"
+import "testing"
+import "path/filepath"
+
+func testGenerateRun(test *testing.T, protocol *Protocol, imports string, testCase string) {
+	// reset data directory
+	dir := "test/generate-run"
+	err := os.RemoveAll(dir)
+	if err != nil { test.Fatal(err) }
+	err = os.MkdirAll(dir, 0750)
+	if err != nil { test.Fatal(err) }
+
+	// open files
+	sourceFile, err := os.Create(filepath.Join(dir, "protocol.go"))
+	if err != nil { test.Fatal(err) }
+	defer sourceFile.Close()
+	mainFile, err := os.Create(filepath.Join(dir, "main.go"))
+	if err != nil { test.Fatal(err) }
+	defer mainFile.Close()
+
+	// generate protocol
+	generator := Generator {
+		Output: sourceFile,
+		PackageName: "main",
+	}
+	_, err = generator.Generate(protocol)
+	if err != nil { test.Fatal(err) }
+
+	// build static source files
+	imports = `
+		import "log"
+		import "bytes"
+		import "git.tebibyte.media/sashakoshka/hopp/tape"
+		import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
+		` + imports
+	setup := `log.Println("*** BEGIN TEST CASE OUTPUT ***")`
+	teardown := `log.Println("--- END TEST CASE OUTPUT ---")`
+	static := `
+		func testEncode(message Message, correct tu.Snake) {
+			buffer := bytes.Buffer { }
+			encoder := tape.NewEncoder(&buffer)
+			n, err := message.Encode(encoder)
+			if err != nil { log.Fatalf("at %d: %v\n", n, err) }
+			encoder.Flush()
+			got := buffer.Bytes()
+			log.Printf("got:     [%s]", tu.HexBytes(got))
+			log.Println("correct:", correct)
+			if n != len(got) {
+				log.Fatalf("n incorrect: %d != %d\n", n, len(got))
+			}
+			if ok, n := correct.Check(got); !ok {
+				log.Fatalln("not equal at", n)
+			}
+		}
+		`
+	fmt.Fprintf(
+		mainFile, "package main\n%s\nfunc main() {\n%s\n%s\n%s\n}\n%s",
+		imports, setup, testCase, teardown, static)
+
+	// build and run test
+	command := exec.Command("go", "run", "./generate/test/generate-run")
+	workingDirAbs, err := filepath.Abs("..")
+	if err != nil { test.Fatal(err) }
+	command.Dir = workingDirAbs
+	command.Env = os.Environ()
+	output, err := command.CombinedOutput()
+	test.Logf("output of %v:\n%s", command, output)
+	if err != nil { test.Fatal(err) }
+}

generate/parse.go

@@ -0,0 +1,207 @@
+package generate
+
+import "io"
+import "strconv"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+func Parse(lx parse.Lexer) (*Protocol, error) {
+	protocol := defaultProtocol()
+	par := parser {
+		Parser: parse.Parser {
+			Lexer: lx,
+			TokenNames: tokenNames,
+		},
+		protocol: &protocol,
+	}
+	err := par.parse()
+	if err != nil { return nil, err }
+	return par.protocol, nil
+}
+
+func defaultProtocol() Protocol {
+	return Protocol {
+		Messages: make(map[uint16] Message),
+		Types: map[string] Type { },
+	}
+}
+
+func ParseReader(reader io.Reader) (*Protocol, error) {
+	lx, err := Lex("test.pdl", reader)
+	if err != nil { return nil, err }
+	return Parse(lx)
+}
+
+type parser struct {
+	parse.Parser
+	protocol *Protocol
+}
+
+func (this *parser) parse() error {
+	err := this.Next()
+	if err != nil { return err }
+	for this.Token.Kind != parse.EOF {
+		err = this.parseTopLevel()
+		if err != nil { return err }
+	}
+	return nil
+}
+
+func (this *parser) parseTopLevel() error {
+	err := this.ExpectDesc("message or typedef", TokenMethod, TokenIdent)
+	if err != nil { return err }
+	if this.EOF() { return nil }
+
+	switch this.Kind() {
+	case TokenMethod: return this.parseMessage()
+	case TokenIdent:  return this.parseTypedef()
+	}
+	panic("bug")
+}
+
+func (this *parser) parseMessage() error {
+	err := this.Expect(TokenMethod)
+	if err != nil { return err }
+	method, err := this.parseHexNumber(this.Value(), 0xFFFF)
+	if err != nil { return err }
+	err = this.ExpectNext(TokenIdent)
+	if err != nil { return err }
+	name := this.Value()
+	err = this.Next()
+	if err != nil { return err }
+	typ, err := this.parseType()
+	if err != nil { return err }
+	this.protocol.Messages[uint16(method)] = Message {
+		Name: name,
+		Type: typ,
+	}
+	return nil
+}
+
+func (this *parser) parseTypedef() error {
+	err := this.Expect(TokenIdent)
+	if err != nil { return err }
+	name := this.Value()
+	err = this.Next()
+	if err != nil { return err }
+	typ, err := this.parseType()
+	if err != nil { return err }
+	this.protocol.Types[name] = typ
+	return nil
+}
+
+func (this *parser) parseType() (Type, error) {
+	err := this.ExpectDesc("type", TokenIdent, TokenLBracket, TokenLBrace)
+	if err != nil { return nil, err }
+
+	switch this.Kind() {
+	case TokenIdent:
+		switch this.Value() {
+		case "U8":     return TypeInt { Bits: 8   }, this.Next()
+		case "U16":    return TypeInt { Bits: 16  }, this.Next()
+		case "U32":    return TypeInt { Bits: 32  }, this.Next()
+		case "U64":    return TypeInt { Bits: 64  }, this.Next()
+		case "U128":   return TypeInt { Bits: 128 }, this.Next()
+		case "U256":   return TypeInt { Bits: 256 }, this.Next()
+		case "I8":     return TypeInt { Bits: 8,   Signed: true }, this.Next()
+		case "I16":    return TypeInt { Bits: 16,  Signed: true }, this.Next()
+		case "I32":    return TypeInt { Bits: 32,  Signed: true }, this.Next()
+		case "I64":    return TypeInt { Bits: 64,  Signed: true }, this.Next()
+		case "I128":   return TypeInt { Bits: 128, Signed: true }, this.Next()
+		case "I256":   return TypeInt { Bits: 256, Signed: true }, this.Next()
+		case "F16":    return TypeFloat { Bits: 16  }, this.Next()
+		case "F32":    return TypeFloat { Bits: 32  }, this.Next()
+		case "F64":    return TypeFloat { Bits: 64  }, this.Next()
+		case "F128":   return TypeFloat { Bits: 128 }, this.Next()
+		case "F256":   return TypeFloat { Bits: 256 }, this.Next()
+		case "String": return TypeString { }, this.Next()
+		case "Buffer": return TypeBuffer { }, this.Next()
+		case "Table":  return TypeTable { }, this.Next()
+		}
+		return this.parseTypeNamed()
+	case TokenLBracket:
+		return this.parseTypeArray()
+	case TokenLBrace:
+		return this.parseTypeTable()
+	}
+	panic("bug")
+}
+
+func (this *parser) parseTypeNamed() (TypeNamed, error) {
+	err := this.Expect(TokenIdent)
+	if err != nil { return TypeNamed { }, err }
+	name := this.Value()
+	err = this.Next()
+	if err != nil { return TypeNamed { }, err }
+	return TypeNamed { Name: name }, nil
+}
+
+func (this *parser) parseTypeArray() (TypeArray, error) {
+	err := this.Expect(TokenLBracket)
+	if err != nil { return TypeArray { }, err }
+	err = this.ExpectNext(TokenRBracket)
+	if err != nil { return TypeArray { }, err }
+	err = this.Next()
+	if err != nil { return TypeArray { }, err }
+	typ, err := this.parseType()
+	if err != nil { return TypeArray { }, err }
+	return TypeArray { Element: typ }, nil
+}
+
+func (this *parser) parseTypeTable() (TypeTableDefined, error) {
+	err := this.Expect(TokenLBrace)
+	if err != nil { return TypeTableDefined { }, err }
+	err = this.Next()
+	if err != nil { return TypeTableDefined { }, err }
+	typ := TypeTableDefined {
+		Fields: make(map[uint16] Field),
+	}
+	for {
+		err := this.ExpectDesc("table field", TokenKey, TokenRBrace)
+		if err != nil { return TypeTableDefined { }, err }
+		if this.Is(TokenRBrace) {
+			break
+		}
+		key, field, err := this.parseField()
+		if err != nil { return TypeTableDefined { }, err }
+		typ.Fields[key] = field
+		err = this.Expect(TokenComma, TokenRBrace)
+		if err != nil { return TypeTableDefined { }, err }
+		if this.Is(TokenRBrace) {
+			break
+		}
+		err = this.Next()
+		if err != nil { return TypeTableDefined { }, err }
+	}
+	err = this.Next()
+	if err != nil { return TypeTableDefined { }, err }
+	return typ, nil
+}
+
+func (this *parser) parseField() (uint16, Field, error) {
+	err := this.Expect(TokenKey)
+	if err != nil { return 0, Field { }, err }
+	key, err := this.parseHexNumber(this.Value(), 0xFFFF)
+	if err != nil { return 0, Field { }, err }
+	err = this.ExpectNext(TokenIdent)
+	if err != nil { return 0, Field { }, err }
+	name := this.Value()
+	err = this.Next()
+	if err != nil { return 0, Field { }, err }
+	typ, err := this.parseType()
+	if err != nil { return 0, Field { }, err }
+	return uint16(key), Field {
+		Name: name,
+		Type: typ,
+	}, nil
+}
+
+func (this *parser) parseHexNumber(input string, maxValue int64) (int64, error) {
+	number, err := strconv.ParseInt(input, 16, 64)
+	if err != nil {
+		return 0, parse.Errorf(this.Pos(), "%v", err)
+	}
+	if maxValue > 0 && number > maxValue {
+		return 0, parse.Errorf(this.Pos(), "value too large (max %X)", maxValue)
+	}
+	return number, nil
+}

generate/parse_test.go

@@ -0,0 +1,68 @@
+package generate
+
+import "fmt"
+import "strings"
+import "testing"
+import "git.tebibyte.media/sashakoshka/goparse"
+
+func TestParse(test *testing.T) {
+	correct := defaultProtocol()
+	correct.Messages[0x0000] = Message {
+		Name: "Connect",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Name",      Type: TypeString { } },
+				0x0001: Field { Name: "Password",  Type: TypeString { } },
+			},
+		},
+	}
+	correct.Messages[0x0001] = Message {
+		Name: "UserList",
+		Type: TypeTableDefined {
+			Fields: map[uint16] Field {
+				0x0000: Field { Name: "Users",  Type: TypeArray { Element: TypeNamed { Name: "User" } } },
+			},
+		},
+	}
+	correct.Types["User"] = TypeTableDefined {
+		Fields: map[uint16] Field {
+			0x0000: Field { Name: "Name",      Type: TypeString { } },
+			0x0001: Field { Name: "Bio",       Type: TypeString { } },
+			0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } },
+		},
+	}
+	test.Log("CORRECT:", &correct)
+	
+	got, err := ParseReader(strings.NewReader(`
+	M0000 Connect {
+		0000 Name String,
+		0001 Password String,
+	}
+	
+	M0001 UserList {
+		0000 Users []User,
+	}
+	
+	User {
+		0000 Name      String,
+		0001 Bio       String,
+		0002 Followers U32,
+	}
+	`))
+	if err != nil { test.Fatal(parse.Format(err)) }
+	test.Log("GOT:    ", got)
+
+	correctStr := fmt.Sprint(&correct)
+	gotStr     := fmt.Sprint(got)
+
+	if correctStr != gotStr {
+		test.Error("not equal")
+		for index := range min(len(correctStr), len(gotStr)) {
+			if correctStr[index] == gotStr[index] { continue }
+			test.Log("C:", correctStr[max(0, index - 8):min(len(correctStr), index + 8)])
+			test.Log("G:", gotStr[max(0, index - 8):min(len(gotStr), index + 8)])
+			break
+		}
+		test.FailNow()
+	}
+}

generate/protocol.go

@@ -0,0 +1,107 @@
+package generate
+
+import "fmt"
+import "maps"
+import "slices"
+import "crypto/md5"
+
+type Protocol struct {
+	Messages map[uint16] Message
+	Types    map[string] Type
+}
+
+type Message struct {
+	Name string
+	Type Type
+}
+
+type Type interface {
+	fmt.Stringer
+}
+
+type TypeInt struct {
+	Bits   int
+	Signed bool
+}
+
+func (typ TypeInt) String() string {
+	output := ""
+	if typ.Signed {
+		output += "I"
+	} else {
+		output += "U"
+	}
+	output += fmt.Sprint(typ.Bits)
+	return output
+}
+
+type TypeFloat struct {
+	Bits int
+}
+
+func (typ TypeFloat) String() string {
+	return fmt.Sprintf("F%d", typ.Bits)
+}
+
+type TypeString struct { }
+
+func (TypeString) String() string {
+	return "String"
+}
+
+type TypeBuffer struct { }
+
+func (TypeBuffer) String() string {
+	return "Buffer"
+}
+
+type TypeArray struct {
+	Element Type
+}
+
+func (typ TypeArray) String() string {
+	return fmt.Sprintf("[]%v", typ.Element)
+}
+
+type TypeTable struct { }
+
+func (TypeTable) String() string {
+	return "Table"
+}
+
+type TypeTableDefined struct {
+	Fields map[uint16] Field
+}
+
+func (typ TypeTableDefined) String() string {
+	output := "{"
+	for _, key := range slices.Sorted(maps.Keys(typ.Fields)) {
+		output += fmt.Sprintf("%04X %v", key, typ.Fields[key])
+	}
+	output += "}"
+	return output
+}
+
+type Field struct {
+	Name string
+	Type Type
+}
+
+func (field Field) String() string {
+	return fmt.Sprintf("%s %v", field.Name, field.Type)
+}
+
+type TypeNamed struct {
+	Name string
+}
+
+func (typ TypeNamed) String() string {
+	return typ.Name
+}
+
+func HashType(typ Type) [16]byte {
+	// TODO: if we ever want to make the compiler more efficient, this would
+	// be a good place to start, complex string concatenation in a hot path
+	// (sorta)
+	return md5.Sum([]byte(typ.String()))
+}

go.mod

@@ -4,5 +4,5 @@ go 1.23.0
 
 require (
 	git.tebibyte.media/sashakoshka/go-util v0.9.1
-	github.com/gomarkdown/markdown v0.0.0-20241205020045-f7e15b2f3e62
+	git.tebibyte.media/sashakoshka/goparse v0.2.0
 )

go.sum

@@ -1,4 +1,4 @@
 git.tebibyte.media/sashakoshka/go-util v0.9.1 h1:eGAbLwYhOlh4aq/0w+YnJcxT83yPhXtxnYMzz6K7xGo=
 git.tebibyte.media/sashakoshka/go-util v0.9.1/go.mod h1:0Q1t+PePdx6tFYkRuJNcpM1Mru7wE6X+it1kwuOH+6Y=
-github.com/gomarkdown/markdown v0.0.0-20241205020045-f7e15b2f3e62 h1:pbAFUZisjG4s6sxvRJvf2N7vhpCvx2Oxb3PmS6pDO1g=
+git.tebibyte.media/sashakoshka/goparse v0.2.0 h1:uQmKvOCV2AOlCHEDjg9uclZCXQZzq2PxaXfZ1aIMiQI=
-github.com/gomarkdown/markdown v0.0.0-20241205020045-f7e15b2f3e62/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
+git.tebibyte.media/sashakoshka/goparse v0.2.0/go.mod h1:tSQwfuD+EujRoKr6Y1oaRy74ZynatzkRLxjE3sbpCmk=

internal/testutil/testutil.go

@@ -0,0 +1,161 @@
+package testutil
+
+import "fmt"
+import "slices"
+import "strings"
+import "reflect"
+
+// Snake lets you compare blocks of data where the ordering of certain parts may
+// be swapped every which way. It is designed for comparing the encoding of
+// maps where the ordering of individual elements is inconsistent.
+//
+// The snake is divided into sectors, which hold a number of variations. For a
+// sector to be satisfied by some data, some ordering of it must match the data
+// exactly. for the snake to be satisfied by some data, its sectors must match
+// the data in order, but the internal ordering of each sector doesn't matter.
+type Snake []    []     []byte
+//         snake sector variation
+
+// S returns a new snake.
+func S(data ...byte) Snake {
+	return (Snake { }).Add(data...)
+}
+
+// AddVar returns a new snake with the given sector added on to it. Successive
+// calls of this method can be chained together to create a big ass snake.
+func (sn Snake) AddVar(sector ...[]byte) Snake {
+	slice := make(Snake, len(sn) + 1)
+	copy(slice, sn)
+	slice[len(slice) - 1] = sector
+	return slice
+}
+
+// Add is like AddVar, but adds a sector with only one variation, which means it
+// does not vary, hence why the method is called that.
+func (sn Snake) Add(data ...byte) Snake {
+	return sn.AddVar(data)
+}
+
+// Check determines if the data satisfies the snake.
+func (sn Snake) Check(data []byte) (ok bool, n int) {
+	left := data
+	variations := map[int] []byte { }
+	for _, sector := range sn {
+		clear(variations)
+		for key, variation := range sector {
+			variations[key] = variation
+		}
+		for len(variations) > 0 {
+			found := false
+			for key, variation := range variations {
+				if len(left) < len(variation) { continue }
+				if !slices.Equal(left[:len(variation)], variation) { continue }
+				n += len(variation)
+				left = data[n:]
+				delete(variations, key)
+				found = true
+			}
+			if !found { return false, n }
+		}
+	}
+	if n < len(data) {
+		return false, n
+	}
+	return true, n
+}
+
+func (sn Snake) String() string {
+	if len(sn) == 0 || len(sn[0]) == 0 || len(sn[0][0]) == 0 {
+		return "EMPTY"
+	}
+
+	out := strings.Builder { }
+	for index, sector := range sn {
+		if index > 0 { out.WriteString(" : ") }
+		out.WriteRune('[')
+		for index, variation := range sector {
+			if index > 0 { out.WriteString(" / ") }
+			for _, byt := range variation {
+				fmt.Fprintf(&out, "%02x", byt)
+			}
+		}
+		out.WriteRune(']')
+	}
+	return out.String()
+}
+
+// HexBytes formats bytes into a hexadecimal string.
+func HexBytes(data []byte) string {
+	if len(data) == 0 { return "EMPTY" }
+	out := strings.Builder { }
+	for _, byt := range data {
+		fmt.Fprintf(&out, "%02x", byt)
+	}
+	return out.String()
+}
+
+// Describe returns a string representing the type and data of the given value.
+func Describe(value any) string {
+	desc := describer { }
+	desc.describe(reflect.ValueOf(value))
+	return desc.String()
+}
+
+type describer struct {
+	strings.Builder
+	indent int
+}
+
+func (this *describer) describe(value reflect.Value) {
+	value = reflect.ValueOf(value.Interface())
+	switch value.Kind() {
+	case reflect.Array, reflect.Slice:
+		this.printf("[\n")
+		this.indent += 1
+		for index := 0; index < value.Len(); index ++ {
+			this.iprintf("")
+			this.describe(value.Index(index))
+			this.iprintf("\n")
+		}
+		this.indent -= 1
+		this.iprintf("]")
+	case reflect.Struct:
+		this.printf("struct {\n")
+		this.indent += 1
+		typ := value.Type()
+		for index := range typ.NumField() {
+			indexBuffer := [1]int { index }
+			this.iprintf("%s: ", typ.Field(index).Name)
+			this.describe(value.FieldByIndex(indexBuffer[:]))
+			this.iprintf("\n")
+		}
+		this.indent -= 1
+		this.iprintf("}\n")
+	case reflect.Map:
+		this.printf("map {\n")
+		this.indent += 1
+		iter := value.MapRange()
+		for iter.Next() {
+			this.iprintf("")
+			this.describe(iter.Key())
+			this.printf(": ")
+			this.describe(iter.Value())
+			this.iprintf("\n")
+		}
+		this.indent -= 1
+		this.iprintf("}\n")
+	case reflect.Pointer:
+		this.printf("& ")
+		this.describe(value.Elem())
+	default:
+		this.printf("<%v %v>", value.Type(), value.Interface())
+	}
+}
+
+func (this *describer) printf(format string, v ...any) {
+	fmt.Fprintf(this, format, v...)
+}
+
+func (this *describer) iprintf(format string, v ...any) {
+	fmt.Fprintf(this, strings.Repeat("\t", this.indent) + format, v...)
+}

internal/testutil/testutil_test.go

@@ -0,0 +1,66 @@
+package testutil
+
+import "testing"
+
+func TestSnakeA(test *testing.T) {
+	snake := S(1, 6).AddVar(
+		[]byte { 1 },
+		[]byte { 2 },
+		[]byte { 3 },
+		[]byte { 4 },
+		[]byte { 5 },
+	).Add(9)
+
+	test.Log(snake)
+
+	ok, n := snake.Check([]byte { 1, 6, 1, 2, 3, 4, 5, 9 })
+	if !ok { test.Fatal("false negative:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 5, 4, 3, 2, 1, 9 })
+	if !ok { test.Fatal("false negative:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 3, 1, 4, 2, 5, 9 })
+	if !ok { test.Fatal("false negative:", n) }
+	
+	ok, n = snake.Check([]byte { 1, 6, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 1, 2, 3, 4, 5, 6, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 0, 2, 3, 4, 5, 6, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 7, 1, 4, 2, 5, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 7, 3, 1, 4, 2, 5, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 7, 3, 1, 4, 2, 5, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 1, 2, 3, 4, 5, 9, 10})
+	if ok { test.Fatal("false positive:", n) }
+}
+
+func TestSnakeB(test *testing.T) {
+	snake := S(1, 6).AddVar(
+		[]byte { 1 },
+		[]byte { 2 },
+	).Add(9).AddVar(
+		[]byte { 3, 2 },
+		[]byte { 0 },
+		[]byte { 1, 1, 2, 3 },
+	)
+
+	test.Log(snake)
+
+	ok, n := snake.Check([]byte { 1, 6, 1, 2, 9, 3, 2, 0, 1, 1, 2, 3})
+	if !ok { test.Fatal("false negative:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 2, 1, 9, 0, 1, 1, 2, 3, 3, 2})
+	if !ok { test.Fatal("false negative:", n) }
+	
+	ok, n = snake.Check([]byte { 1, 6, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 1, 2, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 9, 3, 2, 0, 1, 1, 2, 3})
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 2, 9, 0, 1, 1, 2, 3, 3, 2})
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 1, 2, 9, 3, 2, 1, 1, 2, 3})
+	if ok { test.Fatal("false positive:", n) }
+}

tape/decode.go

@@ -0,0 +1,192 @@
+package tape
+
+import "io"
+import "math"
+import "bufio"
+
+// Decodable is any type that can decode itself from a decoder.
+type Decodable interface {
+	// Decode reads data from decoder, replacing the data of the object. It
+	// returns the amount of bytes written, and an error if the write
+	// stopped early.
+	Decode(decoder *Decoder) (n int, err error)
+}
+
+// Decoder decodes data from an [io.Reader].
+type Decoder struct {
+	bufio.Reader
+}
+
+// NewDecoder creates a new decoder that reads from reader.
+func NewDecoder(reader io.Reader) *Decoder {
+	decoder := &Decoder { }
+	decoder.Reader.Reset(reader)
+	return decoder
+}
+
+// ReadFull calls [io.ReadFull] on the reader.
+func (this *Decoder) ReadFull(buffer []byte) (n int, err error) {
+	return io.ReadFull(this, buffer)
+}
+
+// ReadInt8 decodes an 8-bit signed integer from the input reader.
+func (this *Decoder) ReadInt8() (value int8, n int, err error) {
+	uncasted, n, err := this.ReadUint8()
+	return int8(uncasted), n, err
+}
+
+// ReadUint8 decodes an 8-bit unsigned integer from the input reader.
+func (this *Decoder) ReadUint8() (value uint8, n int, err error) {
+	buffer := [1]byte { }
+	n, err = this.ReadFull(buffer[:])
+	return uint8(buffer[0]), n, err
+}
+
+// ReadInt16 decodes an 16-bit signed integer from the input reader.
+func (this *Decoder) ReadInt16() (value int16, n int, err error) {
+	uncasted, n, err := this.ReadUint16()
+	return int16(uncasted), n, err
+}
+
+// ReadUint16 decodes an 16-bit unsigned integer from the input reader.
+func (this *Decoder) ReadUint16() (value uint16, n int, err error) {
+	buffer := [2]byte { }
+	n, err = this.ReadFull(buffer[:])
+	return uint16(buffer[0]) << 8 |
+		uint16(buffer[1]), n, err
+}
+
+// ReadInt32 decodes an 32-bit signed integer from the input reader.
+func (this *Decoder) ReadInt32() (value int32, n int, err error) {
+	uncasted, n, err := this.ReadUint32()
+	return int32(uncasted), n, err
+}
+
+// ReadUint32 decodes an 32-bit unsigned integer from the input reader.
+func (this *Decoder) ReadUint32() (value uint32, n int, err error) {
+	buffer := [4]byte { }
+	n, err = this.ReadFull(buffer[:])
+	return uint32(buffer[0]) << 24 |
+		uint32(buffer[1]) << 16 |
+		uint32(buffer[2]) << 8 |
+		uint32(buffer[3]), n, err
+}
+
+// ReadInt64 decodes an 64-bit signed integer from the input reader.
+func (this *Decoder) ReadInt64() (value int64, n int, err error) {
+	uncasted, n, err := this.ReadUint64()
+	return int64(uncasted), n, err
+}
+
+// ReadUint64 decodes an 64-bit unsigned integer from the input reader.
+func (this *Decoder) ReadUint64() (value uint64, n int, err error) {
+	buffer := [8]byte { }
+	n, err = this.ReadFull(buffer[:])
+	return uint64(buffer[0]) << 56 |
+		uint64(buffer[1]) << 48 |
+		uint64(buffer[2]) << 40 |
+		uint64(buffer[3]) << 32 |
+		uint64(buffer[4]) << 24 |
+		uint64(buffer[5]) << 16 |
+		uint64(buffer[6]) << 8 |
+		uint64(buffer[7]), n, err
+}
+
+// ReadIntN decodes an N-byte signed integer from the input reader.
+func (this *Decoder) ReadIntN(bytes int) (value int64, n int, err error) {
+	uncasted, n, err := this.ReadUintN(bytes)
+	return int64(uncasted), n, err
+}
+
+// ReadUintN decodes an N-byte unsigned integer from the input reader.
+func (this *Decoder) ReadUintN(bytes int) (value uint64, n int, err error) {
+	// TODO: don't make multiple read calls (without allocating)
+	buffer := [1]byte { }
+	for bytesLeft := bytes; bytesLeft > 0; bytesLeft -- {
+		nn, err := this.ReadFull(buffer[:])
+		n += nn; if err != nil { return 0, n, err }
+		value |= uint64(buffer[0]) << ((bytesLeft - 1) * 8)
+	}
+	// *read* integers too big, but don't return them.
+	if bytes > 8 { value = 0 }
+	return value, n, nil
+}
+
+// ReadFloat16 decodes a 16-bit floating point value from the input reader.
+func (this *Decoder) ReadFloat16() (value float32, n int, err error) {
+	bits, nn, err := this.ReadUint16()
+	n += nn; if err != nil { return 0, n, err }
+	return math.Float32frombits(f16bitsToF32bits(bits)), n, nil
+}
+
+// ReadFloat32 decldes a 32-bit floating point value from the input reader.
+func (this *Decoder) ReadFloat32() (value float32, n int, err error) {
+	bits, nn, err := this.ReadUint32()
+	n += nn; if err != nil { return 0, n, err }
+	return math.Float32frombits(bits), n, nil
+}
+
+// ReadFloat64 decldes a 64-bit floating point value from the input reader.
+func (this *Decoder) ReadFloat64() (value float64, n int, err error) {
+	bits, nn, err := this.ReadUint64()
+	n += nn; if err != nil { return 0, n, err }
+	return math.Float64frombits(bits), n, nil
+}
+
+// ReadTag decodes a [Tag] from the input reader.
+func (this *Decoder) ReadTag() (value Tag, n int, err error) {
+	uncasted, nn, err := this.ReadUint8()
+	n += nn; if err != nil { return 0, n, err }
+	return Tag(uncasted), n, nil
+}
+
+// f16bitsToF32bits returns uint32 (float32 bits) converted from specified uint16.
+// Taken from https://github.com/x448/float16/blob/v0.8.4/float16
+//
+// MIT License
+// 
+// Copyright (c) 2019 Montgomery Edwards⁴⁴⁸ and Faye Amacker
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+func f16bitsToF32bits(in uint16) uint32 {
+	// All 65536 conversions with this were confirmed to be correct
+	// by Montgomery Edwards⁴⁴⁸ (github.com/x448).
+	
+	sign := uint32(in&0x8000) << 16 // sign for 32-bit
+	exp := uint32(in&0x7c00) >> 10  // exponenent for 16-bit
+	coef := uint32(in&0x03ff) << 13 // significand for 32-bit
+
+	if exp == 0x1f {
+		if coef == 0 {
+			// infinity
+			return sign | 0x7f800000 | coef
+		}
+		// NaN
+		return sign | 0x7fc00000 | coef
+	}
+
+	if exp == 0 {
+		if coef == 0 {
+			// zero
+			return sign
+		}
+
+		// normalize subnormal numbers
+		exp++
+		for coef&0x7f800000 == 0 {
+			coef <<= 1
+			exp--
+		}
+		coef &= 0x007fffff
+	}
+
+	return sign | ((exp + (0x7f - 0xf)) << 23) | coef
+}

tape/dynamic.go

@@ -0,0 +1,392 @@
+package tape
+
+// dont smoke reflection, kids!!!!!!!!!
+// totally reflectric, reflectrified, etc. this is probably souper slow but
+// certainly no slower than the built in json encoder i'd imagine.
+// TODO: add support for struct tags: `tape:"0000"`, tape:"0001"` so they can get
+// transformed into tables with a defined schema
+
+// TODO: test all of these smaller functions individually
+
+import "fmt"
+import "reflect"
+
+var dummyMap map[uint16] any
+var dummyBuffer []byte
+
+// EncodeAny encodes an "any" value. Returns an error if the underlying type is
+// unsupported. Supported types are:
+//
+//   - int
+//   - int<N>
+//   - uint
+//   - uint<N>
+//   - string
+//   - []<supported type>
+//   - map[uint16]<supported type>
+func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
+	// primitives
+	reflectValue := reflect.ValueOf(value)
+	switch reflectValue.Kind() {
+	case reflect.Int:    return encoder.WriteInt32(int32(reflectValue.Int()))
+	case reflect.Uint:   return encoder.WriteUint32(uint32(reflectValue.Uint()))
+	case reflect.Int8:   return encoder.WriteInt8(int8(reflectValue.Int()))
+	case reflect.Uint8:  return encoder.WriteUint8(uint8(reflectValue.Uint()))
+	case reflect.Int16:  return encoder.WriteInt16(int16(reflectValue.Int()))
+	case reflect.Uint16: return encoder.WriteUint16(uint16(reflectValue.Uint()))
+	case reflect.Int32:  return encoder.WriteInt32(int32(reflectValue.Int()))
+	case reflect.Uint32: return encoder.WriteUint32(uint32(reflectValue.Uint()))
+	case reflect.Int64:  return encoder.WriteInt64(int64(reflectValue.Int()))
+	case reflect.Uint64: return encoder.WriteUint64(uint64(reflectValue.Uint()))
+	case reflect.String: return EncodeAny(encoder, []byte(reflectValue.String()), tag)
+	}
+	if reflectValue.CanConvert(reflect.TypeOf(dummyBuffer)) {
+		if tag.Is(LBA) {
+			nn, err := encoder.WriteUintN(uint64(reflectValue.Len()), tag.CN() + 1)
+			n += nn; if err != nil { return n, err }
+		}
+		nn, err := encoder.Write(reflectValue.Bytes())
+		n += nn; if err != nil { return n, err }
+		return n, nil
+	}
+
+	// aggregates
+	reflectType := reflect.TypeOf(value)
+	switch reflectType.Kind() {
+	case reflect.Slice:
+		return encodeAnySlice(encoder, value, tag)
+	// case reflect.Array:
+		// return encodeAnySlice(encoder, reflect.ValueOf(value).Slice(0, reflectType.Len()).Interface(), tag)
+	case reflect.Map:
+		if reflectType.Key() == reflect.TypeOf(uint16(0)) {
+			return encodeAnyMap(encoder, value, tag)
+		}
+		return n, fmt.Errorf("cannot encode map key %T, key must be uint16", value)
+	}
+	return n, fmt.Errorf("cannot encode type %T", value)
+}
+
+// DecodeAny decodes data and places it into destination, which must be a
+// pointer to a supported type. See [EncodeAny] for a list of supported types.
+func DecodeAny(decoder *Decoder, destination any, tag Tag) (n int, err error) {
+	reflectDestination := reflect.ValueOf(destination)
+	if reflectDestination.Kind() != reflect.Pointer {
+		return n, fmt.Errorf("expected pointer destination, not %v", destination)
+	}
+	return decodeAny(decoder, reflectDestination.Elem(), tag)
+}
+
+// unknownSlicePlaceholder is inserted by skeletonValue and informs the program
+// that the destination for the slice needs to be generated based on the item
+// tag in the OTA.
+type unknownSlicePlaceholder struct { }
+var unknownSlicePlaceholderType = reflect.TypeOf(unknownSlicePlaceholder { })
+
+// decodeAny is internal to [DecodeAny]. It takes in an addressable
+// [reflect.Value] as the destination.
+func decodeAny(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err error) {
+	errWrongDestinationType := func(expected string) error {
+		panic(fmt.Errorf(
+		// return fmt.Errorf(
+			"expected %s destination, not %v",
+			expected, destination))
+	}
+	
+	switch tag.WithoutCN() {
+	case SI:
+		// SI: (none)
+		err = setInt(destination, uint64(tag.CN()))
+		if err != nil { return n, err }
+	case LI:
+		// LI: <value: IntN>
+		nn, err := decodeAndSetInt(decoder, destination, tag.CN() + 1)
+		n += nn; if err != nil { return n, err }
+	case FP:
+		// FP: <value: FloatN>
+		nn, err := decodeAndSetFloat(decoder, destination, tag.CN() + 1)
+		n += nn; if err != nil { return n, err }
+	case SBA:
+		// SBA: <data: U8>*
+		buffer := make([]byte, tag.CN())
+		nn, err := decoder.Read(buffer)
+		n += nn; if err != nil { return n, err }
+		err = setByteArray(destination, buffer)
+		if err != nil { return n, err }
+	case LBA:
+		// LBA: <length: UN> <data: U8>*
+		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
+		n += nn; if err != nil { return n, err }
+		buffer := make([]byte, length)
+		nn, err = decoder.Read(buffer)
+		n += nn; if err != nil { return n, err }
+		err = setByteArray(destination, buffer)
+		if err != nil { return n, err }
+	case OTA:
+		// OTA: <length: UN> <elementTag: tape.Tag> <values>*
+		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
+		n += nn; if err != nil { return n, err }
+		oneTag, nn, err := decoder.ReadTag()
+		n += nn; if err != nil { return n, err }
+		if destination.Kind() != reflect.Slice {
+			return n, errWrongDestinationType("slice")
+		}
+		if destination.Cap() < int(length) {
+			destination.Grow(int(length) - destination.Cap())
+		}
+		destination.SetLen(int(length))
+		for index := range length {
+			nn, err := decodeAny(decoder, destination.Index(int(index)), oneTag)
+			n += nn; if err != nil { return n, err }
+		}
+	case KTV:
+		// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
+		table := destination
+		if table.Type() != reflect.TypeOf(dummyMap) {
+			return n, errWrongDestinationType("map[uint16] any")
+		}
+		length, nn, err := decoder.ReadUintN(tag.CN() + 1)
+		n += nn; if err != nil { return n, err }
+		table.Clear()
+		for _ = range length {
+			key, nn, err := decoder.ReadUint16()
+			n += nn; if err != nil { return n, err }
+			itemTag, nn, err := decoder.ReadTag()
+			n += nn; if err != nil { return n, err }
+			value, err := skeletonValue(decoder, itemTag)
+			if err != nil { return n, err }
+			nn, err = decodeAny(decoder, value.Elem(), itemTag)
+			n += nn; if err != nil { return n, err }
+			table.SetMapIndex(reflect.ValueOf(key), value.Elem())
+		}
+	default:
+		return n, fmt.Errorf("unknown TN %d", tag.TN())
+	}
+	return n, nil
+}
+
+// TagAny returns the correct tag for an "any" value. Returns an error if the
+// underlying type is unsupported. See [EncodeAny] for a list of supported
+// types.
+func TagAny(value any) (Tag, error) {
+	// TODO use reflection for all of this to ignore type names
+	// primitives
+	switch value := value.(type) {
+	case int,   uint:   return LI.WithCN(3), nil
+	case int8,  uint8:  return LI.WithCN(0), nil
+	case int16, uint16: return LI.WithCN(1), nil
+	case int32, uint32: return LI.WithCN(3), nil
+	case int64, uint64: return LI.WithCN(7), nil
+	case string:        return bufferLenTag(len(value)), nil
+	case []byte:        return bufferLenTag(len(value)), nil
+	}
+	
+	// aggregates
+	reflectType := reflect.TypeOf(value)
+	switch reflectType.Kind() {
+	case reflect.Slice: return OTA.WithCN(IntBytes(uint64(reflect.ValueOf(value).Len())) - 1), nil
+	case reflect.Array: return OTA.WithCN(reflectType.Len()), nil
+	case reflect.Map:
+		if reflectType.Key() == reflect.TypeOf(uint16(0)) {
+			return KTV.WithCN(IntBytes(uint64(reflect.ValueOf(value).Len())) - 1), nil
+		}
+		return 0, fmt.Errorf("cannot encode map key %T, key must be uint16", value)
+	}
+	return 0, fmt.Errorf("cannot get tag of type %T", value)
+}
+
+func encodeAnySlice(encoder *Encoder, value any, tag Tag) (n int, err error) {
+	// OTA: <length: UN> <elementTag: tape.Tag> <values>*
+	reflectValue := reflect.ValueOf(value)
+	nn, err := encoder.WriteUintN(uint64(reflectValue.Len()), tag.CN() + 1)
+	n += nn; if err != nil { return n, err }
+	reflectType := reflect.TypeOf(value)
+	oneTag, err := TagAny(reflect.Zero(reflectType.Elem()).Interface())
+	if err != nil { return n, err }
+	for index := 0; index < reflectValue.Len(); index += 1 {
+		item := reflectValue.Index(index).Interface()
+		itemTag, err := TagAny(item)
+		if err != nil { return n, err }
+		if itemTag.CN() > oneTag.CN() { oneTag = itemTag }
+	}
+	if oneTag.Is(SBA) { oneTag += 1 << 5 }
+	nn, err = encoder.WriteUint8(uint8(oneTag))
+	n += nn; if err != nil { return n, err }
+	for index := 0; index < reflectValue.Len(); index += 1 {
+		item := reflectValue.Index(index).Interface()
+		nn, err = EncodeAny(encoder, item, oneTag)
+		n += nn; if err != nil { return n, err }
+	}
+	return n, err
+}
+
+func encodeAnyMap(encoder *Encoder, value any, tag Tag) (n int, err error) {
+	// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
+	reflectValue := reflect.ValueOf(value)
+	nn, err := encoder.WriteUintN(uint64(reflectValue.Len()), tag.CN() + 1)
+	n += nn; if err != nil { return n, err }
+	iter := reflectValue.MapRange()
+	for iter.Next() {
+		key   := iter.Key().Interface().(uint16)
+		value := iter.Value().Interface()
+		nn, err = encoder.WriteUint16(key)
+		n += nn; if err != nil { return n, err }
+		itemTag, err := TagAny(value)
+		if err != nil { return n, err }
+		nn, err = encoder.WriteUint8(uint8(itemTag))
+		n += nn; if err != nil { return n, err }
+		nn, err = EncodeAny(encoder, value, itemTag)
+		n += nn; if err != nil { return n, err }
+	}
+	return n, nil
+}
+
+// setInt expects a settable destination.
+func setInt(destination reflect.Value, value uint64) error {
+	switch {
+	case destination.CanInt():
+		destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type()))
+	case destination.CanUint():
+		destination.Set(reflect.ValueOf(value).Convert(destination.Type()))
+	default:
+		return fmt.Errorf("cannot assign integer to %T", destination.Interface())
+	}
+	return nil
+}
+
+// setFloat expects a settable destination.
+func setFloat(destination reflect.Value, value float64) error {
+	if !destination.CanFloat() {
+		return fmt.Errorf("cannot assign float to %T", destination.Interface())
+	}
+	destination.Set(reflect.ValueOf(value).Convert(destination.Type()))
+	return nil
+}
+
+// setByteArrayexpects a settable destination.
+func setByteArray(destination reflect.Value, value []byte) error {
+	typ := destination.Type()
+	if typ.Kind() != reflect.Slice {
+		return fmt.Errorf("cannot assign %T to ", value)
+	}
+	if typ.Elem() != reflect.TypeOf(byte(0)) {
+		return fmt.Errorf("cannot convert %T to *[]byte", value)
+	}
+	destination.Set(reflect.ValueOf(value))
+	return nil
+}
+
+// decodeAndSetInt expects a settable destination.
+func decodeAndSetInt(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) {
+	value, nn, err := decoder.ReadUintN(bytes)
+	n += nn; if err != nil { return n, err }
+	return n, setInt(destination, value)
+}
+
+// decodeAndSetInt expects a settable destination.
+func decodeAndSetFloat(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) {
+	switch bytes {
+	case 8:
+		value, nn, err := decoder.ReadFloat64()
+		n += nn; if err != nil { return n, err }
+		return n, setFloat(destination, float64(value))
+	case 4:
+		value, nn, err := decoder.ReadFloat32()
+		n += nn; if err != nil { return n, err }
+		return n, setFloat(destination, float64(value))
+	}
+	return n, fmt.Errorf("cannot decode float%d", bytes * 8)
+}
+
+// skeletonValue returns a pointer value. In order for it to be set, it must be
+// dereferenced using Elem().
+func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) {
+	typ, err := typeOf(decoder, tag)
+	if err != nil { return reflect.Value { }, err }
+	return reflect.New(typ), nil
+}
+
+// typeOf returns the type of the current tag being decoded. It does not use up
+// the decoder, it only peeks.
+func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) {
+	switch tag.WithoutCN() {
+	case SI:
+		return reflect.TypeOf(uint8(0)), nil
+	case LI:
+		switch tag.CN() {
+		case 0: return reflect.TypeOf(uint8(0)), nil
+		case 1: return reflect.TypeOf(uint16(0)), nil
+		case 3: return reflect.TypeOf(uint32(0)), nil
+		case 7: return reflect.TypeOf(uint64(0)), nil
+		}
+		return nil, fmt.Errorf("unknown CN %d for LI", tag.CN())
+	case FP:
+		switch tag.CN() {
+		case 3: return reflect.TypeOf(float32(0)), nil
+		case 7: return reflect.TypeOf(float64(0)), nil
+		}
+		return nil, fmt.Errorf("unknown CN %d for FP", tag.CN())
+	case SBA: return reflect.SliceOf(reflect.TypeOf(byte(0))), nil
+	case LBA: return reflect.SliceOf(reflect.TypeOf(byte(0))), nil
+	case OTA:
+		elemTag, dimension, err := peekSlice(decoder, tag)
+		if err != nil { return nil, err }
+		if elemTag.Is(OTA) { panic("peekSlice cannot return OTA") }
+		typ, err := typeOf(decoder, elemTag)
+		if err != nil { return nil, err }
+		for _ = range dimension {
+			typ = reflect.SliceOf(typ)
+		}
+		return typ, nil
+	case KTV: return reflect.TypeOf(dummyMap), nil
+	}
+	return nil, fmt.Errorf("unknown TN %d", tag.TN())
+}
+
+// peekSlice returns the element tag and dimension count of the OTA currently
+// being decoded. It does not use up the decoder, it only peeks.
+func peekSlice(decoder *Decoder, tag Tag) (Tag, int, error) {
+	offset    := 0
+	dimension := 0
+	currentTag := tag
+	for {
+		elem, populated, n, err := peekSliceOnce(decoder, currentTag, offset)
+		if err != nil { return 0, 0, err }
+		currentTag = elem
+		offset     = n
+		dimension += 1
+		if elem.Is(OTA) {
+			if !populated {
+				// default to a large byte array, will be
+				// interpreted as a string.
+				return LBA, dimension + 1, nil
+			}
+		} else {
+			return elem, dimension, nil
+		}
+	}
+}
+
+// peekSliceOnce returns the element tag of the OTA located offset bytes ahead
+// of the current position. It does not use up the decoder, it only peeks. The n
+// return value denotes how far away from 0 it peeked. If the OTA has more than
+// zero items, populated will be set to true.
+func peekSliceOnce(decoder *Decoder, tag Tag, offset int) (elem Tag, populated bool, n int, err error) {
+	lengthStart  := offset
+	lengthEnd    := lengthStart + tag.CN() + 1
+	elemTagStart := lengthEnd
+	elemTagEnd   := elemTagStart + 1
+
+	headerBytes, err := decoder.Peek(elemTagEnd)
+	if err != nil { return 0, false, 0, err }
+
+	elem = Tag(headerBytes[len(headerBytes) - 1])
+	for index := lengthStart; index < lengthEnd; index += 1 {
+		if headerBytes[index] > 0 {
+			populated = true
+			break
+		}
+	}
+	n = elemTagEnd
+
+	return
+}

tape/dynamic_test.go

@@ -0,0 +1,195 @@
+package tape
+
+import "fmt"
+import "bytes"
+import "testing"
+import "reflect"
+import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil"
+
+func TestEncodeAnyInt(test *testing.T) {
+	err := testEncodeAny(test, uint8(0xCA), LI.WithCN(0), tu.S(0xCA))
+	if err != nil { test.Fatal(err) }
+	err = testEncodeAny(test, 400, LI.WithCN(3), tu.S(
+		0, 0, 0x1, 0x90,
+	))
+	if err != nil { test.Fatal(err) }
+}
+
+func TestEncodeAnyTable(test *testing.T) {
+	err := testEncodeAny(test, map[uint16] any {
+		0xF3B9: 1,
+		0x0102: 2,
+		0x0000: "hi!",
+		0xFFFF: []uint16 { 0xBEE5, 0x7777 },
+		0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} },
+	}, KTV.WithCN(0), tu.S(5).AddVar(
+		[]byte {
+			0xF3, 0xB9,
+			byte(LI.WithCN(3)),
+			0, 0, 0, 1,
+		},
+		[]byte {
+			0x01, 0x02,
+			byte(LI.WithCN(3)),
+			0, 0, 0, 2,
+		},
+		[]byte {
+			0, 0,
+			byte(SBA.WithCN(3)),
+			'h', 'i', '!',
+		},
+		[]byte {
+			0xFF, 0xFF,
+			byte(OTA.WithCN(0)), 2, byte(LI.WithCN(1)),
+			0xBE, 0xE5, 0x77, 0x77,
+		},
+		[]byte {
+			0x12, 0x34,
+			byte(OTA.WithCN(0)), 2, byte(OTA.WithCN(0)),
+			1, byte(LI.WithCN(1)),
+			0, 0x5,
+			2, byte(LI.WithCN(1)),
+			0,    0x17,
+			0xAA, 0xAA,
+		},
+	))
+	if err != nil { test.Fatal(err) }
+}
+
+func TestEncodeDecodeAnyTable(test *testing.T) {
+	err := testEncodeDecodeAny(test, map[uint16] any {
+		0xF3B9: uint32(1),
+		0x0102: uint32(2),
+		0x0000: []byte("hi!"),
+		0xFFFF: []uint16 { 0xBEE5, 0x7777 },
+		0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} },
+	}, nil)
+	if err != nil { test.Fatal(err) }
+}
+
+func TestPeekSlice(test *testing.T) {
+	buffer := bytes.NewBuffer([]byte {
+		2, byte(OTA.WithCN(3)),
+		0, 0, 0, 1, byte(LI.WithCN(1)),
+		0, 0x5,
+		2, byte(LI.WithCN(1)),
+		0,    0x17,
+		0xAA, 0xAA,
+	})
+	decoder := NewDecoder(buffer)
+
+	elem, dimension, err := peekSlice(decoder, OTA.WithCN(0))
+	if err != nil { test.Fatal(err) }
+	if elem != LI.WithCN(1) {
+		test.Fatalf("wrong element tag: %v %02X", elem, byte(elem))
+	}
+	if got, correct := dimension, 2; got != correct {
+		test.Fatalf("wrong dimension: %d != %d", got, correct)
+	}
+}
+
+func TestPeekSliceOnce(test *testing.T) {
+	buffer := bytes.NewBuffer([]byte {
+		2, byte(OTA.WithCN(3)),
+		0, 0, 0, 1, byte(LI.WithCN(1)),
+		0, 0x5,
+		2, byte(LI.WithCN(1)),
+		0,    0x17,
+		0xAA, 0xAA,
+	})
+	decoder := NewDecoder(buffer)
+
+	test.Log("--- stage 1")
+	elem, populated, n, err := peekSliceOnce(decoder, OTA.WithCN(0), 0)
+	if err != nil { test.Fatal(err) }
+	if elem != OTA.WithCN(3) {
+		test.Fatal("wrong element tag:", elem)
+	}
+	if !populated {
+		test.Fatal("wrong populated:", populated)
+	}
+	if got, correct := n, 2; got != correct {
+		test.Fatalf("wrong n: %d != %d", got, correct)
+	}
+	
+	test.Log("--- stage 2")
+	elem, populated, n, err = peekSliceOnce(decoder, elem, n)
+	if err != nil { test.Fatal(err) }
+	if elem != LI.WithCN(1) {
+		test.Fatal("wrong element tag:", elem)
+	}
+	if !populated {
+		test.Fatal("wrong populated:", populated)
+	}
+	if got, correct := n, 7; got != correct {
+		test.Fatalf("wrong n: %d != %d", got, correct)
+	}
+}
+
+func encAny(value any) ([]byte, Tag, int, error) {
+	tag, err := TagAny(value)
+	if err != nil { return nil, 0, 0, err }
+	buffer := bytes.Buffer { }
+	encoder := NewEncoder(&buffer)
+	n, err := EncodeAny(encoder, value, tag)
+	if err != nil { return nil, 0, n, err }
+	encoder.Flush()
+	return buffer.Bytes(), tag, n, nil
+}
+
+func decAny(data []byte) (Tag, any, int, error) {
+	destination := map[uint16] any { }
+	tag, err := TagAny(destination)
+	if err != nil { return 0, nil, 0, err }
+	n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data)), &destination, tag)
+	if err != nil { return 0, nil, n, err }
+	return tag, destination, n, nil
+}
+
+func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.Snake) error {
+	bytes, tag, n, err := encAny(value)
+	if err != nil { return err }
+	test.Log("n:      ", n)
+	test.Log("tag:    ", tag)
+	test.Log("got:    ", tu.HexBytes(bytes))
+	test.Log("correct:", correctBytes)
+	if tag != correctTag {
+		return fmt.Errorf("tag not equal")
+	}
+	if ok, n := correctBytes.Check(bytes); !ok {
+		return fmt.Errorf("bytes not equal: %d", n)
+	}
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	return nil
+}
+
+func testEncodeDecodeAny(test *testing.T, value, correctValue any) error {
+	if correctValue == nil { 
+		correctValue = value
+	}
+
+	test.Log("encoding...")
+	bytes, tag, n, err := encAny(value)
+	if err != nil { return err }
+	test.Log("n:  ", n)
+	test.Log("tag:", tag)
+	test.Log("got:", tu.HexBytes(bytes))
+	test.Log("decoding...", tag)
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	
+	_, decoded, n, err := decAny(bytes)
+	if err != nil { return err }
+	test.Log("got:    ", tu.Describe(decoded))
+	test.Log("correct:", tu.Describe(correctValue))
+	if !reflect.DeepEqual(decoded, correctValue) {
+		return fmt.Errorf("values not equal")
+	}
+	if n != len(bytes) {
+		return fmt.Errorf("n not equal: %d != %d", n, len(bytes))
+	}
+	return nil
+}

tape/encode.go

@@ -0,0 +1,189 @@
+package tape
+
+import "io"
+import "math"
+import "bufio"
+
+// Encodable is any type that can write itself to an encoder.
+type Encodable interface {
+	// Encode sends data to encoder. It returns the amount of bytes written,
+	// and an error if the write stopped early.
+	Encode(encoder *Encoder) (n int, err error)
+}
+	
+// Encoder encodes data to an io.Writer.
+type Encoder struct {
+	bufio.Writer
+}
+
+// NewEncoder creates a new encoder that writes to writer.
+func NewEncoder(writer io.Writer) *Encoder {
+	encoder := &Encoder { }
+	encoder.Reset(writer)
+	return encoder
+}
+
+// WriteInt8 encodes an 8-bit signed integer to the output writer.
+func (this *Encoder) WriteInt8(value int8) (n int, err error) {
+	return this.WriteUint8(uint8(value))
+}
+
+// WriteUint8 encodes an 8-bit unsigned integer to the output writer.
+func (this *Encoder) WriteUint8(value uint8) (n int, err error) {
+	return this.Write([]byte { byte(value) })
+}
+
+// WriteInt16 encodes an 16-bit signed integer to the output writer.
+func (this *Encoder) WriteInt16(value int16) (n int, err error) {
+	return this.WriteUint16(uint16(value))
+}
+
+// WriteUint16 encodes an 16-bit unsigned integer to the output writer.
+func (this *Encoder) WriteUint16(value uint16) (n int, err error) {
+	return this.Write([]byte {
+		byte(value >> 8),
+		byte(value),
+	})
+}
+
+// WriteInt32 encodes an 32-bit signed integer to the output writer.
+func (this *Encoder) WriteInt32(value int32) (n int, err error) {
+	return this.WriteUint32(uint32(value))
+}
+
+// WriteUint32 encodes an 32-bit unsigned integer to the output writer.
+func (this *Encoder) WriteUint32(value uint32) (n int, err error) {
+	return this.Write([]byte {
+		byte(value >> 24),
+		byte(value >> 16),
+		byte(value >> 8),
+		byte(value),
+	})
+}
+
+// WriteInt64 encodes an 64-bit signed integer to the output writer.
+func (this *Encoder) WriteInt64(value int64) (n int, err error) {
+	return this.WriteUint64(uint64(value))
+}
+
+// WriteUint64 encodes an 64-bit unsigned integer to the output writer.
+func (this *Encoder) WriteUint64(value uint64) (n int, err error) {
+	return this.Write([]byte {
+		byte(value >> 56),
+		byte(value >> 48),
+		byte(value >> 40),
+		byte(value >> 32),
+		byte(value >> 24),
+		byte(value >> 16),
+		byte(value >> 8),
+		byte(value),
+	})
+}
+
+// WriteIntN encodes an N-byte signed integer to the output writer.
+func (this *Encoder) WriteIntN(value int64, bytes int) (n int, err error) {
+	return this.WriteUintN(uint64(value), bytes)
+}
+
+// for Write/ReadUintN, increase buffers if go somehow gets support for over 64
+// bit integers. we could also make an expanding int type in goutil to use here,
+// or maybe there is one in the stdlib. keep the int64 versions as well though
+// because its ergonomic.
+	
+// WriteUintN encodes an N-byte unsigned integer to the output writer.
+func (this *Encoder) WriteUintN(value uint64, bytes int) (n int, err error) {
+	// TODO: don't make multiple write calls (without allocating)
+	buffer := [1]byte { }
+	for bytesLeft := bytes; bytesLeft > 0; bytesLeft -- {
+		buffer[0] = byte(value) >> ((bytesLeft - 1) * 8)
+		nn, err := this.Write(buffer[:])
+		n += nn; if err != nil { return n, err }
+	}
+	return n, nil
+}
+
+// WriteFloat16 encodes a 16-bit floating point value to the output writer.
+func (this *Encoder) WriteFloat16(value float32) (n int, err error) {
+	return this.WriteUint16(f32bitsToF16bits(math.Float32bits(value)))
+}
+
+// WriteFloat32 encodes a 32-bit floating point value to the output writer.
+func (this *Encoder) WriteFloat32(value float32) (n int, err error) {
+	return this.WriteUint32(math.Float32bits(value))
+}
+
+// WriteFloat64 encodes a 64-bit floating point value to the output writer.
+func (this *Encoder) WriteFloat64(value float64) (n int, err error) {
+	return this.WriteUint64(math.Float64bits(value))
+}
+
+// WriteTag encodes a [Tag] to the output writer.
+func (this *Encoder) WriteTag(value Tag) (n int, err error) {
+	return this.WriteUint8(uint8(value))
+}
+
+// f32bitsToF16bits returns uint16 (Float16 bits) converted from the specified float32.
+// Conversion rounds to nearest integer with ties to even.
+// Taken from https://github.com/x448/float16/blob/v0.8.4/float16
+//
+// MIT License
+// 
+// Copyright (c) 2019 Montgomery Edwards⁴⁴⁸ and Faye Amacker
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+func f32bitsToF16bits(u32 uint32) uint16 {
+	// Translated from Rust to Go by Montgomery Edwards⁴⁴⁸ (github.com/x448).
+	// All 4294967296 conversions with this were confirmed to be correct by x448.
+	// Original Rust implementation is by Kathryn Long (github.com/starkat99) with MIT license.
+
+	sign := u32 & 0x80000000
+	exp := u32 & 0x7f800000
+	coef := u32 & 0x007fffff
+
+	if exp == 0x7f800000 {
+		// NaN or Infinity
+		nanBit := uint32(0)
+		if coef != 0 {
+			nanBit = uint32(0x0200)
+		}
+		return uint16((sign >> 16) | uint32(0x7c00) | nanBit | (coef >> 13))
+	}
+
+	halfSign := sign >> 16
+
+	unbiasedExp := int32(exp>>23) - 127
+	halfExp := unbiasedExp + 15
+
+	if halfExp >= 0x1f {
+		return uint16(halfSign | uint32(0x7c00))
+	}
+
+	if halfExp <= 0 {
+		if 14-halfExp > 24 {
+			return uint16(halfSign)
+		}
+		coef := coef | uint32(0x00800000)
+		halfCoef := coef >> uint32(14-halfExp)
+		roundBit := uint32(1) << uint32(13-halfExp)
+		if (coef&roundBit) != 0 && (coef&(3*roundBit-1)) != 0 {
+			halfCoef++
+		}
+		return uint16(halfSign | halfCoef)
+	}
+
+	uHalfExp := uint32(halfExp) << 10
+	halfCoef := coef >> 13
+	roundBit := uint32(0x00001000)
+	if (coef&roundBit) != 0 && (coef&(3*roundBit-1)) != 0 {
+		return uint16((halfSign | uHalfExp | halfCoef) + 1)
+	}
+	return uint16(halfSign | uHalfExp | halfCoef)
+}

tape/measure.go

@@ -0,0 +1,12 @@
+package tape
+
+// IntBytes returns the number of bytes required to hold a given unsigned
+// integer.
+func IntBytes(value uint64) int {
+	bytes := 0
+	for value > 0 || bytes == 0 {
+		value >>= 8;
+		bytes ++
+	}
+	return bytes
+}

tape/measure_test.go

@@ -0,0 +1,21 @@
+package tape
+
+import "testing"
+
+func TestIntBytes(test *testing.T) {
+	if correct, got := 1, IntBytes(0); correct != got {
+		test.Fatal("wrong:", got)
+	}
+	if correct, got := 1, IntBytes(1); correct != got {
+		test.Fatal("wrong:", got)
+	}
+	if correct, got := 1, IntBytes(16); correct != got {
+		test.Fatal("wrong:", got)
+	}
+	if correct, got := 1, IntBytes(255); correct != got {
+		test.Fatal("wrong:", got)
+	}
+	if correct, got := 2, IntBytes(256); correct != got {
+		test.Fatal("wrong:", got)
+	}
+}

tape/tag.go

@@ -0,0 +1,68 @@
+package tape
+
+import "fmt"
+
+type Tag byte; const (
+	SI      Tag = 0 << 5 // Small integer
+	LI      Tag = 1 << 5 // Large integer
+	FP      Tag = 2 << 5 // Floating point
+	SBA     Tag = 3 << 5 // Small byte array
+	LBA     Tag = 4 << 5 // Large byte array
+	OTA     Tag = 5 << 5 // One-tag array
+	KTV     Tag = 6 << 5 // Key-tag-value table
+	TNMask  Tag = 0xE0   // The entire TN bitfield
+	CNMask  Tag = 0x1F   // The entire CN bitfield
+	CNLimit Tag = 32     // All valid CNs are < CNLimit
+)
+
+func (tag Tag) TN() int {
+	return int(tag >> 5)
+}
+
+func (tag Tag) CN() int {
+	return int(tag & CNMask)
+}
+
+func (tag Tag) WithCN(cn int) Tag {
+	return (tag & TNMask) | Tag(cn % 32)
+}
+
+func (tag Tag) WithoutCN() Tag {
+	return tag.WithCN(0)
+}
+
+func (tag Tag) Is(other Tag) bool {
+	return tag.TN() == other.TN()
+}
+
+func (tag Tag) String() string {
+	tn := fmt.Sprint(tag.TN())
+	switch tag.WithoutCN() {
+	case SI:  tn = "SI"
+	case LI:  tn = "LI"
+	case FP:  tn = "FP"
+	case SBA: tn = "SBA"
+	case LBA: tn = "LBA"
+	case OTA: tn = "OTA"
+	case KTV: tn = "KTV"
+	}
+	return fmt.Sprintf("%s:%d", tn, tag.CN())
+}
+
+// BufferTag returns the appropriate tag for a buffer.
+func BufferTag(value []byte) Tag {
+	return bufferLenTag(len(value))
+}
+
+// StringTag returns the appropriate tag for a string.
+func StringTag(value string) Tag {
+	return bufferLenTag(len(value))
+}
+
+func bufferLenTag(length int) Tag {
+	if length < int(CNLimit) {
+		return SBA.WithCN(length)
+	} else {
+		return LBA.WithCN(IntBytes(uint64(length)))
+	}
+}