tape: Correctly decode into a table destination all the time

generate/generate.go

@@ -24,7 +24,7 @@ const preamble = `
 
 const static = `
 // Table is a KTV table with an undefined schema.
-type Table map[uint16] any
+type Table = map[uint16] any
 
 // Message is any message that can be sent along this protocol.
 type Message interface {
@@ -372,7 +372,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		nn, err = this.iprintf("}\n")
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf(
-			"nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN())\n",
+			"nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN() + 1)\n",
 			valueSource, tagSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -407,8 +407,8 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 			n += nn; if err != nil { return n, err }
 			nn, err = this.iprintf("nn, err = encoder.WriteTag(itemTag)\n")
 			n += nn; if err != nil { return n, err }
-			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
+			n += nn; if err != nil { return n, err }
 			nn, err = this.iprintf("for _, item := range %s {\n", valueSource)
 			n += nn; if err != nil { return n, err }
 			this.push()
@@ -445,7 +445,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri
 		nn, err = this.iprintf("}\n")
 		n += nn; if err != nil { return n, err }
 		nn, err = this.iprintf(
-			"nn, err = encoder.WriteUintN(%d, %s.CN())\n",
+			"nn, err = encoder.WriteUintN(%d, %s.CN() + 1)\n",
 			len(typ.Fields), tagSource)
 		n += nn; if err != nil { return n, err }
 		nn, err = this.generateErrorCheck()
@@ -709,7 +709,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar)
+			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()) + 1)\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
 			n += nn; if err != nil { return n, err }
@@ -785,7 +785,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st
 			this.pop()
 			nn, err = this.iprintf("}\n")
 			n += nn; if err != nil { return n, err }
-			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar)
+			nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()) + 1)\n", lengthVar)
 			n += nn; if err != nil { return n, err }
 			nn, err = this.generateErrorCheck()
 			n += nn; if err != nil { return n, err }
@@ -952,13 +952,13 @@ func (this *Generator) generateTag(typ Type, source string) (tagVar string, n in
 		nn, err := this.iprintf("%s := tape.BufferTag([]byte(%s))\n", tagVar, source)
 		n += nn; if err != nil { return tagVar, n, err }
 	case TypeArray:
-		nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source)
+		nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))) - 1)\n", tagVar, source)
 		n += nn; if err != nil { return tagVar, n, err }
 	case TypeTable:
-		nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source)
+		nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))) - 1)\n", tagVar, source)
 		n += nn; if err != nil { return tagVar, n, err }
 	case TypeTableDefined:
-		nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields))))
+		nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields))) - 1)
 		n += nn; if err != nil { return tagVar, n, err }
 	case TypeNamed:
 		resolved, err := this.resolveTypeName(typ.Name)

generate/generate_test.go

@@ -103,7 +103,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageConnect,
-			tu.S(0xE1, 0x02).AddVar(
+			tu.S(0xE0, 0x02).AddVar(
 				[]byte { 0x00, 0x00, 0x86, 'r', 'a', 'r', 'i', 't', 'y' },
 				[]byte { 0x00, 0x01, 0x84, 'g', 'e', 'm', 's' },
 				))
@@ -129,8 +129,8 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageUserList,
-			tu.S(0xE1, 0x01, 0x00, 0x00,
-				0xC1, 0x03, 0xE1,
+			tu.S(0xE0, 0x01, 0x00, 0x00,
+				0xC0, 0x03, 0xE0,
 					).Add(0x03).AddVar(
 						[]byte { 0x00, 0x00, 0x86, 'r', 'a', 'r', 'i', 't', 'y' },
 						[]byte { 0x00, 0x01, 0x87, 'a', 's', 'd', 'j', 'a', 'd', 's' },
@@ -155,7 +155,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messagePulse,
-			tu.S(0xE1, 0x05).AddVar(
+			tu.S(0xE0, 0x05).AddVar(
 				[]byte { 0x00, 0x00, 0x09 },
 				[]byte { 0x00, 0x01, 0x41, 0xCA, 0xDF },
 				[]byte { 0x00, 0x02, 0x61, 0x51, 0xAC },
@@ -176,7 +176,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageNestedArray,
-			tu.S(0xC1, 0x02, 0xC1,
+			tu.S(0xC0, 0x02, 0xC0,
 				0x06, 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6,
 				35,   0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6,
 				            0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC,
@@ -202,7 +202,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageIntegers,
-			tu.S(0xE1, 13).AddVar(
+			tu.S(0xE0, 13).AddVar(
 				[]byte { 0x00, 0x00, 0x13 },
 				[]byte { 0x00, 0x01, 0x20, 0xC9 },
 				[]byte { 0x00, 0x02, 0x21, 0x34, 0xC9 },
@@ -243,7 +243,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 		}
 		testEncodeDecode(
 			&messageDynamic,
-			tu.S(0xE1, 14).AddVar(
+			tu.S(0xE0, 14).AddVar(
 				[]byte { 0x00, 0x00, 0x20, 0x23 },
 				[]byte { 0x00, 0x01, 0x21, 0x32, 0x47 },
 				[]byte { 0x00, 0x02, 0x23, 0x87, 0x32, 0x45, 0x23 },
@@ -255,11 +255,15 @@ func TestGenerateRunEncodeDecode(test *testing.T) {
 				[]byte { 0x00, 0x08, 0x63, 0x45, 0x12, 0x63, 0xCE },
 				[]byte { 0x00, 0x09, 0x67, 0x40, 0x74, 0x4E, 0x3D, 0x6F, 0xCD, 0x17, 0x75 },
 				[]byte { 0x00, 0x0A, 0x87, 'f', 'o', 'x', ' ', 'b', 'e', 'd' },
-				[]byte { 0x00, 0x0B, 0xC4, 0x00, 0x07, 0x00, 0x06, 0x00, 0x05, 0x00, 0x04 },
-				[]byte { 0x00, 0x0C, 0xE1, 0x02,
-					0x00, 0x01, 0x20, 0x08,
+				[]byte { 0x00, 0x0B, 0xC0, 0x04, 0x41,
+					0x00, 0x07,
+					0x00, 0x06,
+					0x00, 0x05,
+					0x00, 0x04 },
+				[]byte { 0x00, 0x0C, 0xE0, 0x02,
+					0x00, 0x01, 0x40, 0x08,
 					0x00, 0x02, 0x67, 0x40, 0x11, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A },
-				[]byte { 0x00, 0x0D, 0xE1, 0x03,
+				[]byte { 0x00, 0x0D, 0xE0, 0x03, // ERR
 					0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00,
 					0x00, 0x02, 0x82, 'h', 'i',
 					0x00, 0x03, 0x21, 0x39, 0x92 },

generate/misc_test.go

@@ -100,12 +100,12 @@ func testGenerateRun(test *testing.T, protocol *Protocol, title, imports, testCa
 			log.Println("decoding:")
 			destination := reflect.New(reflect.ValueOf(message).Elem().Type()).Interface().(Message)
 			flat := data.Flatten()
-			log.Println("before: ", destination)
+			log.Println("before: ", tu.Describe(destination))
 			decoder := tape.NewDecoder(bytes.NewBuffer(flat))
 			n, err = destination.Decode(decoder)
 			if err != nil { log.Fatalf("at %d: %v\n", n, err) }
-			log.Println("got:    ", destination)
-			log.Println("correct:", message)
+			log.Println("got:    ", tu.Describe(destination))
+			log.Println("correct:", tu.Describe(message))
 			if n != len(flat) {
 				log.Fatalf("n incorrect: %d != %d\n", n, len(flat))
 			}

internal/testutil/testutil.go

@@ -2,8 +2,9 @@ package testutil
 
 import "fmt"
 import "slices"
-import "strings"
 import "reflect"
+import "strings"
+import "unicode"
 
 // Snake lets you compare blocks of data where the ordering of certain parts may
 // be swapped every which way. It is designed for comparing the encoding of
@@ -124,6 +125,10 @@ func (this *describer) describe(value reflect.Value) {
 		return
 	}
 	value = reflect.ValueOf(value.Interface())
+	if !value.IsValid() {
+		this.printf("<invalid>")
+		return
+	}
 	switch value.Kind() {
 	case reflect.Array, reflect.Slice:
 		this.printf("[\n")
@@ -141,12 +146,20 @@ func (this *describer) describe(value reflect.Value) {
 		typ := value.Type()
 		for index := range typ.NumField() {
 			indexBuffer := [1]int { index }
-			this.iprintf("%s: ", typ.Field(index).Name)
+			field := typ.Field(index)
+			this.iprintf("%s: ", field.Name)
+			for _, char := range field.Name {
+				if unicode.IsUpper(char) {
 					this.describe(value.FieldByIndex(indexBuffer[:]))
+				} else {
+					this.printf("<private>")
+				}
+				break
+			}
 			this.iprintf("\n")
 		}
 		this.indent -= 1
-		this.iprintf("}\n")
+		this.iprintf("}")
 	case reflect.Map:
 		this.printf("map {\n")
 		this.indent += 1
@@ -159,7 +172,7 @@ func (this *describer) describe(value reflect.Value) {
 			this.iprintf("\n")
 		}
 		this.indent -= 1
-		this.iprintf("}\n")
+		this.iprintf("}")
 	case reflect.Pointer:
 		this.printf("& ")
 		this.describe(value.Elem())

tape/dynamic.go

@@ -116,9 +116,9 @@ func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error
 func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) {
 	destination, err := skeletonPointer(decoder, tag)
 	if err != nil { return nil, n, err }
-	nn, err := DecodeAnyInto(decoder, destination, tag)
+	nn, err := decodeAny(decoder, destination.Elem(), tag)
 	n += nn; if err != nil { return nil, n, err }
-	return destination, n, err
+	return destination.Elem().Interface(), n, err
 }
 
 // unknownSlicePlaceholder is inserted by skeletonValue and informs the program
@@ -242,12 +242,22 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i
 		}
 		lengthCast, err := Uint64ToIntSafe(length)
 		if err != nil { return n, err }
-		if isTypeAny(destination.Type()) {
-			// need a skeleton value if we are assigning to any.
+
+		// im fucking so done dude. im so fucking done. this was
+		// supposed to only run when we need it but i guess it runs all
+		// the time, because when we get a map destination (a valid,
+		// allocated one) we break apart on SetMapIndex because of a nil
+		// map. yeah thats right. a fucking nil map panic. on the map we
+		// just allocated. but running this unconditionally (whether or
+		// not we receive an empty any value) actually makes it fucking
+		// work. go figure().
+		//
+		// (the map allocation functionality in skeletonPointer has been
+		// removed)
 		value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast)
 		destination.Set(value)
 		destination = value
-		}
+		
 		destination.Clear()
 		for _ = range lengthCast {
 			key, nn, err := decoder.ReadUint16()
@@ -387,7 +397,11 @@ func canSet(destination reflect.Type, tag Tag) error {
 			return errCantAssignf("cannot assign array to %v", destination)
 		}
 	case KTV:
-		if destination != reflect.TypeOf(dummyMap) {
+		cantAssign :=
+			destination.Kind() != reflect.Map ||
+			destination.Key().Kind() != reflect.Uint16 ||
+			!isTypeAny(destination.Elem())
+		if cantAssign {
 			return errCantAssignf("cannot assign table to %v", destination)
 		}
 	default:
@@ -496,7 +510,8 @@ func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) {
 func skeletonPointer(decoder *Decoder, tag Tag) (reflect.Value, error) {
 	typ, err := typeOf(decoder, tag)
 	if err != nil { return reflect.Value { }, err }
-	return reflect.New(typ), nil
+	value := reflect.New(typ)
+	return value, nil
 }
 
 // typeOf returns the type of the current tag being decoded. It does not use up

tape/dynamic_test.go

@@ -27,6 +27,12 @@ var samplePayloads = [][]byte {
 		0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67,
 		0x02, 0x24, byte(LI.WithCN(3)),  0x45, 0x67, 0x89, 0xAB,
 	},
+	/* map[uint16] any */ []byte {
+		byte(KTV.WithCN(0)), 3,
+		0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00,
+		0x00, 0x02, 0x82, 'h', 'i',
+		0x00, 0x03, 0x21, 0x39, 0x92,
+	},
 }
 
 var sampleValues = []any {
@@ -49,6 +55,11 @@ var sampleValues = []any {
 		0x0223: int16(0x4567),
 		0x0224: uint32(0x456789AB),
 	},
+	/* map[uint16] any */ map[uint16] any {
+		0x0001: float32(489.5),
+		0x0002: "hi",
+		0x0003: uint16(0x3992),
+	},
 }
 
 type userDefinedInteger int16
@@ -195,7 +206,7 @@ func TestDecodeWrongType(test *testing.T) {
 			{ var dest []string; arrayCase(&dest) }
 		}
 		// tables should only assign to other tables
-		if index != 12 {
+		if index != 12 && index != 13 {
 			test.Log("- map[uint16] any")
 			{ var dest = map[uint16] any { }; arrayCase(&dest) }
 		}
@@ -296,3 +307,63 @@ func TestPeekSliceOnce(test *testing.T) {
 		test.Fatalf("wrong n: %d != %d", got, correct)
 	}
 }
+
+func TestTagAny(test *testing.T) {
+	cases := [][2]any {
+		[2]any { LSI.WithCN(3),  int(9)     },
+		[2]any { LSI.WithCN(0),  int8(9)    },
+		[2]any { LSI.WithCN(1),  int16(9)   },
+		[2]any { LSI.WithCN(3),  int32(9)   },
+		[2]any { LSI.WithCN(7),  int64(9)   },
+		[2]any { LI.WithCN(3),   uint(9)    },
+		[2]any { LI.WithCN(0),   uint8(9)   },
+		[2]any { LI.WithCN(1),   uint16(9)  },
+		[2]any { LI.WithCN(3),   uint32(9)  },
+		[2]any { LI.WithCN(7),   uint64(9)  },
+		[2]any { FP.WithCN(3),   float32(9) },
+		[2]any { FP.WithCN(7),   float64(9) },
+		[2]any { SBA.WithCN(12), "small string" },
+		[2]any { SBA.WithCN(12), []byte("small string") },
+		[2]any { LBA.WithCN(0),  "this is a very long string that is long" },
+		[2]any { LBA.WithCN(0),  []byte("this is a very long string that is long") },
+		[2]any { LBA.WithCN(1),  lipsum },
+		[2]any { OTA.WithCN(0),  []int { 1, 2, 3, 4, 5 } },
+		[2]any { OTA.WithCN(0),  []string { "1, 2, 3, 4, 5" } },
+		[2]any { KTV.WithCN(0),  map[uint16] any {
+			0:  1,
+			1:  "wow",
+			2:  10.238,
+			45: -9,
+			9:  map[uint16] any { },
+		}},
+	}
+	for _, cas := range cases {
+		test.Log(cas)
+		got, err := TagAny(cas[1])
+		if err != nil { test.Fatal(err) }
+		if correct := cas[0].(Tag); correct != got {
+			test.Fatalf("wrong tag: %v != %v", got, correct)
+		}
+	}
+}
+
+func TestDecodeAny(test *testing.T) {
+	for index, payload := range samplePayloads {
+		correctValue := sampleValues[index]
+		data := payload[1:]
+		decoder := NewDecoder(bytes.NewBuffer(data))
+		tag     := Tag(payload[0])
+		decoded, n, err := DecodeAny(decoder, tag)
+		test.Log("n:      ", n)
+		test.Log("tag:    ", tag)
+		test.Log("got:    ", tu.Describe(decoded))
+		test.Log("correct:", tu.Describe(correctValue))
+		if err != nil { test.Fatal(err) }
+		if !reflect.DeepEqual(decoded, correctValue) {
+			test.Fatal("values not equal")
+		}
+		if n != len(data) {
+			test.Fatalf("n not equal: %d != %d", n, len(data))
+		}
+	}
+}

tape/strings.go

@@ -0,0 +1,11 @@
+package tape
+
+const lipsum = `Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.
+
+Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.`

tape/tag.go

@@ -18,6 +18,17 @@ type Tag byte; const (
 	CNLimit Tag = 32     // All valid CNs are < CNLimit
 )
 
+// what the first nybble of a tag means:
+//
+//   0-1 : SI 
+//   2-3 : LI 
+//   4-5 : LSI
+//   6-7 : FP 
+//   8-9 : SBA
+//   A-B : LBA
+//   C-D : OTA
+//   E-F : KTV
+
 func (tag Tag) TN() int {
 	return int(tag >> 5)
 }
@@ -67,6 +78,6 @@ func bufferLenTag(length int) Tag {
 	if length < int(CNLimit) {
 		return SBA.WithCN(length)
 	} else {
-		return LBA.WithCN(IntBytes(uint64(length)))
+		return LBA.WithCN(IntBytes(uint64(length)) - 1)
 	}
 }