From 5778616965aff9bffa41223c3728a76081300eac Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 13:11:47 -0400 Subject: [PATCH 01/16] design: Codify usage of CN + 1 bytes everywhere --- design/protocol.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/design/protocol.md b/design/protocol.md index f1a107e..da3444b 100644 --- a/design/protocol.md +++ b/design/protocol.md @@ -75,16 +75,16 @@ connection. Thus, the value may range from 0 to 31 if unsigned, and from -16 to 17 if signed. #### Large Integer (LI) -LI encodes an integer of up to 256 bits, which are stored in the payload. The CN -determine the length of the payload in bytes. The integer is big-endian. Whether +LI encodes an integer of up to 256 bits, which are stored in the payload. The +length of the payload (in bytes) is CN + 1. The integer is big-endian. Whether the payload is interpreted as unsigned or as signed two's complement is semantic information and must be agreed upon by both sides of the connection. Thus, the value may range from 0 to 31 if unsigned, and from -16 to 17 if signed. #### Floating Point (FP) FP encodes an IEEE 754 floating point number of up to 256 bits, which are stored -in the payload. The CN determines the length of the payload in bytes, and it may -only be one of these values: 16, 32, 64, 128, or 256. +in the payload. The length of the payload (in bytes) is CN + 1. The only +supported bit widths for floats are as follows: 16, 32, 64, 128, and 256. #### Small Byte Array (SBA) SBA encodes an array of up to 32 bytes, which are stored in the paylod. The @@ -98,15 +98,16 @@ in bytes is determined by the CN. #### One-Tag Array (OTA) OTA encodes an array of up to 2^256 items, which are stored in the payload after the length field and the item tag, where the length field comes first. Each item -must be the same length, as they all share the same tag. The length of the data -length field in bytes is determined by the CN. +must be the same length, as they all share the same tag. The length of the +length field (in bytes) is CN + 1. #### Key-Tag-Value Table (KTV) KTV encodes a table of up to 2^256 key/value pairs, which are stored in the payload after the length field. The pairs themselves consist of a 16-bit unsigned big-endian key followed by a tag and then the payload. Pair values can be of different types and sizes. The order of the pairs is not significant and -should never be treated as such. +should never be treated as such. The length of the length field (in bytes) is +CN + 1. ## Transports A transport is a protocol that HOPP connections can run on top of. HOPP -- 2.51.0 From 405b4587026e8904b588f43adaa718856ca25c88 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 13:37:50 -0400 Subject: [PATCH 02/16] tape: Comment chart for reading tags from hexdumps --- tape/tag.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tape/tag.go b/tape/tag.go index f4001f2..103f01b 100644 --- a/tape/tag.go +++ b/tape/tag.go @@ -18,6 +18,17 @@ type Tag byte; const ( CNLimit Tag = 32 // All valid CNs are < CNLimit ) +// what the first nybble of a tag means: +// +// 0-1 : SI +// 2-3 : LI +// 4-5 : LSI +// 6-7 : FP +// 8-9 : SBA +// A-B : LBA +// C-D : OTA +// E-F : KTV + func (tag Tag) TN() int { return int(tag >> 5) } -- 2.51.0 From b44d364f0f039ea6f21368313020e746b5e454e2 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 13:38:33 -0400 Subject: [PATCH 03/16] tape: Test TagAny function --- tape/dynamic_test.go | 39 +++++++++++++++++++++++++++++++++++++++ tape/strings.go | 11 +++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tape/strings.go diff --git a/tape/dynamic_test.go b/tape/dynamic_test.go index a068563..b13ba17 100644 --- a/tape/dynamic_test.go +++ b/tape/dynamic_test.go @@ -296,3 +296,42 @@ func TestPeekSliceOnce(test *testing.T) { test.Fatalf("wrong n: %d != %d", got, correct) } } + +func TestTagAny(test *testing.T) { + cases := [][2]any { + [2]any { LSI.WithCN(3), int(9) }, + [2]any { LSI.WithCN(0), int8(9) }, + [2]any { LSI.WithCN(1), int16(9) }, + [2]any { LSI.WithCN(3), int32(9) }, + [2]any { LSI.WithCN(7), int64(9) }, + [2]any { LI.WithCN(3), uint(9) }, + [2]any { LI.WithCN(0), uint8(9) }, + [2]any { LI.WithCN(1), uint16(9) }, + [2]any { LI.WithCN(3), uint32(9) }, + [2]any { LI.WithCN(7), uint64(9) }, + [2]any { FP.WithCN(3), float32(9) }, + [2]any { FP.WithCN(7), float64(9) }, + [2]any { SBA.WithCN(12), "small string" }, + [2]any { SBA.WithCN(12), []byte("small string") }, + [2]any { LBA.WithCN(0), "this is a very long string that is long" }, + [2]any { LBA.WithCN(0), []byte("this is a very long string that is long") }, + [2]any { LBA.WithCN(1), lipsum }, + [2]any { OTA.WithCN(0), []int { 1, 2, 3, 4, 5 } }, + [2]any { OTA.WithCN(0), []string { "1, 2, 3, 4, 5" } }, + [2]any { KTV.WithCN(0), map[uint16] any { + 0: 1, + 1: "wow", + 2: 10.238, + 45: -9, + 9: map[uint16] any { }, + }}, + } + for _, cas := range cases { + test.Log(cas) + got, err := TagAny(cas[1]) + if err != nil { test.Fatal(err) } + if correct := cas[0].(Tag); correct != got { + test.Fatalf("wrong tag: %v != %v", got, correct) + } + } +} diff --git a/tape/strings.go b/tape/strings.go new file mode 100644 index 0000000..9b2085b --- /dev/null +++ b/tape/strings.go @@ -0,0 +1,11 @@ +package tape + +const lipsum = `Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos. + +Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos. + +Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos. + +Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos. + +Lorem ipsum dolor sit amet consectetur adipiscing elit. Quisque faucibus ex sapien vitae pellentesque sem placerat. In id cursus mi pretium tellus duis convallis. Tempus leo eu aenean sed diam urna tempor. Pulvinar vivamus fringilla lacus nec metus bibendum egestas. Iaculis massa nisl malesuada lacinia integer nunc posuere. Ut hendrerit semper vel class aptent taciti sociosqu. Ad litora torquent per conubia nostra inceptos himenaeos.` -- 2.51.0 From 813d21958075333ead2034f6023522bec685f8de Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 13:39:37 -0400 Subject: [PATCH 04/16] tape: Fix bufferLenTag off by one error --- tape/tag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tape/tag.go b/tape/tag.go index 103f01b..19683e8 100644 --- a/tape/tag.go +++ b/tape/tag.go @@ -78,6 +78,6 @@ func bufferLenTag(length int) Tag { if length < int(CNLimit) { return SBA.WithCN(length) } else { - return LBA.WithCN(IntBytes(uint64(length))) + return LBA.WithCN(IntBytes(uint64(length)) - 1) } } -- 2.51.0 From c185f5058f78886afe4bee08b74d52d89a13b2e1 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 13:43:19 -0400 Subject: [PATCH 05/16] generate: Fix some off by one errors in TestGenerateRunEncodeDecode --- generate/generate_test.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/generate/generate_test.go b/generate/generate_test.go index 1cc7801..fb9d519 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -255,11 +255,15 @@ func TestGenerateRunEncodeDecode(test *testing.T) { []byte { 0x00, 0x08, 0x63, 0x45, 0x12, 0x63, 0xCE }, []byte { 0x00, 0x09, 0x67, 0x40, 0x74, 0x4E, 0x3D, 0x6F, 0xCD, 0x17, 0x75 }, []byte { 0x00, 0x0A, 0x87, 'f', 'o', 'x', ' ', 'b', 'e', 'd' }, - []byte { 0x00, 0x0B, 0xC4, 0x00, 0x07, 0x00, 0x06, 0x00, 0x05, 0x00, 0x04 }, - []byte { 0x00, 0x0C, 0xE1, 0x02, + []byte { 0x00, 0x0B, 0xC0, 0x04, + 0x00, 0x07, + 0x00, 0x06, + 0x00, 0x05, + 0x00, 0x04 }, + []byte { 0x00, 0x0C, 0xE0, 0x02, // ERR 0x00, 0x01, 0x20, 0x08, 0x00, 0x02, 0x67, 0x40, 0x11, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A }, - []byte { 0x00, 0x0D, 0xE1, 0x03, + []byte { 0x00, 0x0D, 0xE0, 0x03, // ERR 0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00, 0x00, 0x02, 0x82, 'h', 'i', 0x00, 0x03, 0x21, 0x39, 0x92 }, -- 2.51.0 From f6b12d43fbda3527ac2c31e4e811ff1eed989f2b Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 17:11:44 -0400 Subject: [PATCH 06/16] generate: Fix off by one errors in generated code and tests --- generate/generate.go | 16 ++++++++-------- generate/generate_test.go | 20 ++++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 520171f..66fb192 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -372,7 +372,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } nn, err = this.iprintf( - "nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN())\n", + "nn, err = encoder.WriteUintN(uint64(len(%s)), %s.CN() + 1)\n", valueSource, tagSource) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() @@ -407,8 +407,8 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri n += nn; if err != nil { return n, err } nn, err = this.iprintf("nn, err = encoder.WriteTag(itemTag)\n") n += nn; if err != nil { return n, err } - n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } nn, err = this.iprintf("for _, item := range %s {\n", valueSource) n += nn; if err != nil { return n, err } this.push() @@ -445,7 +445,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } nn, err = this.iprintf( - "nn, err = encoder.WriteUintN(%d, %s.CN())\n", + "nn, err = encoder.WriteUintN(%d, %s.CN() + 1)\n", len(typ.Fields), tagSource) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() @@ -709,7 +709,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st this.pop() nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } - nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar) + nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()) + 1)\n", lengthVar) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } @@ -785,7 +785,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st this.pop() nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } - nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar) + nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()) + 1)\n", lengthVar) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } @@ -952,13 +952,13 @@ func (this *Generator) generateTag(typ Type, source string) (tagVar string, n in nn, err := this.iprintf("%s := tape.BufferTag([]byte(%s))\n", tagVar, source) n += nn; if err != nil { return tagVar, n, err } case TypeArray: - nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source) + nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))) - 1)\n", tagVar, source) n += nn; if err != nil { return tagVar, n, err } case TypeTable: - nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source) + nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))) - 1)\n", tagVar, source) n += nn; if err != nil { return tagVar, n, err } case TypeTableDefined: - nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields)))) + nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields))) - 1) n += nn; if err != nil { return tagVar, n, err } case TypeNamed: resolved, err := this.resolveTypeName(typ.Name) diff --git a/generate/generate_test.go b/generate/generate_test.go index fb9d519..bcf1b77 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -103,7 +103,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) { } testEncodeDecode( &messageConnect, - tu.S(0xE1, 0x02).AddVar( + tu.S(0xE0, 0x02).AddVar( []byte { 0x00, 0x00, 0x86, 'r', 'a', 'r', 'i', 't', 'y' }, []byte { 0x00, 0x01, 0x84, 'g', 'e', 'm', 's' }, )) @@ -129,8 +129,8 @@ func TestGenerateRunEncodeDecode(test *testing.T) { } testEncodeDecode( &messageUserList, - tu.S(0xE1, 0x01, 0x00, 0x00, - 0xC1, 0x03, 0xE1, + tu.S(0xE0, 0x01, 0x00, 0x00, + 0xC0, 0x03, 0xE0, ).Add(0x03).AddVar( []byte { 0x00, 0x00, 0x86, 'r', 'a', 'r', 'i', 't', 'y' }, []byte { 0x00, 0x01, 0x87, 'a', 's', 'd', 'j', 'a', 'd', 's' }, @@ -155,7 +155,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) { } testEncodeDecode( &messagePulse, - tu.S(0xE1, 0x05).AddVar( + tu.S(0xE0, 0x05).AddVar( []byte { 0x00, 0x00, 0x09 }, []byte { 0x00, 0x01, 0x41, 0xCA, 0xDF }, []byte { 0x00, 0x02, 0x61, 0x51, 0xAC }, @@ -176,7 +176,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) { } testEncodeDecode( &messageNestedArray, - tu.S(0xC1, 0x02, 0xC1, + tu.S(0xC0, 0x02, 0xC0, 0x06, 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 35, 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, @@ -202,7 +202,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) { } testEncodeDecode( &messageIntegers, - tu.S(0xE1, 13).AddVar( + tu.S(0xE0, 13).AddVar( []byte { 0x00, 0x00, 0x13 }, []byte { 0x00, 0x01, 0x20, 0xC9 }, []byte { 0x00, 0x02, 0x21, 0x34, 0xC9 }, @@ -243,7 +243,7 @@ func TestGenerateRunEncodeDecode(test *testing.T) { } testEncodeDecode( &messageDynamic, - tu.S(0xE1, 14).AddVar( + tu.S(0xE0, 14).AddVar( []byte { 0x00, 0x00, 0x20, 0x23 }, []byte { 0x00, 0x01, 0x21, 0x32, 0x47 }, []byte { 0x00, 0x02, 0x23, 0x87, 0x32, 0x45, 0x23 }, @@ -255,13 +255,13 @@ func TestGenerateRunEncodeDecode(test *testing.T) { []byte { 0x00, 0x08, 0x63, 0x45, 0x12, 0x63, 0xCE }, []byte { 0x00, 0x09, 0x67, 0x40, 0x74, 0x4E, 0x3D, 0x6F, 0xCD, 0x17, 0x75 }, []byte { 0x00, 0x0A, 0x87, 'f', 'o', 'x', ' ', 'b', 'e', 'd' }, - []byte { 0x00, 0x0B, 0xC0, 0x04, + []byte { 0x00, 0x0B, 0xC0, 0x04, 0x41, 0x00, 0x07, 0x00, 0x06, 0x00, 0x05, 0x00, 0x04 }, - []byte { 0x00, 0x0C, 0xE0, 0x02, // ERR - 0x00, 0x01, 0x20, 0x08, + []byte { 0x00, 0x0C, 0xE0, 0x02, + 0x00, 0x01, 0x40, 0x08, 0x00, 0x02, 0x67, 0x40, 0x11, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A }, []byte { 0x00, 0x0D, 0xE0, 0x03, // ERR 0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00, -- 2.51.0 From b2504cda2d697d4100c6d393f8f95be3e058353d Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 17:58:10 -0400 Subject: [PATCH 07/16] internal/testutil: Describe no longer panics on private struct fields --- internal/testutil/testutil.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index e5a3167..11df015 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -2,8 +2,9 @@ package testutil import "fmt" import "slices" -import "strings" import "reflect" +import "strings" +import "unicode" // Snake lets you compare blocks of data where the ordering of certain parts may // be swapped every which way. It is designed for comparing the encoding of @@ -141,8 +142,16 @@ func (this *describer) describe(value reflect.Value) { typ := value.Type() for index := range typ.NumField() { indexBuffer := [1]int { index } - this.iprintf("%s: ", typ.Field(index).Name) - this.describe(value.FieldByIndex(indexBuffer[:])) + field := typ.Field(index) + this.iprintf("%s: ", field.Name) + for _, char := range field.Name { + if unicode.IsUpper(char) { + this.describe(value.FieldByIndex(indexBuffer[:])) + } else { + this.printf("") + } + break + } this.iprintf("\n") } this.indent -= 1 -- 2.51.0 From 7a03d8d6b5ed68ee5804055a99149838fba1ebe0 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 18:07:00 -0400 Subject: [PATCH 08/16] tape: Test DecodeAny method --- tape/dynamic_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tape/dynamic_test.go b/tape/dynamic_test.go index b13ba17..57986c5 100644 --- a/tape/dynamic_test.go +++ b/tape/dynamic_test.go @@ -335,3 +335,24 @@ func TestTagAny(test *testing.T) { } } } + +func TestDecodeAny(test *testing.T) { + for index, payload := range samplePayloads { + correctValue := sampleValues[index] + data := payload[1:] + decoder := NewDecoder(bytes.NewBuffer(data)) + tag := Tag(payload[0]) + decoded, n, err := DecodeAny(decoder, tag) + test.Log("n: ", n) + test.Log("tag: ", tag) + test.Log("got: ", tu.Describe(decoded)) + test.Log("correct:", tu.Describe(correctValue)) + if err != nil { test.Fatal(err) } + if !reflect.DeepEqual(decoded, correctValue) { + test.Fatal("values not equal") + } + if n != len(data) { + test.Fatalf("n not equal: %d != %d", n, len(data)) + } + } +} -- 2.51.0 From 2e03867c668026bb487b3294fb0f52d3b5e342de Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 18:09:55 -0400 Subject: [PATCH 09/16] tape: Fix DecodeAny method not converting values properly --- tape/dynamic.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index 31966b6..e2dc2d4 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -116,9 +116,9 @@ func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) { destination, err := skeletonPointer(decoder, tag) if err != nil { return nil, n, err } - nn, err := DecodeAnyInto(decoder, destination, tag) + nn, err := decodeAny(decoder, destination, tag) n += nn; if err != nil { return nil, n, err } - return destination, n, err + return destination.Elem().Interface(), n, err } // unknownSlicePlaceholder is inserted by skeletonValue and informs the program -- 2.51.0 From ba2dc6b53f45e9f4a649575188cb4f22b255cff7 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 18:28:36 -0400 Subject: [PATCH 10/16] tape: Properly allocate maps when decoding KTV --- tape/dynamic.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index e2dc2d4..12e1089 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -116,7 +116,7 @@ func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) { destination, err := skeletonPointer(decoder, tag) if err != nil { return nil, n, err } - nn, err := decodeAny(decoder, destination, tag) + nn, err := decodeAny(decoder, destination.Elem(), tag) n += nn; if err != nil { return nil, n, err } return destination.Elem().Interface(), n, err } @@ -496,7 +496,11 @@ func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) { func skeletonPointer(decoder *Decoder, tag Tag) (reflect.Value, error) { typ, err := typeOf(decoder, tag) if err != nil { return reflect.Value { }, err } - return reflect.New(typ), nil + value := reflect.New(typ) + if tag.Is(KTV) { + value.Elem().Set(reflect.MakeMap(typ)) + } + return value, nil } // typeOf returns the type of the current tag being decoded. It does not use up -- 2.51.0 From c3d0f33700be7938698bda085d7c6af098069a73 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 18:41:32 -0400 Subject: [PATCH 11/16] tape: Test troublesome data structure --- tape/dynamic_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tape/dynamic_test.go b/tape/dynamic_test.go index 57986c5..1023063 100644 --- a/tape/dynamic_test.go +++ b/tape/dynamic_test.go @@ -27,6 +27,12 @@ var samplePayloads = [][]byte { 0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67, 0x02, 0x24, byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB, }, + /* map[uint16] any */ []byte { + byte(KTV.WithCN(0)), 3, + 0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00, + 0x00, 0x02, 0x82, 'h', 'i', + 0x00, 0x03, 0x21, 0x39, 0x92, + }, } var sampleValues = []any { @@ -49,6 +55,11 @@ var sampleValues = []any { 0x0223: int16(0x4567), 0x0224: uint32(0x456789AB), }, + /* map[uint16] any */ map[uint16] any { + 0x0001: float32(489.5), + 0x0002: "hi", + 0x0003: uint16(0x3992), + }, } type userDefinedInteger int16 @@ -195,7 +206,7 @@ func TestDecodeWrongType(test *testing.T) { { var dest []string; arrayCase(&dest) } } // tables should only assign to other tables - if index != 12 { + if index != 12 && index != 13 { test.Log("- map[uint16] any") { var dest = map[uint16] any { }; arrayCase(&dest) } } -- 2.51.0 From f402b46b1c184bea3fb0c354068d1a99ae034b66 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 18:50:17 -0400 Subject: [PATCH 12/16] internal/testutil: More fixes for Describe --- internal/testutil/testutil.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 11df015..0e3cf62 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -125,6 +125,10 @@ func (this *describer) describe(value reflect.Value) { return } value = reflect.ValueOf(value.Interface()) + if !value.IsValid() { + this.printf("") + return + } switch value.Kind() { case reflect.Array, reflect.Slice: this.printf("[\n") @@ -155,7 +159,7 @@ func (this *describer) describe(value reflect.Value) { this.iprintf("\n") } this.indent -= 1 - this.iprintf("}\n") + this.iprintf("}") case reflect.Map: this.printf("map {\n") this.indent += 1 @@ -168,7 +172,7 @@ func (this *describer) describe(value reflect.Value) { this.iprintf("\n") } this.indent -= 1 - this.iprintf("}\n") + this.iprintf("}") case reflect.Pointer: this.printf("& ") this.describe(value.Elem()) -- 2.51.0 From f10327356e43d432e0f8e1e81d40c992edcaaf25 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 18:50:28 -0400 Subject: [PATCH 13/16] generate: Describe more values in tests --- generate/misc_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generate/misc_test.go b/generate/misc_test.go index 3e08fc8..acf86f4 100644 --- a/generate/misc_test.go +++ b/generate/misc_test.go @@ -100,12 +100,12 @@ func testGenerateRun(test *testing.T, protocol *Protocol, title, imports, testCa log.Println("decoding:") destination := reflect.New(reflect.ValueOf(message).Elem().Type()).Interface().(Message) flat := data.Flatten() - log.Println("before: ", destination) + log.Println("before: ", tu.Describe(destination)) decoder := tape.NewDecoder(bytes.NewBuffer(flat)) n, err = destination.Decode(decoder) if err != nil { log.Fatalf("at %d: %v\n", n, err) } - log.Println("got: ", destination) - log.Println("correct:", message) + log.Println("got: ", tu.Describe(destination)) + log.Println("correct:", tu.Describe(message)) if n != len(flat) { log.Fatalf("n incorrect: %d != %d\n", n, len(flat)) } -- 2.51.0 From cbfb513933960378e7839d41d74f8ed9cc5b7f0d Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 21:27:35 -0400 Subject: [PATCH 14/16] tape: canAssign now reports true for named table types --- tape/dynamic.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index 12e1089..84f87f1 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -355,8 +355,12 @@ func encodeAnyMap(encoder *Encoder, value any, tag Tag) (n int, err error) { } func canSet(destination reflect.Type, tag Tag) error { + fmt.Println("== CAN SET ====================") + fmt.Println(destination) + fmt.Println(tag) // anything can be assigned to `any` if isTypeAny(destination) { + fmt.Println("OK") return nil } switch tag.WithoutCN() { @@ -387,12 +391,17 @@ func canSet(destination reflect.Type, tag Tag) error { return errCantAssignf("cannot assign array to %v", destination) } case KTV: - if destination != reflect.TypeOf(dummyMap) { + cantAssign := + destination.Kind() != reflect.Map || + destination.Key().Kind() != reflect.Uint16 || + !isTypeAny(destination.Elem()) + if cantAssign { return errCantAssignf("cannot assign table to %v", destination) } default: return fmt.Errorf("unknown TN %d", tag.TN()) } + fmt.Println("OK") return nil } -- 2.51.0 From 4575fa229bd8d7bcce803c3711e576ed33dc1b90 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 13 Oct 2025 10:33:36 -0400 Subject: [PATCH 15/16] generate: Make the table type an alias so we don't have a million of em --- generate/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/generate.go b/generate/generate.go index 66fb192..4bfd423 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -24,7 +24,7 @@ const preamble = ` const static = ` // Table is a KTV table with an undefined schema. -type Table map[uint16] any +type Table = map[uint16] any // Message is any message that can be sent along this protocol. type Message interface { -- 2.51.0 From 5c2b8a0582d6d0e9769ffa080330615a3f07f841 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 13 Oct 2025 10:33:59 -0400 Subject: [PATCH 16/16] tape: Correctly decode into a table destination all the time --- tape/dynamic.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index 84f87f1..22a4f2c 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -242,12 +242,22 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i } lengthCast, err := Uint64ToIntSafe(length) if err != nil { return n, err } - if isTypeAny(destination.Type()) { - // need a skeleton value if we are assigning to any. - value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast) - destination.Set(value) - destination = value - } + + // im fucking so done dude. im so fucking done. this was + // supposed to only run when we need it but i guess it runs all + // the time, because when we get a map destination (a valid, + // allocated one) we break apart on SetMapIndex because of a nil + // map. yeah thats right. a fucking nil map panic. on the map we + // just allocated. but running this unconditionally (whether or + // not we receive an empty any value) actually makes it fucking + // work. go figure(). + // + // (the map allocation functionality in skeletonPointer has been + // removed) + value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast) + destination.Set(value) + destination = value + destination.Clear() for _ = range lengthCast { key, nn, err := decoder.ReadUint16() @@ -355,12 +365,8 @@ func encodeAnyMap(encoder *Encoder, value any, tag Tag) (n int, err error) { } func canSet(destination reflect.Type, tag Tag) error { - fmt.Println("== CAN SET ====================") - fmt.Println(destination) - fmt.Println(tag) // anything can be assigned to `any` if isTypeAny(destination) { - fmt.Println("OK") return nil } switch tag.WithoutCN() { @@ -401,7 +407,6 @@ func canSet(destination reflect.Type, tag Tag) error { default: return fmt.Errorf("unknown TN %d", tag.TN()) } - fmt.Println("OK") return nil } @@ -506,9 +511,6 @@ func skeletonPointer(decoder *Decoder, tag Tag) (reflect.Value, error) { typ, err := typeOf(decoder, tag) if err != nil { return reflect.Value { }, err } value := reflect.New(typ) - if tag.Is(KTV) { - value.Elem().Set(reflect.MakeMap(typ)) - } return value, nil } -- 2.51.0