From 45dfdb255e3c4ec27316507473f863bb66808cd2 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 7 Sep 2025 23:46:22 -0400 Subject: [PATCH 01/18] design: Add Any type to PDL language document --- design/pdl-language.md | 1 + 1 file changed, 1 insertion(+) diff --git a/design/pdl-language.md b/design/pdl-language.md index 27195bd..ba77e4a 100644 --- a/design/pdl-language.md +++ b/design/pdl-language.md @@ -30,6 +30,7 @@ PDL allows defining a protocol using HOPP and TAPE. | []\ | OTA | * | Array of any type[^1] | Table | KTV | * | Table with undefined schema | {...} | KTV | * | Table with defined schema +| Any | * | * | Value of an undefined type [^1]: Excluding SI and SBA. I5 and U5 cannot be used in an array, but String and Buffer are simply forced to use their "long" variant. -- 2.50.1 From b7bdaba69418ba0e7e169f697d5d47bde6c3cd2e Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 8 Sep 2025 09:42:39 -0400 Subject: [PATCH 02/18] tape: Split DecodeAny into two funcs, one autocreates skeleton value --- tape/dynamic.go | 14 ++++++++++++-- tape/dynamic_test.go | 6 +++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index 501ed63..abe9742 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -86,9 +86,9 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) { return n, fmt.Errorf("cannot encode type %T", value) } -// DecodeAny decodes data and places it into destination, which must be a +// DecodeAnyInto decodes data and places it into destination, which must be a // pointer to a supported type. See [EncodeAny] for a list of supported types. -func DecodeAny(decoder *Decoder, destination any, tag Tag) (n int, err error) { +func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error) { reflectDestination := reflect.ValueOf(destination) if reflectDestination.Kind() != reflect.Pointer { return n, fmt.Errorf("expected pointer destination, not %v", destination) @@ -96,6 +96,16 @@ func DecodeAny(decoder *Decoder, destination any, tag Tag) (n int, err error) { return decodeAny(decoder, reflectDestination.Elem(), tag) } +// DecodeAny is like [DecodeAnyInto], but it automatically creates the +// destination from the tag and data. +func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) { + destination, err := skeletonValue(decoder, tag) + if err != nil { return nil, n, err } + nn, err := DecodeAnyInto(decoder, destination, tag) + n += nn; if err != nil { return nil, n, err } + return destination, n, err +} + // unknownSlicePlaceholder is inserted by skeletonValue and informs the program // that the destination for the slice needs to be generated based on the item // tag in the OTA. diff --git a/tape/dynamic_test.go b/tape/dynamic_test.go index 7e7bab7..bf3b66e 100644 --- a/tape/dynamic_test.go +++ b/tape/dynamic_test.go @@ -104,7 +104,7 @@ func TestDecodeWrongType(test *testing.T) { // integers should only assign to other integers if index > 8 { cas := func(destination any) { - n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0])) + n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0])) if err != nil { test.Fatalf("error: %v | n: %d", err, n) } reflectValue := reflect.ValueOf(destination).Elem() if reflectValue.CanInt() { @@ -138,7 +138,7 @@ func TestDecodeWrongType(test *testing.T) { { var dest uint64; cas(&dest) } } arrayCase := func(destination any) { - n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0])) + n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data[1:])), destination, Tag(data[0])) if err != nil { test.Fatalf("error: %v | n: %d", err, n) } reflectDestination := reflect.ValueOf(destination) reflectValue := reflectDestination.Elem() @@ -256,7 +256,7 @@ func decAny(data []byte) (Tag, any, int, error) { destination := map[uint16] any { } tag, err := TagAny(destination) if err != nil { return 0, nil, 0, err } - n, err := DecodeAny(NewDecoder(bytes.NewBuffer(data)), &destination, tag) + n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data)), &destination, tag) if err != nil { return 0, nil, n, err } return tag, destination, n, nil } -- 2.50.1 From 8dac25035fbadc0864522a836698b15d5faa576c Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 8 Sep 2025 09:47:37 -0400 Subject: [PATCH 03/18] generate: Use DecodeAnyInto in generated code --- generate/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/generate.go b/generate/generate.go index 5538075..08eb8c0 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -627,7 +627,7 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS case TypeTable: // KTV: ( )* nn, err := this.iprintf( - "nn, err = tape.DecodeAny(decoder, %s, %s)\n", + "nn, err = tape.DecodeAnyInto(decoder, %s, %s)\n", valueSource, tagSource) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() -- 2.50.1 From 419c3651bf26d20cc9de7433ae27bc1e219cfed7 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 8 Sep 2025 09:58:50 -0400 Subject: [PATCH 04/18] generate: Add Any type to parser and syntax tree --- generate/parse.go | 1 + generate/parse_test.go | 3 +++ generate/protocol.go | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/generate/parse.go b/generate/parse.go index e6e1ca3..f389772 100644 --- a/generate/parse.go +++ b/generate/parse.go @@ -116,6 +116,7 @@ func (this *parser) parseType() (Type, error) { case "String": return TypeString { }, this.Next() case "Buffer": return TypeBuffer { }, this.Next() case "Table": return TypeTable { }, this.Next() + case "Any": return TypeAny { }, this.Next() } return this.parseTypeNamed() case TokenLBracket: diff --git a/generate/parse_test.go b/generate/parse_test.go index a447ebb..c65c906 100644 --- a/generate/parse_test.go +++ b/generate/parse_test.go @@ -31,6 +31,7 @@ func TestParse(test *testing.T) { 0x0002: Field { Name: "Followers", Type: TypeInt { Bits: 32 } }, }, } + correct.Types["Anything"] = TypeAny { } test.Log("CORRECT:", &correct) got, err := ParseReader("test.pdl", strings.NewReader(` @@ -48,6 +49,8 @@ func TestParse(test *testing.T) { 0001 Bio String, 0002 Followers U32, } + + Anything Any `)) if err != nil { test.Fatal(parse.Format(err)) } test.Log("GOT: ", got) diff --git a/generate/protocol.go b/generate/protocol.go index 1610c86..0e214b9 100644 --- a/generate/protocol.go +++ b/generate/protocol.go @@ -99,6 +99,12 @@ func (typ TypeNamed) String() string { return typ.Name } +type TypeAny struct { } + +func (typ TypeAny) String() string { + return "Any" +} + func HashType(typ Type) [16]byte { // TODO: if we ever want to make the compiler more efficient, this would // be a good place to start, complex string concatenation in a hot path -- 2.50.1 From 785b48085d81f68328d3ece99b7014a45992d262 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 8 Sep 2025 18:24:21 -0400 Subject: [PATCH 05/18] tape: Dynamically encode floating point values --- tape/dynamic.go | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index abe9742..9e9594a 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -36,16 +36,18 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) { // primitives reflectValue := reflect.ValueOf(value) switch reflectValue.Kind() { - case reflect.Int: return encoder.WriteInt32(int32(reflectValue.Int())) - case reflect.Uint: return encoder.WriteUint32(uint32(reflectValue.Uint())) - case reflect.Int8: return encoder.WriteInt8(int8(reflectValue.Int())) - case reflect.Uint8: return encoder.WriteUint8(uint8(reflectValue.Uint())) - case reflect.Int16: return encoder.WriteInt16(int16(reflectValue.Int())) - case reflect.Uint16: return encoder.WriteUint16(uint16(reflectValue.Uint())) - case reflect.Int32: return encoder.WriteInt32(int32(reflectValue.Int())) - case reflect.Uint32: return encoder.WriteUint32(uint32(reflectValue.Uint())) - case reflect.Int64: return encoder.WriteInt64(int64(reflectValue.Int())) - case reflect.Uint64: return encoder.WriteUint64(uint64(reflectValue.Uint())) + case reflect.Int: return encoder.WriteInt32(int32(reflectValue.Int())) + case reflect.Uint: return encoder.WriteUint32(uint32(reflectValue.Uint())) + case reflect.Int8: return encoder.WriteInt8(int8(reflectValue.Int())) + case reflect.Uint8: return encoder.WriteUint8(uint8(reflectValue.Uint())) + case reflect.Int16: return encoder.WriteInt16(int16(reflectValue.Int())) + case reflect.Uint16: return encoder.WriteUint16(uint16(reflectValue.Uint())) + case reflect.Int32: return encoder.WriteInt32(int32(reflectValue.Int())) + case reflect.Uint32: return encoder.WriteUint32(uint32(reflectValue.Uint())) + case reflect.Int64: return encoder.WriteInt64(int64(reflectValue.Int())) + case reflect.Uint64: return encoder.WriteUint64(uint64(reflectValue.Uint())) + case reflect.Float32: return encoder.WriteFloat32(float32(reflectValue.Float())) + case reflect.Float64: return encoder.WriteFloat64(float64(reflectValue.Float())) case reflect.String: if reflectValue.Len() > MaxStructureLength { return 0, ErrTooLong @@ -244,17 +246,19 @@ func TagAny(value any) (Tag, error) { func tagAny(reflectValue reflect.Value) (Tag, error) { // primitives switch reflectValue.Kind() { - case reflect.Int: return LSI.WithCN(3), nil - case reflect.Int8: return LSI.WithCN(0), nil - case reflect.Int16: return LSI.WithCN(1), nil - case reflect.Int32: return LSI.WithCN(3), nil - case reflect.Int64: return LSI.WithCN(7), nil - case reflect.Uint: return LI.WithCN(3), nil - case reflect.Uint8: return LI.WithCN(0), nil - case reflect.Uint16: return LI.WithCN(1), nil - case reflect.Uint32: return LI.WithCN(3), nil - case reflect.Uint64: return LI.WithCN(7), nil - case reflect.String: return bufferLenTag(reflectValue.Len()), nil + case reflect.Int: return LSI.WithCN(3), nil + case reflect.Int8: return LSI.WithCN(0), nil + case reflect.Int16: return LSI.WithCN(1), nil + case reflect.Int32: return LSI.WithCN(3), nil + case reflect.Int64: return LSI.WithCN(7), nil + case reflect.Uint: return LI.WithCN(3), nil + case reflect.Uint8: return LI.WithCN(0), nil + case reflect.Uint16: return LI.WithCN(1), nil + case reflect.Uint32: return LI.WithCN(3), nil + case reflect.Uint64: return LI.WithCN(7), nil + case reflect.Float32: return FP.WithCN(3), nil + case reflect.Float64: return FP.WithCN(7), nil + case reflect.String: return bufferLenTag(reflectValue.Len()), nil } if reflectValue.CanConvert(reflect.TypeOf(dummyBuffer)) { return bufferLenTag(reflectValue.Len()), nil -- 2.50.1 From 8b0915dff1d4c973d50eedef60a011523bd658a3 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 8 Sep 2025 21:37:39 -0400 Subject: [PATCH 06/18] tape: Test that floating point values can be dynamically encoded --- tape/dynamic_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tape/dynamic_test.go b/tape/dynamic_test.go index bf3b66e..5050ccb 100644 --- a/tape/dynamic_test.go +++ b/tape/dynamic_test.go @@ -26,7 +26,9 @@ func TestEncodeAnyTable(test *testing.T) { 0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} }, 0x2345: [][]int16 { []int16 { 0x5 }, []int16 { 0x17, -0xAAA } }, 0x3456: userDefinedInteger(0x3921), - }, KTV.WithCN(0), tu.S(7).AddVar( + 0x1F1F: float32(67.26), + 0x0F0F: float64(5.3), + }, KTV.WithCN(0), tu.S(9).AddVar( []byte { 0xF3, 0xB9, byte(LSI.WithCN(3)), @@ -70,6 +72,16 @@ func TestEncodeAnyTable(test *testing.T) { byte(LSI.WithCN(1)), 0x39, 0x21, }, + []byte { + 0x1F, 0x1F, + byte(FP.WithCN(3)), + 0x42, 0x86, 0x85, 0x1F, + }, + []byte { + 0x0F, 0x0F, + byte(FP.WithCN(7)), + 0x40, 0x15, 0x33, 0x33, 0x33, 0x33, 0x33, 0x33, + }, )) if err != nil { test.Fatal(err) } } -- 2.50.1 From c4ab60515bd1cb10b21ca78a518b138dc47885b6 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 8 Sep 2025 21:39:50 -0400 Subject: [PATCH 07/18] tape: Test that floating point values can be decoded dynamically --- tape/dynamic_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tape/dynamic_test.go b/tape/dynamic_test.go index 5050ccb..85b83d3 100644 --- a/tape/dynamic_test.go +++ b/tape/dynamic_test.go @@ -190,6 +190,8 @@ func TestEncodeDecodeAnyTable(test *testing.T) { 0x0000: []byte("hi!"), 0xFFFF: []uint16 { 0xBEE5, 0x7777 }, 0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} }, + 0x1F1F: float32(67.26), + 0x0F0F: float64(5.3), }, nil) if err != nil { test.Fatal(err) } } -- 2.50.1 From 1bb565c6fea1be27346f1e9edb52ab6131380d24 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 10 Sep 2025 09:45:09 -0400 Subject: [PATCH 08/18] generate: Write tests for Any type --- generate/generate_test.go | 68 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/generate/generate_test.go b/generate/generate_test.go index a4d0706..1cc7801 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -62,6 +62,27 @@ func init() { }, }, } + exampleProtocol.Messages[0x0005] = Message { + Name: "Dynamic", + Type: TypeTableDefined { + Fields: map[uint16] Field { + 0x0000: Field { Name: "AU8", Type: TypeAny { } }, + 0x0001: Field { Name: "AU16", Type: TypeAny { } }, + 0x0002: Field { Name: "AU32", Type: TypeAny { } }, + 0x0003: Field { Name: "AU64", Type: TypeAny { } }, + 0x0004: Field { Name: "AI8", Type: TypeAny { } }, + 0x0005: Field { Name: "AI16", Type: TypeAny { } }, + 0x0006: Field { Name: "AI32", Type: TypeAny { } }, + 0x0007: Field { Name: "AI64", Type: TypeAny { } }, + 0x0008: Field { Name: "AF32", Type: TypeAny { } }, + 0x0009: Field { Name: "AF64", Type: TypeAny { } }, + 0x000A: Field { Name: "AString", Type: TypeAny { } }, + 0x000B: Field { Name: "AArray", Type: TypeAny { } }, + 0x000C: Field { Name: "ATable", Type: TypeAny { } }, + 0x000D: Field { Name: "T0", Type: TypeTable { } }, + }, + }, + } exampleProtocol.Types["User"] = TypeTableDefined { Fields: map[uint16] Field { 0x0000: Field { Name: "Name", Type: TypeString { } }, @@ -196,6 +217,53 @@ func TestGenerateRunEncodeDecode(test *testing.T) { []byte { 0x00, 0x0D, 0x43, 0xEF, 0x1E, 0xCB, 0x37 }, []byte { 0x00, 0x0E, 0x47, 0x9C, 0x6E, 0xF6, 0x43, 0xEF, 0x1E, 0xCB, 0x37 }, )) + log.Println("MessageDynamic") + messageDynamic := MessageDynamic { + AU8: uint8(0x23), + AU16: uint16(0x3247), + AU32: uint32(0x87324523), + AU64: uint64(0x3284029034098234), + AI8: int8(0x23), + AI16: int16(0x3247), + AI32: int32(0x57324523), + AI64: int64(0x3284029034098234), + AF32: float32(2342.2378), + AF64: float64(324.8899992), + AString: "fox bed", + AArray: []int16 { 0x7, 0x6, 0x5, 0x4 }, + ATable: map[uint16] any { + 0x0001: int8(0x8), + 0x0002: float64(4.4), + }, + T0: map[uint16] any { + 0x0001: float32(489.5), + 0x0002: "hi", + 0x0003: uint16(0x3992), + }, + } + testEncodeDecode( + &messageDynamic, + tu.S(0xE1, 14).AddVar( + []byte { 0x00, 0x00, 0x20, 0x23 }, + []byte { 0x00, 0x01, 0x21, 0x32, 0x47 }, + []byte { 0x00, 0x02, 0x23, 0x87, 0x32, 0x45, 0x23 }, + []byte { 0x00, 0x03, 0x27, 0x32, 0x84, 0x02, 0x90, 0x34, 0x09, 0x82, 0x34 }, + []byte { 0x00, 0x04, 0x40, 0x23 }, + []byte { 0x00, 0x05, 0x41, 0x32, 0x47 }, + []byte { 0x00, 0x06, 0x43, 0x57, 0x32, 0x45, 0x23 }, + []byte { 0x00, 0x07, 0x47, 0x32, 0x84, 0x02, 0x90, 0x34, 0x09, 0x82, 0x34 }, + []byte { 0x00, 0x08, 0x63, 0x45, 0x12, 0x63, 0xCE }, + []byte { 0x00, 0x09, 0x67, 0x40, 0x74, 0x4E, 0x3D, 0x6F, 0xCD, 0x17, 0x75 }, + []byte { 0x00, 0x0A, 0x87, 'f', 'o', 'x', ' ', 'b', 'e', 'd' }, + []byte { 0x00, 0x0B, 0xC4, 0x00, 0x07, 0x00, 0x06, 0x00, 0x05, 0x00, 0x04 }, + []byte { 0x00, 0x0C, 0xE1, 0x02, + 0x00, 0x01, 0x20, 0x08, + 0x00, 0x02, 0x67, 0x40, 0x11, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9A }, + []byte { 0x00, 0x0D, 0xE1, 0x03, + 0x00, 0x01, 0x63, 0x43, 0xF4, 0xC0, 0x00, + 0x00, 0x02, 0x82, 'h', 'i', + 0x00, 0x03, 0x21, 0x39, 0x92 }, + )) `) } -- 2.50.1 From 92040a1bc44ace59e37190cafd74828ca3baa65c Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 10 Sep 2025 09:45:25 -0400 Subject: [PATCH 09/18] generate: Implement encoding and decoding of Any type --- generate/generate.go | 127 +++++++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 53 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 08eb8c0..520171f 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -235,17 +235,13 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e this.resolveMessageName(message.Name)) n += nn; if err != nil { return n, err } this.push() - nn, err = this.iprintf("tag := ") + tagVar, nn, err := this.generateTag(message.Type, "(*this)") n += nn; if err != nil { return n, err } - nn, err = this.generateTag(message.Type, "(*this)") - n += nn; if err != nil { return n, err } - nn, err = this.println() - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("nn, err := encoder.WriteTag(tag)\n") + nn, err = this.iprintf("nn, err := encoder.WriteTag(%s)\n", tagVar) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - nn, err = this.generateEncodeValue(message.Type, "(*this)", "tag") + nn, err = this.generateEncodeValue(message.Type, "(*this)", tagVar) n += nn; if err != nil { return n, err } nn, err = this.iprintf("return n, nil\n") n += nn; if err != nil { return n, err } @@ -398,15 +394,11 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri this.push() nn, err = this.iprintf("_ = item\n") n += nn; if err != nil { return n, err } - nn, err = this.iprintf("tag := ") + tagVar, nn, err := this.generateTag(typ.Element, "item") n += nn; if err != nil { return n, err } - nn, err = this.generateTag(typ.Element, "item") + nn, err = this.iprintf("if %s.Is(tape.SBA) { continue }\n", tagVar) n += nn; if err != nil { return n, err } - nn, err = this.println() - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("if tag.Is(tape.SBA) { continue }\n") - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("if tag.CN() > itemTag.CN() { itemTag = tag }\n") + nn, err = this.iprintf("if %s.CN() > itemTag.CN() { itemTag = %s }\n", tagVar, tagVar) n += nn; if err != nil { return n, err } this.pop() nn, err = this.iprintf("}\n") @@ -461,25 +453,19 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri nn, err = this.iprintf("{\n") n += nn; if err != nil { return n, err } this.push() - nn, err = this.iprintf("var tag tape.Tag\n") - n += nn; if err != nil { return n, err } for key, field := range typ.Fields { nn, err = this.iprintf("nn, err = encoder.WriteUint16(0x%04X)\n", key) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - nn, err = this.iprintf("tag = ") - n += nn; if err != nil { return n, err } fieldSource := fmt.Sprintf("%s.%s", valueSource, field.Name) - nn, err = this.generateTag(field.Type, fieldSource) + tagVar, nn, err := this.generateTag(field.Type, fieldSource) n += nn; if err != nil { return n, err } - nn, err = this.println() - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(tag))\n") + nn, err = this.iprintf("nn, err = encoder.WriteUint8(uint8(%s))\n", tagVar) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - nn, err = this.generateEncodeValue(field.Type, fieldSource, "tag") + nn, err = this.generateEncodeValue(field.Type, fieldSource, tagVar) n += nn; if err != nil { return n, err } } this.pop() @@ -491,6 +477,12 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } + case TypeAny: + // WHATEVER: [WHATEVER] + nn, err := this.iprintf("nn, err = tape.EncodeAny(encoder, %s, %s)\n", valueSource, tagSource) + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } default: panic(fmt.Errorf("unknown type: %T", typ)) } @@ -642,6 +634,12 @@ func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagS n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } + case TypeAny: + // WHATEVER: [WHATEVER] + nn, err := this.iprintf("*%s, nn, err = tape.DecodeAny(decoder, %s)\n", valueSource, tagSource) + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } default: panic(fmt.Errorf("unknown type: %T", typ)) } @@ -924,49 +922,60 @@ func (this *Generator) generateErrorCheck() (n int, err error) { return this.iprintf("n += nn; if err != nil { return n, err }\n") } +func (this *Generator) generateBareErrorCheck() (n int, err error) { + return this.iprintf("if err != nil { return n, err }\n") +} + // generateTag generates the preferred TN and CN for the given type and value. -// The generated code is INLINE. -func (this *Generator) generateTag(typ Type, source string) (n int, err error) { +// The generated code is a BLOCK. +func (this *Generator) generateTag(typ Type, source string) (tagVar string, n int, err error) { + tagVar = this.newTemporaryVar("tag") switch typ := typ.(type) { case TypeInt: if typ.Bits <= 5 { - nn, err := this.printf("tape.SI.WithCN(int(%s))", source) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.SI.WithCN(int(%s))\n", tagVar, source) + n += nn; if err != nil { return tagVar, n, err } } else if typ.Signed { - nn, err := this.printf("tape.LSI.WithCN(%d)", bitsToCN(typ.Bits)) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.LSI.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits)) + n += nn; if err != nil { return tagVar, n, err } } else { - nn, err := this.printf("tape.LI.WithCN(%d)", bitsToCN(typ.Bits)) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.LI.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits)) + n += nn; if err != nil { return tagVar, n, err } } case TypeFloat: - nn, err := this.printf("tape.FP.WithCN(%d)", bitsToCN(typ.Bits)) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.FP.WithCN(%d)\n", tagVar, bitsToCN(typ.Bits)) + n += nn; if err != nil { return tagVar, n, err } case TypeString: - nn, err := this.printf("tape.StringTag(string(%s))", source) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.StringTag(string(%s))\n", tagVar, source) + n += nn; if err != nil { return tagVar, n, err } case TypeBuffer: - nn, err := this.printf("tape.BufferTag([]byte(%s))", source) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.BufferTag([]byte(%s))\n", tagVar, source) + n += nn; if err != nil { return tagVar, n, err } case TypeArray: - nn, err := this.printf("tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))))", source) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.OTA.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source) + n += nn; if err != nil { return tagVar, n, err } case TypeTable: - nn, err := this.printf("tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))))", source) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.KTV.WithCN(tape.IntBytes(uint64(len(%s))))\n", tagVar, source) + n += nn; if err != nil { return tagVar, n, err } case TypeTableDefined: - nn, err := this.printf("tape.KTV.WithCN(%d)", tape.IntBytes(uint64(len(typ.Fields)))) - n += nn; if err != nil { return n, err } + nn, err := this.iprintf("%s := tape.KTV.WithCN(%d)\n", tagVar, tape.IntBytes(uint64(len(typ.Fields)))) + n += nn; if err != nil { return tagVar, n, err } case TypeNamed: resolved, err := this.resolveTypeName(typ.Name) - if err != nil { return n, err } - nn, err := this.generateTag(resolved, source) - n += nn; if err != nil { return n, err } + if err != nil { return tagVar, n, err } + subTagVar, nn, err := this.generateTag(resolved, source) + n += nn; if err != nil { return tagVar, n, err } + tagVar = subTagVar + case TypeAny: + nn, err := this.iprintf("%s, err := tape.TagAny(%s)\n", tagVar, source) + n += nn; if err != nil { return tagVar, n, err } + nn, err = this.generateBareErrorCheck() + n += nn; if err != nil { return tagVar, n, err } default: panic(fmt.Errorf("unknown type: %T", typ)) } - return n, nil + return tagVar, n, nil } // generateTN generates the appropriate TN for the given type. The generated @@ -1009,6 +1018,8 @@ func (this *Generator) generateTN(typ Type) (n int, err error) { if err != nil { return n, err } nn, err := this.generateTN(resolved) n += nn; if err != nil { return n, err } + default: + panic(fmt.Errorf("unknown type: %T", typ)) } return n, nil @@ -1063,6 +1074,11 @@ func (this *Generator) generateType(typ Type) (n int, err error) { case TypeNamed: nn, err := this.print(typ.Name) n += nn; if err != nil { return n, err } + case TypeAny: + nn, err := this.print("any") + n += nn; if err != nil { return n, err } + default: + panic(fmt.Errorf("unknown type: %T", typ)) } return n, nil } @@ -1092,12 +1108,17 @@ func (this *Generator) generateTypeTableDefined(typ TypeTableDefined) (n int, er // by tagSource can be assigned to a Go destination generated from typ. The // generated code is INLINE. func (this *Generator) generateCanAssign(typ Type, tagSource string) (n int, err error) { - nn, err := this.printf("canAssign(") - n += nn; if err != nil { return n, err } - nn, err = this.generateTN(typ) - n += nn; if err != nil { return n, err } - nn, err = this.printf(", %s)", tagSource) - n += nn; if err != nil { return n, err } + if _, ok := typ.(TypeAny); ok { + nn, err := this.printf("true") + n += nn; if err != nil { return n, err } + } else { + nn, err := this.printf("canAssign(") + n += nn; if err != nil { return n, err } + nn, err = this.generateTN(typ) + n += nn; if err != nil { return n, err } + nn, err = this.printf(", %s)", tagSource) + n += nn; if err != nil { return n, err } + } return n, nil } -- 2.50.1 From 81391ef1015ad9697a32431f3e19b258b5367c74 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 10 Sep 2025 10:20:44 -0400 Subject: [PATCH 10/18] testutil: Print instead of crashing on invalid value --- internal/testutil/testutil.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 1a3addd..e5a3167 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -119,6 +119,10 @@ type describer struct { } func (this *describer) describe(value reflect.Value) { + if !value.IsValid() { + this.printf("") + return + } value = reflect.ValueOf(value.Interface()) switch value.Kind() { case reflect.Array, reflect.Slice: -- 2.50.1 From 85a66a3e70ceb9a18a9de143a8d762b783154cc0 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 10 Sep 2025 10:21:29 -0400 Subject: [PATCH 11/18] tape: Create test to ensure DecodeAnyInto can receive a pointer to any --- tape/dynamic_test.go | 162 ++++++++++++++++++------------------------- tape/misc_test.go | 75 ++++++++++++++++++++ 2 files changed, 143 insertions(+), 94 deletions(-) create mode 100644 tape/misc_test.go diff --git a/tape/dynamic_test.go b/tape/dynamic_test.go index 85b83d3..a0078ae 100644 --- a/tape/dynamic_test.go +++ b/tape/dynamic_test.go @@ -1,11 +1,56 @@ package tape -import "fmt" import "bytes" import "testing" import "reflect" import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil" +var samplePayloads = [][]byte { + /* int8 */ []byte { byte(LSI.WithCN(0)), 0x45 }, + /* int16 */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 }, + /* int32 */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB }, + /* int64 */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 }, + /* uint5 */ []byte { byte(SI.WithCN(12)) }, + /* uint8 */ []byte { byte(LI.WithCN(0)), 0x45 }, + /* uint16 */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 }, + /* uint32 */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB }, + /* uint64 */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 }, + /* string */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' }, + /* []byte */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' }, + /* []string */ []byte { + byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)), + 0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23, + 0x05, 0x11, 0x11, 0x11, 0x11, 0x11, + }, + /* map[uint16] any */ []byte { + byte(KTV.WithCN(0)), 2, + 0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67, + 0x02, 0x24, byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB, + }, +} + +var sampleValues = []any { + /* int8 */ int8(0x45), + /* int16 */ int16(0x4567), + /* int32 */ int32(0x456789AB), + /* int64 */ int64(0x456789ABCDEF0123), + /* uint5 */ uint8(12), + /* uint8 */ uint8(0x45), + /* uint16 */ uint16(0x4567), + /* uint32 */ uint32(0x456789AB), + /* uint64 */ uint64(0x456789ABCDEF0123), + /* string */ "pupever", + /* []byte */ "blarg", + /* []string */ []string { + "\x45\x67\x89\xAB\xCD\xEF\x01\x23", + "\x11\x11\x11\x11\x11", + }, + /* map[uint16] any */ map[uint16] any { + 0x0223: int16(0x4567), + 0x0224: uint32(0x456789AB), + }, +} + type userDefinedInteger int16 func TestEncodeAnyInt(test *testing.T) { @@ -87,31 +132,7 @@ func TestEncodeAnyTable(test *testing.T) { } func TestDecodeWrongType(test *testing.T) { - datas := [][]byte { - /* int8 */ []byte { byte(LSI.WithCN(0)), 0x45 }, - /* int16 */ []byte { byte(LSI.WithCN(1)), 0x45, 0x67 }, - /* int32 */ []byte { byte(LSI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB }, - /* int64 */ []byte { byte(LSI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 }, - /* uint5 */ []byte { byte(SI.WithCN(12)) }, - /* uint8 */ []byte { byte(LI.WithCN(0)), 0x45 }, - /* uint16 */ []byte { byte(LI.WithCN(1)), 0x45, 0x67 }, - /* uint32 */ []byte { byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB }, - /* uint64 */ []byte { byte(LI.WithCN(7)), 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23 }, - /* string */ []byte { byte(SBA.WithCN(7)), 'p', 'u', 'p', 'e', 'v', 'e', 'r' }, - /* []byte */ []byte { byte(SBA.WithCN(5)), 'b', 'l', 'a', 'r', 'g' }, - /* []string */ []byte { - byte(OTA.WithCN(0)), 2, byte(LBA.WithCN(0)), - 0x08, 0x45, 0x67, 0x89, 0xAB, 0xCD, 0xEF, 0x01, 0x23, - 0x05, 0x11, 0x11, 0x11, 0x11, 0x11, - }, - /* map[uint16] any */ []byte { - byte(KTV.WithCN(0)), 2, - 0x02, 0x23, byte(LSI.WithCN(1)), 0x45, 0x67, - 0x02, 0x23, byte(LI.WithCN(3)), 0x45, 0x67, 0x89, 0xAB, - }, - } - - for index, data := range datas { + for index, data := range samplePayloads { test.Logf("data %2d %v [%s]", index, Tag(data[0]), tu.HexBytes(data[1:])) // integers should only assign to other integers if index > 8 { @@ -196,6 +217,27 @@ func TestEncodeDecodeAnyTable(test *testing.T) { if err != nil { test.Fatal(err) } } +func TestEncodeDecodeAnyDestination(test *testing.T) { + var destination any + for index, data := range samplePayloads { + tag := Tag(data[0]) + payload := data[1:] + test.Logf("data %2d %v [%s]", index, tag, tu.HexBytes(payload)) + n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(payload)), &destination, tag) + if err != nil { test.Fatalf("error: %v | n: %d", err, n) } + got := destination + correct := sampleValues[index] + test.Log("got: ", tu.Describe(got)) + test.Log("correct:", tu.Describe(correct)) + if !reflect.DeepEqual(got, correct) { + test.Fatalf("values not equal") + } + if n != len(payload) { + test.Fatalf("n not equal: %d != %d", n, len(payload)) + } + } +} + func TestPeekSlice(test *testing.T) { buffer := bytes.NewBuffer([]byte { 2, byte(OTA.WithCN(3)), @@ -254,71 +296,3 @@ func TestPeekSliceOnce(test *testing.T) { test.Fatalf("wrong n: %d != %d", got, correct) } } - -func encAny(value any) ([]byte, Tag, int, error) { - tag, err := TagAny(value) - if err != nil { return nil, 0, 0, err } - buffer := bytes.Buffer { } - encoder := NewEncoder(&buffer) - n, err := EncodeAny(encoder, value, tag) - if err != nil { return nil, 0, n, err } - encoder.Flush() - return buffer.Bytes(), tag, n, nil -} - -func decAny(data []byte) (Tag, any, int, error) { - destination := map[uint16] any { } - tag, err := TagAny(destination) - if err != nil { return 0, nil, 0, err } - n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data)), &destination, tag) - if err != nil { return 0, nil, n, err } - return tag, destination, n, nil -} - -func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.Snake) error { - bytes, tag, n, err := encAny(value) - if err != nil { return err } - test.Log("n: ", n) - test.Log("tag: ", tag) - test.Log("got: ", tu.HexBytes(bytes)) - test.Log("correct:", correctBytes) - if tag != correctTag { - return fmt.Errorf("tag not equal: %v != %v", tag, correctTag) - } - if ok, n := correctBytes.Check(bytes); !ok { - return fmt.Errorf("bytes not equal at index %d", n) - } - if n != len(bytes) { - return fmt.Errorf("n not equal: %d != %d", n, len(bytes)) - } - return nil -} - -func testEncodeDecodeAny(test *testing.T, value, correctValue any) error { - if correctValue == nil { - correctValue = value - } - - test.Log("encoding...") - bytes, tag, n, err := encAny(value) - if err != nil { return err } - test.Log("n: ", n) - test.Log("tag:", tag) - test.Log("got:", tu.HexBytes(bytes)) - test.Log("decoding...", tag) - if n != len(bytes) { - return fmt.Errorf("n not equal: %d != %d", n, len(bytes)) - } - - _, decoded, n, err := decAny(bytes) - if err != nil { return err } - test.Log("got: ", tu.Describe(decoded)) - test.Log("correct:", tu.Describe(correctValue)) - if !reflect.DeepEqual(decoded, correctValue) { - return fmt.Errorf("values not equal") - } - if n != len(bytes) { - return fmt.Errorf("n not equal: %d != %d", n, len(bytes)) - } - return nil -} diff --git a/tape/misc_test.go b/tape/misc_test.go new file mode 100644 index 0000000..5151926 --- /dev/null +++ b/tape/misc_test.go @@ -0,0 +1,75 @@ +package tape + +import "fmt" +import "bytes" +import "testing" +import "reflect" +import tu "git.tebibyte.media/sashakoshka/hopp/internal/testutil" + +func encAny(value any) ([]byte, Tag, int, error) { + tag, err := TagAny(value) + if err != nil { return nil, 0, 0, err } + buffer := bytes.Buffer { } + encoder := NewEncoder(&buffer) + n, err := EncodeAny(encoder, value, tag) + if err != nil { return nil, 0, n, err } + encoder.Flush() + return buffer.Bytes(), tag, n, nil +} + +func decAny(data []byte) (Tag, any, int, error) { + destination := map[uint16] any { } + tag, err := TagAny(destination) + if err != nil { return 0, nil, 0, err } + n, err := DecodeAnyInto(NewDecoder(bytes.NewBuffer(data)), &destination, tag) + if err != nil { return 0, nil, n, err } + return tag, destination, n, nil +} + +func testEncodeAny(test *testing.T, value any, correctTag Tag, correctBytes tu.Snake) error { + bytes, tag, n, err := encAny(value) + if err != nil { return err } + test.Log("n: ", n) + test.Log("tag: ", tag) + test.Log("got: ", tu.HexBytes(bytes)) + test.Log("correct:", correctBytes) + if tag != correctTag { + return fmt.Errorf("tag not equal: %v != %v", tag, correctTag) + } + if ok, n := correctBytes.Check(bytes); !ok { + return fmt.Errorf("bytes not equal at index %d", n) + } + if n != len(bytes) { + return fmt.Errorf("n not equal: %d != %d", n, len(bytes)) + } + return nil +} + +func testEncodeDecodeAny(test *testing.T, value, correctValue any) error { + if correctValue == nil { + correctValue = value + } + + test.Log("encoding...") + bytes, tag, n, err := encAny(value) + if err != nil { return err } + test.Log("n: ", n) + test.Log("tag:", tag) + test.Log("got:", tu.HexBytes(bytes)) + test.Log("decoding...", tag) + if n != len(bytes) { + return fmt.Errorf("n not equal: %d != %d", n, len(bytes)) + } + + _, decoded, n, err := decAny(bytes) + if err != nil { return err } + test.Log("got: ", tu.Describe(decoded)) + test.Log("correct:", tu.Describe(correctValue)) + if !reflect.DeepEqual(decoded, correctValue) { + return fmt.Errorf("values not equal") + } + if n != len(bytes) { + return fmt.Errorf("n not equal: %d != %d", n, len(bytes)) + } + return nil +} -- 2.50.1 From 84b96ed8f3281a7a91d96ecba16fe32f3c9d21f1 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Fri, 26 Sep 2025 00:20:31 -0400 Subject: [PATCH 12/18] Add /debug to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6f9c924..8d41422 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /generate/test +/debug -- 2.50.1 From 7df18f7d260b076f415a4e1da490e625d0313cfb Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Fri, 26 Sep 2025 00:20:53 -0400 Subject: [PATCH 13/18] tape: Assorted changes i forgor --- tape/dynamic.go | 63 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index 9e9594a..15f3483 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -8,6 +8,17 @@ package tape // TODO: test all of these smaller functions individually +// For an explanation as to why this package always treats LBA/SBA as strings, +// refer to https://go.dev/blog/strings: +// +// It’s important to state right up front that a string holds arbitrary +// bytes. It is not required to hold Unicode text, UTF-8 text, or any other +// predefined format. As far as the content of a string is concerned, it is +// exactly equivalent to a slice of bytes. +// +// Arbitrary byte slices and blobs won't be as common of a use case as text +// data, and if you need that anyway you can just cast it to a byte slice. + import "fmt" import "reflect" @@ -142,7 +153,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i switch tag.WithoutCN() { case SI: // SI: (none) - setInt(destination, uint64(tag.CN())) + setUint(destination, uint64(tag.CN()), 1) case LI: // LI: nn, err := decodeAndSetUint(decoder, destination, tag.CN() + 1) @@ -164,7 +175,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i buffer := make([]byte, length) nn, err := decoder.Read(buffer) n += nn; if err != nil { return n, err } - setByteArray(destination, buffer) + setString(destination, string(buffer)) case LBA: // LBA: * length, nn, err := decoder.ReadUintN(tag.CN() + 1) @@ -175,7 +186,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i buffer := make([]byte, length) nn, err = decoder.Read(buffer) n += nn; if err != nil { return n, err } - setByteArray(destination, buffer) + setString(destination, string(buffer)) case OTA: // OTA: * length, nn, err := decoder.ReadUintN(tag.CN() + 1) @@ -325,6 +336,10 @@ func encodeAnyMap(encoder *Encoder, value any, tag Tag) (n int, err error) { } func canSet(destination reflect.Type, tag Tag) error { + // anything can be assigned to `any` + if isTypeAny(destination) { + return nil + } switch tag.WithoutCN() { case SI, LI, LSI: switch destination.Kind() { @@ -362,17 +377,43 @@ func canSet(destination reflect.Type, tag Tag) error { } // setInt expects a settable destination. -func setInt[T int64 | uint64](destination reflect.Value, value T) { +func setInt(destination reflect.Value, value int64, bytes int) { switch { case destination.CanInt(): destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type())) case destination.CanUint(): destination.Set(reflect.ValueOf(value).Convert(destination.Type())) + case isTypeAny(destination.Type()): + switch { + case bytes > 4: destination.Set(reflect.ValueOf(int64(value))) + case bytes > 2: destination.Set(reflect.ValueOf(int32(value))) + case bytes > 1: destination.Set(reflect.ValueOf(int16(value))) + default: destination.Set(reflect.ValueOf(int8(value))) + } default: panic("setInt called on an unsupported type") } } +// setUint expects a settable destination. +func setUint(destination reflect.Value, value uint64, bytes int) { + switch { + case destination.CanInt(): + destination.Set(reflect.ValueOf(int64(value)).Convert(destination.Type())) + case destination.CanUint(): + destination.Set(reflect.ValueOf(value).Convert(destination.Type())) + case isTypeAny(destination.Type()): + switch { + case bytes > 4: destination.Set(reflect.ValueOf(uint64(value))) + case bytes > 2: destination.Set(reflect.ValueOf(uint32(value))) + case bytes > 1: destination.Set(reflect.ValueOf(uint16(value))) + default: destination.Set(reflect.ValueOf(uint8(value))) + } + default: + panic("setUint called on an unsupported type") + } +} + // setFloat expects a settable destination. func setFloat(destination reflect.Value, value float64) { destination.Set(reflect.ValueOf(value).Convert(destination.Type())) @@ -387,7 +428,7 @@ func setByteArray(destination reflect.Value, value []byte) { func decodeAndSetInt(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) { value, nn, err := decoder.ReadIntN(bytes) n += nn; if err != nil { return n, err } - setInt(destination, value) + setInt(destination, value, bytes) return n, nil } @@ -395,7 +436,7 @@ func decodeAndSetInt(decoder *Decoder, destination reflect.Value, bytes int) (n func decodeAndSetUint(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) { value, nn, err := decoder.ReadUintN(bytes) n += nn; if err != nil { return n, err } - setInt(destination, value) + setUint(destination, value, bytes) return n, nil } @@ -452,8 +493,8 @@ func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) { case 7: return reflect.TypeOf(float64(0)), nil } return nil, fmt.Errorf("unknown CN %d for FP", tag.CN()) - case SBA: return reflect.SliceOf(reflect.TypeOf(byte(0))), nil - case LBA: return reflect.SliceOf(reflect.TypeOf(byte(0))), nil + case SBA: return reflect.TypeOf(""), nil + case LBA: return reflect.TypeOf(""), nil case OTA: elemTag, dimension, err := peekSlice(decoder, tag) if err != nil { return nil, err } @@ -469,6 +510,12 @@ func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) { return nil, fmt.Errorf("unknown TN %d", tag.TN()) } +// isTypeAny returns whether the given reflect.Type is an interface with no +// methods. +func isTypeAny(typ reflect.Type) bool { + return typ.Kind() == reflect.Interface && typ.NumMethod() == 0 +} + // peekSlice returns the element tag and dimension count of the OTA currently // being decoded. It does not use up the decoder, it only peeks. func peekSlice(decoder *Decoder, tag Tag) (Tag, int, error) { -- 2.50.1 From 19f02d61373df6df15398707e2e76eed4e5a4979 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 10:43:04 -0400 Subject: [PATCH 14/18] tape: Implement setString --- tape/dynamic.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tape/dynamic.go b/tape/dynamic.go index 15f3483..9712b14 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -424,6 +424,11 @@ func setByteArray(destination reflect.Value, value []byte) { destination.Set(reflect.ValueOf(value)) } +// setString exepctes a settable destination +func setString(destination reflect.Value, value string) { + destination.Set(reflect.ValueOf(value)) +} + // decodeAndSetInt expects a settable destination. func decodeAndSetInt(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) { value, nn, err := decoder.ReadIntN(bytes) -- 2.50.1 From 56c376cd4e889233cc9566e24606a1ffffabb15c Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 11:17:41 -0400 Subject: [PATCH 15/18] tape: Decode OTAs into any, and allow assignment of SBA/LBA to string --- tape/dynamic.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index 9712b14..52f833d 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -101,6 +101,7 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) { // DecodeAnyInto decodes data and places it into destination, which must be a // pointer to a supported type. See [EncodeAny] for a list of supported types. +// The head of the decoder must be at the start of the payload. func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error) { reflectDestination := reflect.ValueOf(destination) if reflectDestination.Kind() != reflect.Pointer { @@ -110,7 +111,8 @@ func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error } // DecodeAny is like [DecodeAnyInto], but it automatically creates the -// destination from the tag and data. +// destination from the tag and data. The head of the decoder must be at the +// start of the payload. func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) { destination, err := skeletonValue(decoder, tag) if err != nil { return nil, n, err } @@ -128,7 +130,8 @@ var unknownSlicePlaceholderType = reflect.TypeOf(unknownSlicePlaceholder { }) // decodeAny is internal to [DecodeAny]. It takes in an addressable // [reflect.Value] as the destination. If the decoded value cannot fit in the // destination, it skims over the payload, leaves the destination empty, and -// returns without an error. +// returns without an error. The head of the decoder must be at the start of the +// payload. func decodeAny(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err error) { n, err = decodeAnyOrError(decoder, destination, tag) if _, ok := err.(errCantAssign); ok { @@ -145,7 +148,7 @@ func decodeAny(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err // destination, it decodes nothing and returns an error of type errCantAssign, // except for the case of a mismatched OTA element tag, wherein it will skim // over the rest of the payload, leave the destination empty, and return without -// an error. +// an error. The head of the decoder must be at the start of the payload. func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err error) { err = canSet(destination.Type(), tag) if err != nil { return n, err } @@ -189,6 +192,13 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i setString(destination, string(buffer)) case OTA: // OTA: * + if isTypeAny(destination.Type()) { + // need a skeleton value if we are assigning to any. + value, err := skeletonValue(decoder, tag) + if err != nil { return n, err } + destination.Set(value) + destination = value.Elem() + } length, nn, err := decoder.ReadUintN(tag.CN() + 1) n += nn; if err != nil { return n, err } if length > uint64(MaxStructureLength) { @@ -356,6 +366,7 @@ func canSet(destination reflect.Type, tag Tag) error { return errCantAssignf("cannot assign float to %v", destination) } case SBA, LBA: + if destination.Kind() == reflect.String { return nil } if destination.Kind() != reflect.Slice { return errCantAssignf("cannot assign byte array to %v", destination) } @@ -463,7 +474,8 @@ func decodeAndSetFloat(decoder *Decoder, destination reflect.Value, bytes int) ( } // skeletonValue returns a pointer value. In order for it to be set, it must be -// dereferenced using Elem(). +// dereferenced using Elem(). The head of the decoder must be at the start of +// the payload. func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) { typ, err := typeOf(decoder, tag) if err != nil { return reflect.Value { }, err } @@ -471,7 +483,8 @@ func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) { } // typeOf returns the type of the current tag being decoded. It does not use up -// the decoder, it only peeks. +// the decoder, it only peeks. The head of the decoder must be at the start of +// the payload. func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) { switch tag.WithoutCN() { case SI: -- 2.50.1 From 3f51beddb653e21212b7b6e60262ed795823d311 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 11:26:59 -0400 Subject: [PATCH 16/18] tape: Decoding OTA into any no longer results in a pointer --- tape/dynamic.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index 52f833d..120eb50 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -114,7 +114,7 @@ func DecodeAnyInto(decoder *Decoder, destination any, tag Tag) (n int, err error // destination from the tag and data. The head of the decoder must be at the // start of the payload. func DecodeAny(decoder *Decoder, tag Tag) (value any, n int, err error) { - destination, err := skeletonValue(decoder, tag) + destination, err := skeletonPointer(decoder, tag) if err != nil { return nil, n, err } nn, err := DecodeAnyInto(decoder, destination, tag) n += nn; if err != nil { return nil, n, err } @@ -192,12 +192,12 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i setString(destination, string(buffer)) case OTA: // OTA: * + oldDestination := destination if isTypeAny(destination.Type()) { // need a skeleton value if we are assigning to any. value, err := skeletonValue(decoder, tag) if err != nil { return n, err } - destination.Set(value) - destination = value.Elem() + destination = value } length, nn, err := decoder.ReadUintN(tag.CN() + 1) n += nn; if err != nil { return n, err } @@ -232,6 +232,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i return n, err } } + oldDestination.Set(destination) case KTV: // KTV: ( )* length, nn, err := decoder.ReadUintN(tag.CN() + 1) @@ -245,7 +246,7 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i n += nn; if err != nil { return n, err } itemTag, nn, err := decoder.ReadTag() n += nn; if err != nil { return n, err } - value, err := skeletonValue(decoder, itemTag) + value, err := skeletonPointer(decoder, itemTag) if err != nil { return n, err } nn, err = decodeAny(decoder, value.Elem(), itemTag) n += nn; if err != nil { return n, err } @@ -473,10 +474,18 @@ func decodeAndSetFloat(decoder *Decoder, destination reflect.Value, bytes int) ( return n, errCantAssignf("unsupported bit width float%d", bytes * 8) } -// skeletonValue returns a pointer value. In order for it to be set, it must be -// dereferenced using Elem(). The head of the decoder must be at the start of -// the payload. +// skeletonValue returns an addressable value. It can be set directly. The head +// of the decoder must be at the start of the payload when calling. func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) { + ptr, err := skeletonPointer(decoder, tag) + if err != nil { return reflect.Value { }, err } + return ptr.Elem(), nil +} + +// skeletonPointer returns a pointer value. In order for it to be set, it must +// be dereferenced using Elem(). The head of the decoder must be at the start of +// the payload when calling. +func skeletonPointer(decoder *Decoder, tag Tag) (reflect.Value, error) { typ, err := typeOf(decoder, tag) if err != nil { return reflect.Value { }, err } return reflect.New(typ), nil @@ -484,7 +493,7 @@ func skeletonValue(decoder *Decoder, tag Tag) (reflect.Value, error) { // typeOf returns the type of the current tag being decoded. It does not use up // the decoder, it only peeks. The head of the decoder must be at the start of -// the payload. +// the payload when calling. func typeOf(decoder *Decoder, tag Tag) (reflect.Type, error) { switch tag.WithoutCN() { case SI: -- 2.50.1 From ef3f5cf4bb8c039f416651511365ae7b5c1f91d3 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 11:34:49 -0400 Subject: [PATCH 17/18] tape: Decode KTV into any --- tape/dynamic.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tape/dynamic.go b/tape/dynamic.go index 120eb50..31966b6 100644 --- a/tape/dynamic.go +++ b/tape/dynamic.go @@ -240,8 +240,16 @@ func decodeAnyOrError(decoder *Decoder, destination reflect.Value, tag Tag) (n i if length > uint64(MaxStructureLength) { return 0, ErrTooLong } + lengthCast, err := Uint64ToIntSafe(length) + if err != nil { return n, err } + if isTypeAny(destination.Type()) { + // need a skeleton value if we are assigning to any. + value := reflect.MakeMapWithSize(reflect.TypeOf(dummyMap), lengthCast) + destination.Set(value) + destination = value + } destination.Clear() - for _ = range length { + for _ = range lengthCast { key, nn, err := decoder.ReadUint16() n += nn; if err != nil { return n, err } itemTag, nn, err := decoder.ReadTag() -- 2.50.1 From aebc6972ad75117ed16dce194a70d9ef1915a476 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 12 Oct 2025 11:44:14 -0400 Subject: [PATCH 18/18] tape: Fix TestEncodeDecodeAnyTable --- tape/dynamic_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tape/dynamic_test.go b/tape/dynamic_test.go index a0078ae..a068563 100644 --- a/tape/dynamic_test.go +++ b/tape/dynamic_test.go @@ -208,7 +208,7 @@ func TestEncodeDecodeAnyTable(test *testing.T) { 0x0102: uint32(2), 0x0103: int64(23432), 0x0104: int64(-88777), - 0x0000: []byte("hi!"), + 0x0000: "hi!", 0xFFFF: []uint16 { 0xBEE5, 0x7777 }, 0x1234: [][]uint16 { []uint16 { 0x5 }, []uint16 { 0x17, 0xAAAA} }, 0x1F1F: float32(67.26), -- 2.50.1