From dcbfbe9141f638443813971ecdf303532de24e87 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 3 Aug 2025 20:59:59 -0400 Subject: [PATCH 01/24] design: Import issue from Tebitea (it is down) --- design/branched-generated-encoder.md | 123 +++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 design/branched-generated-encoder.md diff --git a/design/branched-generated-encoder.md b/design/branched-generated-encoder.md new file mode 100644 index 0000000..9360b6f --- /dev/null +++ b/design/branched-generated-encoder.md @@ -0,0 +1,123 @@ +# Branched Generated Decoder + +Pasted here because Tebitea is down + +## The problem + +TAPE is designed so that the decoder can gloss over data it does not understand. +Technically the protocol allows for this, but I completely forgot to implement +this in the generated decoder, oops. This would be trivial if TAPE messages were +still flat tables, but they aren't, because those aren't useful enough. So, +let's analyze the problem. + +## When it happens + +There are two reasons something might not match up with the expected data: + +The first and most obvious is unrecognized keys. If the key is not in the set of +recognized keys for a KTV, it should leave the corresponding struct field blank. +Once #6 has been implemented, throw an error if the data was not optional. + +The second is wrong types. If we are expecting KTV and get SBA, we should leave +the data as empty. The aforementioned concern about #6 also applies here. We +don't need to worry about special cases at the structure root, because it would +be technically possible to make the structure root an option, so it really is +just a normal value. Until #6, we will leave that blank too. + +## Preliminary ideas + +The first is going to be pretty simple. All we need to do is have a skimmer +function that skims over TAPE data very, and then call that on the KTV value +each time we run into a mystery key. It should only return an error if the +structure of the data is malformed in such a way that it cannot continue to the +next one. This should be stored in the tape package alongside the dynamic +decoding functions, because they will essentially function the same way and +could probably share lots of code. + +The second is a bit more complicated because of the existence of KTV and OTA +because they are aggregate types. Go types work a bit differently, as if you +have an array of an array of an array of ints, that information is represented +in one place, whereas TAPE doesn't really do that. All of that information is +sort of buried within the data structure, so we don't know what we will be +decoding before we actually do it. Whenever we encounter a type we don't expect, +we would need to abort decoding of the entire data structure, and then skim over +whatever detritus is left, which would literally be in a half-decoded state. The +fact that the code is generated flat and thus cannot use return or defer +statements contributes to the complexity of this problem. We need to go up, but +we can't. There is no up, only forward. + +Of course, the dynamic decoder does not have this problem in the first place +because it doesn't expect anything, and constructs the destination to fit +whatever it sees in the TAPE structure as it is decoding it. KTVs are completely +dynamic because they are implemented as maps, so the only time it needs to +completely comprehend a type is with OTAs. There is a function called typeOf +that gets the type of the current tag and returns it as a reflect.Type, which +necessitates recursion and peeking at OTAs and their elements. + +We could try to do the same thing in the generated decoder, comparing the +determined type against the expected type to try to figure out whether we should +decode an array or a table, etc. This is immediately problematic as it requires +memory to be allocated, both for the peek buffer and the resulting tree of type +information. If we end up with some crazy way to keep track of the types, that's +only one half of the allocation problem and we would still be spending extra +cycles going over all of that twice. + +## Performance constraints + +The generated decoder is supposed to blaze through data, and it can't do that if +it does all the singing and dancing that the dynamic decoder does. It's time for +some performance constraints: + +- No allocations, except as required to build the destination for the data +- No redundant work +- So, no freaking peeking +- It should take well under 500 lines of generated code to decode one message of +reasonable size (i.e. be careful not to bloat the binary) + +I'm not really going to do my usual thing here of making a slow version and +speeding it up over time based on evidence and experimentation because these +constraints inform the design so much it would be impossible to continue without +them. I am 99% confident that these constraints will allow for an acceptable +baseline of performance (for generated code) and we can still profile and +micro-optimize later. This is good enough for me. +Heavy solution + +There is a solution that might work very well which involves completely redoing +the generated decoding code. We could create a function for every source type to +destination type mapping that exists in protocol, and then compose them all +together. The decoding methods for each message or type would be wrappers around +the correct function for their root TAPE -> Go type mapping. The main benefit of +this is it would make this problem a lot more manageable because the interface +points between the data would be represented by function boundaries. This would +allow the use of return and defer statements, and would allow more code sharing, +producing a smaller binary. Go would probably inline these where needed. + +Would this work? Probably. More investigation is required to make sure. I want +to stop re-writing things I don't need to. On the other hand, it is just the +decoder. + +## Light solution + +TODO: find a solution that satisfies the performance constraints, keeps the same +identical interface, and works off the same code. I am convinced this is doable, +and it might even allow us to extract more data from an unexpected structure. +However, continuing this way might introduce unmanageable complexity. It is +already a little unmanageable and I am just one pony (kind of). + +## Implementation + +Heavy solution is going to work here, applied to only the points of +`Generator.generateDecodeValue` where it decodes an aggregate data structure. +That way, only minimal amounts of code need to be redone. + +Whenever a branch needs to happen, a call shall be generated, a deferred +implementation request shall be added to a special FIFO queue within the +generator. After generating data structures and their root decoding functions, +the generator shall pick away at this queue until no requests remain. The +generator shall accept new items during this process, so that recursion is +possible. This is all to ensure it is only ever writing one function at a time + +The functions shall take a pointer to a type that accepts any type like (~) the +destination's base type. We should also probably just call +`Generator.generateDecodeValue` directly on user defined types this way, keeping +their public `Decode` methods just for convenience. -- 2.50.1 From c86f9b03f2ba199f132e0b4270ea2dd03a10fa1d Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 3 Aug 2025 21:00:24 -0400 Subject: [PATCH 02/24] generated: Remove unneeded code --- generate/generate.go | 113 +------------------------------------------ 1 file changed, 2 insertions(+), 111 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 1e99f4c..fa5c12a 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -485,50 +485,7 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource, abo } case TypeArray: // OTA: * - lengthVar := this.newTemporaryVar("length") - nn, err := this.iprintf("var %s uint64\n", lengthVar) - n += nn; if err != nil { return n, err } - nn, err = this.iprintf( - "%s, nn, err = decoder.ReadUintN(int(%s.CN()))\n", - lengthVar, tagSource) - n += nn; if err != nil { return n, err } - nn, err = this.generateErrorCheck() - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("*%s = make(", valueSource) - n += nn; if err != nil { return n, err } - nn, err = this.generateType(typ) - n += nn; if err != nil { return n, err } - nn, err = this.printf(", int(%s))\n", lengthVar) - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("var itemTag tape.Tag\n") - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("itemTag, nn, err = decoder.ReadTag()\n") - n += nn; if err != nil { return n, err } - nn, err = this.generateErrorCheck() - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("if !itemTag.Is(") - n += nn; if err != nil { return n, err } - nn, err = this.generateTN(typ.Element) - n += nn; if err != nil { return n, err } - nn, err = this.iprintf(") {\n") - n += nn; if err != nil { return n, err } - this.push() - nn, err = this.iprintf("%s\n", abort) - n += nn; if err != nil { return n, err } - this.pop() - nn, err = this.iprintf("}\n") - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("for index := range %s {\n", lengthVar) - n += nn; if err != nil { return n, err } - this.push() - nn, err = this.generateDecodeValue( - typ.Element, - fmt.Sprintf("(&(*%s)[index])", valueSource), - "itemTag", abort) - n += nn; if err != nil { return n, err } - this.pop() - nn, err = this.iprintf("}\n") - n += nn; if err != nil { return n, err } + // TODO: branch case TypeTable: // KTV: ( )* nn, err := this.iprintf( @@ -539,73 +496,7 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource, abo n += nn; if err != nil { return n, err } case TypeTableDefined: // KTV: ( )* - lengthVar := this.newTemporaryVar("length") - nn, err := this.iprintf("var %s uint64\n", lengthVar) - n += nn; if err != nil { return n, err } - nn, err = this.iprintf( - "%s, nn, err = decoder.ReadUintN(int(%s.CN()))\n", - lengthVar, tagSource) - n += nn; if err != nil { return n, err } - nn, err = this.generateErrorCheck() - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("for _ = range %s {\n", lengthVar) - n += nn; if err != nil { return n, err } - this.push() - nn, err = this.iprintf("var key uint16\n") - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("key, nn, err = decoder.ReadUint16()\n") - n += nn; if err != nil { return n, err } - nn, err = this.generateErrorCheck() - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("var itemTag tape.Tag\n") - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("itemTag, nn, err = decoder.ReadTag()\n") - n += nn; if err != nil { return n, err } - nn, err = this.generateErrorCheck() - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("switch key {\n") - n += nn; if err != nil { return n, err } - keys := slices.Collect(maps.Keys(typ.Fields)) - slices.Sort(keys) - for _, key := range keys { - field := typ.Fields[key] - nn, err = this.iprintf("case 0x%04X:\n", key) - n += nn; if err != nil { return n, err } - this.push() - labelVar := this.newTemporaryVar("label") - fieldAbort := fmt.Sprintf("goto %s", labelVar) // TODO: skip value somehow - nn, err = this.iprintf("if !itemTag.Is(") - n += nn; if err != nil { return n, err } - nn, err = this.generateTN(field.Type) - n += nn; if err != nil { return n, err } - nn, err = this.printf(") {\n") - n += nn; if err != nil { return n, err } - this.push() - nn, err = this.iprintf("%s\n", fieldAbort) - n += nn; if err != nil { return n, err } - this.pop() - nn, err = this.iprintf("}\n") - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("{\n") - n += nn; if err != nil { return n, err } - this.push() - nn, err = this.generateDecodeValue( - field.Type, - fmt.Sprintf("(&%s.%s)", valueSource, field.Name), - "itemTag", fieldAbort) - n += nn; if err != nil { return n, err } - this.pop() - nn, err = this.iprintf("}\n") - n += nn; if err != nil { return n, err } - nn, err = this.iprintf("%s:;\n", labelVar) - n += nn; if err != nil { return n, err } - this.pop() - } - nn, err = this.iprintf("}\n") - n += nn; if err != nil { return n, err } - this.pop() - nn, err = this.iprintf("}\n") - n += nn; if err != nil { return n, err } + // TODO: branch case TypeNamed: // WHATEVER: [WHATEVER] nn, err := this.iprintf("nn, err = %s.DecodeValue(decoder, %s)\n", valueSource, tagSource) -- 2.50.1 From fae702edfd83472ac18a9496363223eacd0d3456 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 3 Aug 2025 22:07:31 -0400 Subject: [PATCH 03/24] generate: Add String functions, TypeHash function for types --- generate/protocol.go | 62 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/generate/protocol.go b/generate/protocol.go index 1c3ebc4..1610c86 100644 --- a/generate/protocol.go +++ b/generate/protocol.go @@ -1,5 +1,10 @@ package generate +import "fmt" +import "maps" +import "slices" +import "crypto/md5" + type Protocol struct { Messages map[uint16] Message Types map[string] Type @@ -11,7 +16,7 @@ type Message struct { } type Type interface { - + fmt.Stringer } type TypeInt struct { @@ -19,29 +24,84 @@ type TypeInt struct { Signed bool } +func (typ TypeInt) String() string { + output := "" + if typ.Signed { + output += "I" + } else { + output += "U" + } + output += fmt.Sprint(typ.Bits) + return output +} + type TypeFloat struct { Bits int } +func (typ TypeFloat) String() string { + return fmt.Sprintf("F%d", typ.Bits) +} + type TypeString struct { } +func (TypeString) String() string { + return "String" +} + type TypeBuffer struct { } +func (TypeBuffer) String() string { + return "Buffer" +} + type TypeArray struct { Element Type } +func (typ TypeArray) String() string { + return fmt.Sprintf("[]%v", typ.Element) +} + type TypeTable struct { } +func (TypeTable) String() string { + return "Table" +} + type TypeTableDefined struct { Fields map[uint16] Field } +func (typ TypeTableDefined) String() string { + output := "{" + for _, key := range slices.Sorted(maps.Keys(typ.Fields)) { + output += fmt.Sprintf("%04X %v", key, typ.Fields[key]) + } + output += "}" + return output +} + type Field struct { Name string Type Type } +func (field Field) String() string { + return fmt.Sprintf("%s %v", field.Name, field.Type) +} + type TypeNamed struct { Name string } + +func (typ TypeNamed) String() string { + return typ.Name +} + +func HashType(typ Type) [16]byte { + // TODO: if we ever want to make the compiler more efficient, this would + // be a good place to start, complex string concatenation in a hot path + // (sorta) + return md5.Sum([]byte(typ.String())) +} -- 2.50.1 From 41b3376fa3a1b99ad0699f32595e8b6be9bb6822 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 3 Aug 2025 22:19:06 -0400 Subject: [PATCH 04/24] generate: Add Generator.generateDecodeBranch stub --- generate/generate.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index fa5c12a..0ceb666 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -485,7 +485,8 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource, abo } case TypeArray: // OTA: * - // TODO: branch + nn, err := this.generateDecodeBranch(typ, valueSource, tagSource) + n += nn; if err != nil { return n, err } case TypeTable: // KTV: ( )* nn, err := this.iprintf( @@ -496,7 +497,8 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource, abo n += nn; if err != nil { return n, err } case TypeTableDefined: // KTV: ( )* - // TODO: branch + nn, err := this.generateDecodeBranch(typ, valueSource, tagSource) + n += nn; if err != nil { return n, err } case TypeNamed: // WHATEVER: [WHATEVER] nn, err := this.iprintf("nn, err = %s.DecodeValue(decoder, %s)\n", valueSource, tagSource) @@ -510,6 +512,23 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource, abo return n, nil } + +// generateDencodeValue generates code to call an aggregate decoder function, +// for a specified type. The definition of the function is deferred so no +// duplicates are created. The function overwrites memory pointed to by the +// variable (or parenthetical statement) specified by valueSource, and the value +// will be encoded according to the tag stored in the variable (or parenthetical +// statement) specified by tagSource. the code generated is a BLOCK and expects +// these variables to be defined: +// +// - decoder *tape.Decoder +// - n int +// - err error +// - nn int +func (this *Generator) generateDecodeBranch(typ Type, valueSource, tagSource string) (n int, err error) { + // TODO +} + func (this *Generator) generateErrorCheck() (n int, err error) { return this.iprintf("n += nn; if err != nil { return n, err }\n") } -- 2.50.1 From df3fe1280dd308256bfcda109bfe0a41cf16e396 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 3 Aug 2025 22:27:14 -0400 Subject: [PATCH 05/24] generate: Remove abort parameter --- generate/generate.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 0ceb666..685308e 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -146,7 +146,7 @@ func (this *Generator) generateTypedef(name string, typ Type) (n int, err error) this.push() nn, err = this.iprintf("var nn int\n") n += nn; if err != nil { return n, err } - nn, err = this.generateDecodeValue(typ, "this", "tag", "return n, nil") + nn, err = this.generateDecodeValue(typ, "this", "tag") n += nn; if err != nil { return n, err } nn, err = this.iprintf("return n, nil\n") n += nn; if err != nil { return n, err } @@ -217,20 +217,16 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - abort := "return n, nil" // TODO: skip value somehow nn, err = this.iprintf("if !tag.Is(") n += nn; if err != nil { return n, err } nn, err = this.generateTN(message.Type) n += nn; if err != nil { return n, err } nn, err = this.printf(") {\n") n += nn; if err != nil { return n, err } - this.push() - nn, err = this.iprintf("%s\n", abort) - n += nn; if err != nil { return n, err } - this.pop() + // TODO skip value using the correct TAPE function and return nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } - nn, err = this.generateDecodeValue(message.Type, "this", "tag", abort) + nn, err = this.generateDecodeValue(message.Type, "this", "tag") n += nn; if err != nil { return n, err } nn, err = this.iprintf("return n, nil\n") n += nn; if err != nil { return n, err } @@ -419,7 +415,7 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri // - n int // - err error // - nn int -func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource, abort string) (n int, err error) { +func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource string) (n int, err error) { switch typ := typ.(type) { case TypeInt: // SI: (none) -- 2.50.1 From a1bfae443cc1bf7dc6cb26bd98a440423201bfa0 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Sun, 3 Aug 2025 22:27:27 -0400 Subject: [PATCH 06/24] design: Add paragraph about how we need a skimming function in tape --- design/branched-generated-encoder.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/design/branched-generated-encoder.md b/design/branched-generated-encoder.md index 9360b6f..5bacc24 100644 --- a/design/branched-generated-encoder.md +++ b/design/branched-generated-encoder.md @@ -121,3 +121,8 @@ The functions shall take a pointer to a type that accepts any type like (~) the destination's base type. We should also probably just call `Generator.generateDecodeValue` directly on user defined types this way, keeping their public `Decode` methods just for convenience. + +The tape package shall contain a skimming function that takes a decoder and a +tag, and recursively consumes the decoder given the context of the tag. This +shall be utilized by the decoder functions to skip over values if their tags +or keys do not match up with what is expected. -- 2.50.1 From b15c3aa76c4acc5ccc889daba9b352ae34e8b7c1 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 4 Aug 2025 09:36:52 -0400 Subject: [PATCH 07/24] generate: Implement Generator.generateDecodeBranch --- generate/generate.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/generate/generate.go b/generate/generate.go index 685308e..ae36942 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -6,6 +6,7 @@ import "maps" import "math" import "slices" import "strings" +import "encoding/hex" import "git.tebibyte.media/sashakoshka/hopp/tape" const imports = @@ -79,6 +80,8 @@ func (this *Generator) Generate(protocol *Protocol) (n int, err error) { n += nn; if err != nil { return n, err } } + // TODO: pull items from the request queue + return n, nil } @@ -522,7 +525,17 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri // - err error // - nn int func (this *Generator) generateDecodeBranch(typ Type, valueSource, tagSource string) (n int, err error) { - // TODO + hash := (HashType(typ)) + hashString := hex.EncodeToString(hash[:]) + nn, err := this.iprintf("nn, err = decodeBranch_%(%s, %s)\n", hashString, valueSource, tagSource) + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + // TODO: define this function to check if the hash exists in the + // request queue, and if not, put it at the back. then, when we actually + // generate these, continually pick items from the front. + this.pushDecodeBranchRequest(hash, typ) + return n, nil } func (this *Generator) generateErrorCheck() (n int, err error) { -- 2.50.1 From 711ac30486487856fbc990a674d1b7bb99ea3880 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 4 Aug 2025 12:26:16 -0400 Subject: [PATCH 08/24] generate: Add branch decode function request queue --- generate/generate.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index ae36942..8cb081a 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -47,6 +47,13 @@ type Generator struct { nestingLevel int temporaryVar int protocol *Protocol + + decodeBranchRequestQueue []decodeBranchRequest +} + +type decodeBranchRequest struct { + hash [16]byte + typ Type } func (this *Generator) Generate(protocol *Protocol) (n int, err error) { @@ -531,13 +538,29 @@ func (this *Generator) generateDecodeBranch(typ Type, valueSource, tagSource str n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - // TODO: define this function to check if the hash exists in the - // request queue, and if not, put it at the back. then, when we actually - // generate these, continually pick items from the front. this.pushDecodeBranchRequest(hash, typ) return n, nil } +func (this *Generator) pushDecodeBranchRequest(hash [16]byte, typ Type) { + for _, item := range this.decodeBranchRequestQueue { + if item.hash == hash { return } + } + this.decodeBranchRequestQueue = append(this.decodeBranchRequestQueue, decodeBranchRequest { + hash: hash, + typ: typ, + }) +} + +func (this *Generator) pullDecodeBranchRequest() (hash [16]byte, typ Type, ok bool) { + if len(this.decodeBranchRequestQueue) < 1 { + return [16]byte { }, nil, false + } + request := this.decodeBranchRequestQueue[0] + this.decodeBranchRequestQueue = this.decodeBranchRequestQueue[1:] + return request.hash, request.typ, true +} + func (this *Generator) generateErrorCheck() (n int, err error) { return this.iprintf("n += nn; if err != nil { return n, err }\n") } -- 2.50.1 From 7dcfc0867862fd071a712a2f5cd31d6ec8d4d373 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Mon, 4 Aug 2025 16:01:50 -0400 Subject: [PATCH 09/24] generate: Add "stub" for actually generating branch functions --- generate/generate.go | 55 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 8cb081a..427f638 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -87,7 +87,13 @@ func (this *Generator) Generate(protocol *Protocol) (n int, err error) { n += nn; if err != nil { return n, err } } - // TODO: pull items from the request queue + // request queue + for { + hash, typ, ok := this.pullDecodeBranchRequest() + if !ok { break } + nn, err := this.generateDecodeBranch(hash, typ) + n += nn; if err != nil { return n, err } + } return n, nil } @@ -491,7 +497,7 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri } case TypeArray: // OTA: * - nn, err := this.generateDecodeBranch(typ, valueSource, tagSource) + nn, err := this.generateDecodeBranchCall(typ, valueSource, tagSource) n += nn; if err != nil { return n, err } case TypeTable: // KTV: ( )* @@ -503,7 +509,7 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri n += nn; if err != nil { return n, err } case TypeTableDefined: // KTV: ( )* - nn, err := this.generateDecodeBranch(typ, valueSource, tagSource) + nn, err := this.generateDecodeBranchCall(typ, valueSource, tagSource) n += nn; if err != nil { return n, err } case TypeNamed: // WHATEVER: [WHATEVER] @@ -518,8 +524,7 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri return n, nil } - -// generateDencodeValue generates code to call an aggregate decoder function, +// generateDecodeBranchCall generates code to call an aggregate decoder function, // for a specified type. The definition of the function is deferred so no // duplicates are created. The function overwrites memory pointed to by the // variable (or parenthetical statement) specified by valueSource, and the value @@ -531,10 +536,9 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri // - n int // - err error // - nn int -func (this *Generator) generateDecodeBranch(typ Type, valueSource, tagSource string) (n int, err error) { - hash := (HashType(typ)) - hashString := hex.EncodeToString(hash[:]) - nn, err := this.iprintf("nn, err = decodeBranch_%(%s, %s)\n", hashString, valueSource, tagSource) +func (this *Generator) generateDecodeBranchCall(typ Type, valueSource, tagSource string) (n int, err error) { + hash := HashType(typ) + nn, err := this.iprintf("nn, err = %s(%s, %s)\n", this.decodeBranchName(hash), valueSource, tagSource) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } @@ -542,6 +546,37 @@ func (this *Generator) generateDecodeBranch(typ Type, valueSource, tagSource str return n, nil } +// generateDecodeBranch generates an aggregate decoder function definition for a +// specified type. It assumes that hash == HashType(typ). +func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err error) { + nn, err := this.iprintf("func %s(this *", this.decodeBranchName(hash)) + n += nn; if err != nil { return n, err } + nn, err = this.generateType(typ) + n += nn; if err != nil { return n, err } + nn, err = this.printf(" decoder *tape.Decoder, tag tape.Tag) (nn int, err error) \n{") + n += nn; if err != nil { return n, err } + + switch typ := typ.(type) { + case TypeArray: + // OTA: * + + case TypeTableDefined: + // KTV: ( )* + // TODO + default: return n, fmt.Errorf("unexpected type: %T", typ) + } + + nn, err = this.iprintf("}") + n += nn; if err != nil { return n, err } + return n, nil +} + +func (this *Generator) decodeBranchName(hash [16]byte) string { + return fmt.Sprintf("decodeBranch_%s", hex.EncodeToString(hash[:])) +} + +// pushDecodeBranchRequest pushes a new branch decode function request to the +// back of the queue, if it is not already in the queue. func (this *Generator) pushDecodeBranchRequest(hash [16]byte, typ Type) { for _, item := range this.decodeBranchRequestQueue { if item.hash == hash { return } @@ -552,6 +587,8 @@ func (this *Generator) pushDecodeBranchRequest(hash [16]byte, typ Type) { }) } +// pullDecodeBranchRequest pulls a branch decode function request from the front +// of the queue. func (this *Generator) pullDecodeBranchRequest() (hash [16]byte, typ Type, ok bool) { if len(this.decodeBranchRequestQueue) < 1 { return [16]byte { }, nil, false -- 2.50.1 From 2cbf58d55808703c42f4f5c4a332362eac70cee6 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Tue, 5 Aug 2025 06:22:27 -0400 Subject: [PATCH 10/24] generate: Decode arrays (but don't validate their length yet) --- generate/generate.go | 89 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 9 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 427f638..7919219 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -555,17 +555,88 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err n += nn; if err != nil { return n, err } nn, err = this.printf(" decoder *tape.Decoder, tag tape.Tag) (nn int, err error) \n{") n += nn; if err != nil { return n, err } + this.push() - switch typ := typ.(type) { - case TypeArray: - // OTA: * + nn, err = this.iprintf("var nn int\n") + n += nn; if err != nil { return n, err } - case TypeTableDefined: - // KTV: ( )* - // TODO - default: return n, fmt.Errorf("unexpected type: %T", typ) - } - + switch typ := typ.(type) { + case TypeArray: + // OTA: * + // read header + lengthVar := this.newTemporaryVar("length") + nn, err := this.iprintf("var %s uint64\n", lengthVar) + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar) + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + elementTagVar := this.newTemporaryVar("elementTag") + nn, err = this.iprintf("var %s uint64\n", lengthVar) + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("%s, nn, err = decoder.ReadTag()\n", elementTagVar) + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + + // abort macro + abort := func() (n int, err error) { + // skim entire array + nn, err = this.iprintf("for _ = range %s {\n", lengthVar) + n += nn; if err != nil { return n, err } + this.push() + nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)") + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + this.pop() + nn, err = this.iprintf("}\n") + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("return n, nil") + n += nn; if err != nil { return n, err } + return n, nil + } + + // validate header + // TODO: here, validate that length is less than the + // max, whatever that is configured to be. the reason we + // want to read it here is that we would have to skip + // the tag anyway so why not. + nn, err = this.iprintf("if %s != ", elementTagVar) + n += nn; if err != nil { return n, err } + nn, err = this.generateTag(typ.Element, "(*this)") + n += nn; if err != nil { return n, err } + nn, err = this.printf(" {\n") + n += nn; if err != nil { return n, err } + this.push() + nn, err = abort() + n += nn; if err != nil { return n, err } + this.pop() + nn, err = this.iprintf("}\n") + n += nn; if err != nil { return n, err } + + // decode payloads + nn, err = this.iprintf("*this = make(") + n += nn; if err != nil { return n, err } + nn, err = this.generateType(typ) + n += nn; if err != nil { return n, err } + nn, err = this.printf(", %s)\n", lengthVar) + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("for index := range %s {\n", lengthVar) + n += nn; if err != nil { return n, err } + this.push() + nn, err = this.generateDecodeValue(typ.Element, "(*this)[index]", elementTagVar) + n += nn; if err != nil { return n, err } + this.pop() + nn, err = this.iprintf("}\n") + n += nn; if err != nil { return n, err } + case TypeTableDefined: + // KTV: ( )* + // TODO + default: return n, fmt.Errorf("unexpected type: %T", typ) + } + + this.pop() nn, err = this.iprintf("}") n += nn; if err != nil { return n, err } return n, nil -- 2.50.1 From c4dd129fc50949cf6d34ddbf29a2ef00b3c4d99e Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 17:00:39 -0400 Subject: [PATCH 11/24] generate: Decode tables (but don't validate their length yet) --- generate/generate.go | 130 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/generate/generate.go b/generate/generate.go index 7919219..3fd2a73 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -34,6 +34,19 @@ type Message interface { // Method returns the method code of the message. Method() uint16 } + +// canAssign determines if data from the given source tag can be assigned to +// a Go type represented by destination. It is designed to receive destination +// values from [generate.Generator.generateCanAssign]. The eventual Go type and +// the destination tag must come from the same (or hash-equivalent) PDL type. +func canAssign(destination, source tape.Tag) bool { + if destination.Is(source) { return true } + if (destination == tape.SBA || destination == tape.LBA) && + (source == tape.SBA || source == tape.LBA) { + return true + } + return false +} ` // Generator converts protocols into Go code. @@ -585,6 +598,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err nn, err = this.iprintf("for _ = range %s {\n", lengthVar) n += nn; if err != nil { return n, err } this.push() + // FIXME: wrong tag var nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)") n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() @@ -597,6 +611,12 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err return n, nil } + // FIXME: the tag is validated improperly, the result of + // generateTag means basically nothing here because + // there is no data in the destination, we need another + // function that checks if a tag can be assigned to a + // PDL type + // validate header // TODO: here, validate that length is less than the // max, whatever that is configured to be. the reason we @@ -632,7 +652,102 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err n += nn; if err != nil { return n, err } case TypeTableDefined: // KTV: ( )* - // TODO + // read header + lengthVar := this.newTemporaryVar("length") + nn, err := this.iprintf("var %s uint64\n", lengthVar) + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("%s, nn, err = decoder.ReadUintN(int(tag.CN()))\n", lengthVar) + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + indexVar := this.newTemporaryVar("index") + nn, err = this.iprintf("%s := 0\n", indexVar) + n += nn; if err != nil { return n, err } + + // validate header + // TODO: here, validate that length is less than the + // max, whatever that is configured to be. if not, stop + // ALL decoding. skimming huge big ass data could cause + // problems + + // read fields + nn, err = this.iprintf("for index = range %s {\n", lengthVar) + n += nn; if err != nil { return n, err } + this.push() + // read field header + fieldKeyVar := this.newTemporaryVar("fieldKey") + nn, err = this.iprintf("var %s uint16\n", fieldKeyVar) + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("%s, nn, err = decoder.ReadUint16()\n", fieldKeyVar) + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + fieldTagVar := this.newTemporaryVar("fieldTag") + nn, err = this.iprintf("var %s tape.Tag\n", fieldTagVar) + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("%s, nn, err = decoder.ReadTag()\n", fieldTagVar) + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + + // abort field macro + abortField := func() (n int, err error) { + nn, err = this.iprintf("tape.Skim(decoder, %s)", fieldTagVar) + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("continue") + n += nn; if err != nil { return n, err } + return n, nil + } + + // switch on tag + nn, err = this.iprintf("switch %s {\n", fieldTagVar) + n += nn; if err != nil { return n, err } + this.push() + for _, key := range slices.Sorted(maps.Keys(typ.Fields)) { + field := typ.Fields[key] + nn, err = this.iprintf("case %s:\n", fieldTagVar) + n += nn; if err != nil { return n, err } + this.push() + + // validate field header + nn, err = this.iprintf("if !(") + n += nn; if err != nil { return n, err } + nn, err = this.generateCanAssign(field.Type, fieldTagVar) + n += nn; if err != nil { return n, err } + nn, err = this.printf(") {\n") + n += nn; if err != nil { return n, err } + this.push() + nn, err = abortField() + n += nn; if err != nil { return n, err } + this.pop() + nn, err = this.iprintf("}\n") + n += nn; if err != nil { return n, err } + + // decode payload + nn, err = this.generateDecodeValue(field.Type, fmt.Sprintf("&(this.%s)", field.Name), fieldTagVar) + n += nn; if err != nil { return n, err } + this.pop() + } + nn, err = this.iprintf("default:\n") + n += nn; if err != nil { return n, err } + this.push() + abortField() + this.pop() + nn, err = this.iprintf("}\n") + n += nn; if err != nil { return n, err } + this.pop() + nn, err = this.iprintf("}\n") + n += nn; if err != nil { return n, err } + + // TODO once options are implemented, have a set of + // bools for each non-optional field, and check here + // that they are all true. a counter will not work + // because if someone specifies a non-optional field + // twice, they can neglect to specify another + // non-optional field and we won't even know because the + // count will still be even. we shouldn't use a map + // either because its an allocation and its way more + // memory than just, like 5 bools (on the stack no less) default: return n, fmt.Errorf("unexpected type: %T", typ) } @@ -831,6 +946,19 @@ func (this *Generator) generateTypeTableDefined(typ TypeTableDefined) (n int, er return n, nil } +// generateCanAssign generates an expression which checks if the tag specified +// by tagSource can be assigned to a Go destination generated from typ. The +// generated code is INLINE. +func (this *Generator) generateCanAssign(typ Type, tagSource string) (n int, err error) { + nn, err := this.printf("canAssign(") + n += nn; if err != nil { return n, err } + nn, err = this.generateTN(typ) + n += nn; if err != nil { return n, err } + nn, err = this.printf(", %s)", tagSource) + n += nn; if err != nil { return n, err } + return n, nil +} + func (this *Generator) validateIntBitSize(size int) error { switch size { case 5, 8, 16, 32, 64: return nil -- 2.50.1 From a9f583d2e7386efe706e22c7a200698591cb84d7 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 17:02:33 -0400 Subject: [PATCH 12/24] generate: Validate OTA tags properly --- generate/generate.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 3fd2a73..a22f6f5 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -610,23 +610,17 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err n += nn; if err != nil { return n, err } return n, nil } - - // FIXME: the tag is validated improperly, the result of - // generateTag means basically nothing here because - // there is no data in the destination, we need another - // function that checks if a tag can be assigned to a - // PDL type // validate header // TODO: here, validate that length is less than the // max, whatever that is configured to be. the reason we // want to read it here is that we would have to skip // the tag anyway so why not. - nn, err = this.iprintf("if %s != ", elementTagVar) + nn, err = this.iprintf("if !(") n += nn; if err != nil { return n, err } - nn, err = this.generateTag(typ.Element, "(*this)") + nn, err = this.generateCanAssign(typ.Element, elementTagVar) n += nn; if err != nil { return n, err } - nn, err = this.printf(" {\n") + nn, err = this.printf(") {\n") n += nn; if err != nil { return n, err } this.push() nn, err = abort() -- 2.50.1 From 7343cf5853898eefb0039fd3b65068119b2ed4cd Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 17:03:47 -0400 Subject: [PATCH 13/24] generate: Fix array element tag variable --- generate/generate.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index a22f6f5..2f6c634 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -598,8 +598,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err nn, err = this.iprintf("for _ = range %s {\n", lengthVar) n += nn; if err != nil { return n, err } this.push() - // FIXME: wrong tag var - nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)") + nn, err = this.iprintf("nn, err = tape.Skim(decoder, %s)", elementTagVar) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } -- 2.50.1 From 1118b11bcd5b2967ade5c17f4e42feaa98712899 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 17:24:51 -0400 Subject: [PATCH 14/24] generate: Properly check assignment within .Decode --- generate/generate.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 2f6c634..8763b2b 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -246,15 +246,24 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - nn, err = this.iprintf("if !tag.Is(") + + nn, err = this.iprintf("if !(") n += nn; if err != nil { return n, err } - nn, err = this.generateTN(message.Type) + nn, err = this.generateCanAssign(message.Type, "tag") n += nn; if err != nil { return n, err } nn, err = this.printf(") {\n") n += nn; if err != nil { return n, err } - // TODO skip value using the correct TAPE function and return + this.push() + nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)") + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("return n, nil") + n += nn; if err != nil { return n, err } + this.pop() nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } + nn, err = this.generateDecodeValue(message.Type, "this", "tag") n += nn; if err != nil { return n, err } nn, err = this.iprintf("return n, nil\n") -- 2.50.1 From 30e9ead1abc51b6ac2394d12cf909472d4301812 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 17:27:04 -0400 Subject: [PATCH 15/24] generate: Do the same for .Decode --- generate/generate.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/generate/generate.go b/generate/generate.go index 8763b2b..5f81d26 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -175,6 +175,24 @@ func (this *Generator) generateTypedef(name string, typ Type) (n int, err error) this.push() nn, err = this.iprintf("var nn int\n") n += nn; if err != nil { return n, err } + + nn, err = this.iprintf("if !(") + n += nn; if err != nil { return n, err } + nn, err = this.generateCanAssign(typ, "tag") + n += nn; if err != nil { return n, err } + nn, err = this.printf(") {\n") + n += nn; if err != nil { return n, err } + this.push() + nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)") + n += nn; if err != nil { return n, err } + nn, err = this.generateErrorCheck() + n += nn; if err != nil { return n, err } + nn, err = this.iprintf("return n, nil") + n += nn; if err != nil { return n, err } + this.pop() + nn, err = this.iprintf("}\n") + n += nn; if err != nil { return n, err } + nn, err = this.generateDecodeValue(typ, "this", "tag") n += nn; if err != nil { return n, err } nn, err = this.iprintf("return n, nil\n") -- 2.50.1 From 12142706e1f4c39da40e5c0a01a483dad2d7ae7b Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 17:59:26 -0400 Subject: [PATCH 16/24] generate: Fix syntax and formatting errors --- generate/generate.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 5f81d26..7355bf8 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -183,11 +183,11 @@ func (this *Generator) generateTypedef(name string, typ Type) (n int, err error) nn, err = this.printf(") {\n") n += nn; if err != nil { return n, err } this.push() - nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)") + nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)\n") n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - nn, err = this.iprintf("return n, nil") + nn, err = this.iprintf("return n, nil\n") n += nn; if err != nil { return n, err } this.pop() nn, err = this.iprintf("}\n") @@ -272,11 +272,11 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e nn, err = this.printf(") {\n") n += nn; if err != nil { return n, err } this.push() - nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)") + nn, err = this.iprintf("nn, err = tape.Skim(decoder, tag)\n") n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - nn, err = this.iprintf("return n, nil") + nn, err = this.iprintf("return n, nil\n") n += nn; if err != nil { return n, err } this.pop() nn, err = this.iprintf("}\n") @@ -589,11 +589,11 @@ func (this *Generator) generateDecodeBranchCall(typ Type, valueSource, tagSource // generateDecodeBranch generates an aggregate decoder function definition for a // specified type. It assumes that hash == HashType(typ). func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err error) { - nn, err := this.iprintf("func %s(this *", this.decodeBranchName(hash)) + nn, err := this.iprintf("\nfunc %s(this *", this.decodeBranchName(hash)) n += nn; if err != nil { return n, err } nn, err = this.generateType(typ) n += nn; if err != nil { return n, err } - nn, err = this.printf(" decoder *tape.Decoder, tag tape.Tag) (nn int, err error) \n{") + nn, err = this.printf(", decoder *tape.Decoder, tag tape.Tag) (nn int, err error) {\n") n += nn; if err != nil { return n, err } this.push() @@ -625,7 +625,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err nn, err = this.iprintf("for _ = range %s {\n", lengthVar) n += nn; if err != nil { return n, err } this.push() - nn, err = this.iprintf("nn, err = tape.Skim(decoder, %s)", elementTagVar) + nn, err = this.iprintf("nn, err = tape.Skim(decoder, %s)\n", elementTagVar) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } @@ -712,7 +712,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err // abort field macro abortField := func() (n int, err error) { - nn, err = this.iprintf("tape.Skim(decoder, %s)", fieldTagVar) + nn, err = this.iprintf("tape.Skim(decoder, %s)\n", fieldTagVar) n += nn; if err != nil { return n, err } nn, err = this.iprintf("continue") n += nn; if err != nil { return n, err } @@ -722,7 +722,6 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err // switch on tag nn, err = this.iprintf("switch %s {\n", fieldTagVar) n += nn; if err != nil { return n, err } - this.push() for _, key := range slices.Sorted(maps.Keys(typ.Fields)) { field := typ.Fields[key] nn, err = this.iprintf("case %s:\n", fieldTagVar) @@ -772,7 +771,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err } this.pop() - nn, err = this.iprintf("}") + nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } return n, nil } -- 2.50.1 From fa4f5911267531169189e33cc29cba2eb52485df Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 18:38:30 -0400 Subject: [PATCH 17/24] generate: make branch functions generic, take in ~ of base type --- generate/generate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 7355bf8..6245064 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -589,11 +589,11 @@ func (this *Generator) generateDecodeBranchCall(typ Type, valueSource, tagSource // generateDecodeBranch generates an aggregate decoder function definition for a // specified type. It assumes that hash == HashType(typ). func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err error) { - nn, err := this.iprintf("\nfunc %s(this *", this.decodeBranchName(hash)) + nn, err := this.iprintf("\nfunc %s[T ~", this.decodeBranchName(hash)) n += nn; if err != nil { return n, err } nn, err = this.generateType(typ) n += nn; if err != nil { return n, err } - nn, err = this.printf(", decoder *tape.Decoder, tag tape.Tag) (nn int, err error) {\n") + nn, err = this.printf("](this *T, decoder *tape.Decoder, tag tape.Tag) (nn int, err error) {\n") n += nn; if err != nil { return n, err } this.push() @@ -632,7 +632,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err this.pop() nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } - nn, err = this.iprintf("return n, nil") + nn, err = this.iprintf("return n, nil\n") n += nn; if err != nil { return n, err } return n, nil } -- 2.50.1 From 195d0f97251b86ad5a37ffc7d3ddb8e73a84acf6 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 18:40:28 -0400 Subject: [PATCH 18/24] generate: Pass decoder to branch functions --- generate/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/generate.go b/generate/generate.go index 6245064..9c92199 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -578,7 +578,7 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri // - nn int func (this *Generator) generateDecodeBranchCall(typ Type, valueSource, tagSource string) (n int, err error) { hash := HashType(typ) - nn, err := this.iprintf("nn, err = %s(%s, %s)\n", this.decodeBranchName(hash), valueSource, tagSource) + nn, err := this.iprintf("nn, err = %s(%s, decoder, %s)\n", this.decodeBranchName(hash), valueSource, tagSource) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } -- 2.50.1 From 77c6b67d654308db333103004ba09062d7513aa1 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 18:41:37 -0400 Subject: [PATCH 19/24] generate: Break line after continue statements --- generate/generate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/generate.go b/generate/generate.go index 9c92199..b5c4912 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -714,7 +714,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err abortField := func() (n int, err error) { nn, err = this.iprintf("tape.Skim(decoder, %s)\n", fieldTagVar) n += nn; if err != nil { return n, err } - nn, err = this.iprintf("continue") + nn, err = this.iprintf("continue\n") n += nn; if err != nil { return n, err } return n, nil } -- 2.50.1 From 170f79c91434c55ae148f1183e3aea7237cc19be Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 18:48:17 -0400 Subject: [PATCH 20/24] generate: Fix bad variable names --- generate/generate.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index b5c4912..5e64b71 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -593,7 +593,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err n += nn; if err != nil { return n, err } nn, err = this.generateType(typ) n += nn; if err != nil { return n, err } - nn, err = this.printf("](this *T, decoder *tape.Decoder, tag tape.Tag) (nn int, err error) {\n") + nn, err = this.printf("](this *T, decoder *tape.Decoder, tag tape.Tag) (n int, err error) {\n") n += nn; if err != nil { return n, err } this.push() @@ -691,7 +691,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err // problems // read fields - nn, err = this.iprintf("for index = range %s {\n", lengthVar) + nn, err = this.iprintf("for %s = range %s {\n", indexVar, lengthVar) n += nn; if err != nil { return n, err } this.push() // read field header @@ -720,11 +720,11 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err } // switch on tag - nn, err = this.iprintf("switch %s {\n", fieldTagVar) + nn, err = this.iprintf("switch %s {\n", fieldKeyVar) n += nn; if err != nil { return n, err } for _, key := range slices.Sorted(maps.Keys(typ.Fields)) { field := typ.Fields[key] - nn, err = this.iprintf("case %s:\n", fieldTagVar) + nn, err = this.iprintf("case 0x%04X:\n", key) n += nn; if err != nil { return n, err } this.push() -- 2.50.1 From c18e251b4a8dc6cdf495d4b6516beeb3120d87bd Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 18:57:31 -0400 Subject: [PATCH 21/24] generate: Convert int64 to int to satisfy range This is a stupid fucking restriction --- generate/generate.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 5e64b71..07559fc 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -662,7 +662,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err n += nn; if err != nil { return n, err } nn, err = this.printf(", %s)\n", lengthVar) n += nn; if err != nil { return n, err } - nn, err = this.iprintf("for index := range %s {\n", lengthVar) + nn, err = this.iprintf("for index := range int(%s) {\n", lengthVar) n += nn; if err != nil { return n, err } this.push() nn, err = this.generateDecodeValue(typ.Element, "(*this)[index]", elementTagVar) @@ -691,7 +691,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err // problems // read fields - nn, err = this.iprintf("for %s = range %s {\n", indexVar, lengthVar) + nn, err = this.iprintf("for %s = range int(%s) {\n", indexVar, lengthVar) n += nn; if err != nil { return n, err } this.push() // read field header -- 2.50.1 From a99d4dee66fa1b9ad30da904762048af8997252f Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 19:07:57 -0400 Subject: [PATCH 22/24] generate: Fix no return statement, unused variables --- generate/generate.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 07559fc..796af57 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -680,9 +680,6 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - indexVar := this.newTemporaryVar("index") - nn, err = this.iprintf("%s := 0\n", indexVar) - n += nn; if err != nil { return n, err } // validate header // TODO: here, validate that length is less than the @@ -691,7 +688,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err // problems // read fields - nn, err = this.iprintf("for %s = range int(%s) {\n", indexVar, lengthVar) + nn, err = this.iprintf("for _ = range int(%s) {\n", lengthVar) n += nn; if err != nil { return n, err } this.push() // read field header @@ -770,6 +767,8 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err default: return n, fmt.Errorf("unexpected type: %T", typ) } + nn, err = this.iprintf("return n, nil\n") + this.pop() nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } -- 2.50.1 From a270c22cb99c66d715b08c519fd9580a70c30006 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 19:39:41 -0400 Subject: [PATCH 23/24] generate: The generics idea didn't work, use type names instead --- generate/generate.go | 73 +++++++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 796af57..b0be848 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -67,6 +67,7 @@ type Generator struct { type decodeBranchRequest struct { hash [16]byte typ Type + name string } func (this *Generator) Generate(protocol *Protocol) (n int, err error) { @@ -102,9 +103,9 @@ func (this *Generator) Generate(protocol *Protocol) (n int, err error) { // request queue for { - hash, typ, ok := this.pullDecodeBranchRequest() + hash, typ, name, ok := this.pullDecodeBranchRequest() if !ok { break } - nn, err := this.generateDecodeBranch(hash, typ) + nn, err := this.generateDecodeBranch(hash, typ, name) n += nn; if err != nil { return n, err } } @@ -193,7 +194,7 @@ func (this *Generator) generateTypedef(name string, typ Type) (n int, err error) nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } - nn, err = this.generateDecodeValue(typ, "this", "tag") + nn, err = this.generateDecodeValue(typ, name, "this", "tag") n += nn; if err != nil { return n, err } nn, err = this.iprintf("return n, nil\n") n += nn; if err != nil { return n, err } @@ -282,7 +283,7 @@ func (this *Generator) generateMessage(method uint16, message Message) (n int, e nn, err = this.iprintf("}\n") n += nn; if err != nil { return n, err } - nn, err = this.generateDecodeValue(message.Type, "this", "tag") + nn, err = this.generateDecodeValue(message.Type, this.resolveMessageName(message.Name), "this", "tag") n += nn; if err != nil { return n, err } nn, err = this.iprintf("return n, nil\n") n += nn; if err != nil { return n, err } @@ -471,7 +472,10 @@ func (this *Generator) generateEncodeValue(typ Type, valueSource, tagSource stri // - n int // - err error // - nn int -func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource string) (n int, err error) { +// +// The typeName paramterer is handled in the way described in the documentation +// for [Generator.generateDecodeBranch]. +func (this *Generator) generateDecodeValue(typ Type, typeName, valueSource, tagSource string) (n int, err error) { switch typ := typ.(type) { case TypeInt: // SI: (none) @@ -537,7 +541,7 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri } case TypeArray: // OTA: * - nn, err := this.generateDecodeBranchCall(typ, valueSource, tagSource) + nn, err := this.generateDecodeBranchCall(typ, typeName, valueSource, tagSource) n += nn; if err != nil { return n, err } case TypeTable: // KTV: ( )* @@ -549,7 +553,7 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri n += nn; if err != nil { return n, err } case TypeTableDefined: // KTV: ( )* - nn, err := this.generateDecodeBranchCall(typ, valueSource, tagSource) + nn, err := this.generateDecodeBranchCall(typ, typeName, valueSource, tagSource) n += nn; if err != nil { return n, err } case TypeNamed: // WHATEVER: [WHATEVER] @@ -576,24 +580,36 @@ func (this *Generator) generateDecodeValue(typ Type, valueSource, tagSource stri // - n int // - err error // - nn int -func (this *Generator) generateDecodeBranchCall(typ Type, valueSource, tagSource string) (n int, err error) { +// +// The typeName paramterer is handled in the way described in the documentation +// for [Generator.generateDecodeBranch]. +func (this *Generator) generateDecodeBranchCall(typ Type, typeName, valueSource, tagSource string) (n int, err error) { hash := HashType(typ) - nn, err := this.iprintf("nn, err = %s(%s, decoder, %s)\n", this.decodeBranchName(hash), valueSource, tagSource) + nn, err := this.iprintf( + "nn, err = %s(%s, decoder, %s)\n", + this.decodeBranchName(hash, typeName), valueSource, tagSource) n += nn; if err != nil { return n, err } nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } - this.pushDecodeBranchRequest(hash, typ) + this.pushDecodeBranchRequest(hash, typ, typeName) return n, nil } // generateDecodeBranch generates an aggregate decoder function definition for a -// specified type. It assumes that hash == HashType(typ). -func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err error) { - nn, err := this.iprintf("\nfunc %s[T ~", this.decodeBranchName(hash)) +// specified type. It assumes that hash == HashType(typ). If typeName is not +// empty, it will be used as the type in the argument list instead of the result +// of [Generator.generateType]. +func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName string) (n int, err error) { + nn, err := this.iprintf("\nfunc %s(this *", this.decodeBranchName(hash, typeName)) n += nn; if err != nil { return n, err } - nn, err = this.generateType(typ) - n += nn; if err != nil { return n, err } - nn, err = this.printf("](this *T, decoder *tape.Decoder, tag tape.Tag) (n int, err error) {\n") + if typeName == "" { + nn, err = this.generateType(typ) + n += nn; if err != nil { return n, err } + } else { + nn, err = this.print(typeName) + n += nn; if err != nil { return n, err } + } + nn, err = this.printf(", decoder *tape.Decoder, tag tape.Tag) (n int, err error) {\n") n += nn; if err != nil { return n, err } this.push() @@ -665,7 +681,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err nn, err = this.iprintf("for index := range int(%s) {\n", lengthVar) n += nn; if err != nil { return n, err } this.push() - nn, err = this.generateDecodeValue(typ.Element, "(*this)[index]", elementTagVar) + nn, err = this.generateDecodeValue(typ.Element, "", "(*this)[index]", elementTagVar) n += nn; if err != nil { return n, err } this.pop() nn, err = this.iprintf("}\n") @@ -740,7 +756,9 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err n += nn; if err != nil { return n, err } // decode payload - nn, err = this.generateDecodeValue(field.Type, fmt.Sprintf("&(this.%s)", field.Name), fieldTagVar) + nn, err = this.generateDecodeValue( + field.Type, "", + fmt.Sprintf("&(this.%s)", field.Name), fieldTagVar) n += nn; if err != nil { return n, err } this.pop() } @@ -775,31 +793,36 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type) (n int, err return n, nil } -func (this *Generator) decodeBranchName(hash [16]byte) string { - return fmt.Sprintf("decodeBranch_%s", hex.EncodeToString(hash[:])) +func (this *Generator) decodeBranchName(hash [16]byte, name string) string { + if name == "" { + return fmt.Sprintf("decodeBranch_%s", hex.EncodeToString(hash[:])) + } else { + return fmt.Sprintf("decodeBranch_%s_%s", hex.EncodeToString(hash[:]), name) + } } // pushDecodeBranchRequest pushes a new branch decode function request to the // back of the queue, if it is not already in the queue. -func (this *Generator) pushDecodeBranchRequest(hash [16]byte, typ Type) { +func (this *Generator) pushDecodeBranchRequest(hash [16]byte, typ Type, name string) { for _, item := range this.decodeBranchRequestQueue { - if item.hash == hash { return } + if item.hash == hash && item.name == name { return } } this.decodeBranchRequestQueue = append(this.decodeBranchRequestQueue, decodeBranchRequest { hash: hash, typ: typ, + name: name, }) } // pullDecodeBranchRequest pulls a branch decode function request from the front // of the queue. -func (this *Generator) pullDecodeBranchRequest() (hash [16]byte, typ Type, ok bool) { +func (this *Generator) pullDecodeBranchRequest() (hash [16]byte, typ Type, name string, ok bool) { if len(this.decodeBranchRequestQueue) < 1 { - return [16]byte { }, nil, false + return [16]byte { }, nil, "", false } request := this.decodeBranchRequestQueue[0] this.decodeBranchRequestQueue = this.decodeBranchRequestQueue[1:] - return request.hash, request.typ, true + return request.hash, request.typ, request.name, true } func (this *Generator) generateErrorCheck() (n int, err error) { -- 2.50.1 From 57c30ac6690dbbe4024adbd2b1d08d18deaeb665 Mon Sep 17 00:00:00 2001 From: Sasha Koshka Date: Wed, 6 Aug 2025 20:19:31 -0400 Subject: [PATCH 24/24] generate: Generator compiles --- generate/generate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index b0be848..0f21e37 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -628,7 +628,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st nn, err = this.generateErrorCheck() n += nn; if err != nil { return n, err } elementTagVar := this.newTemporaryVar("elementTag") - nn, err = this.iprintf("var %s uint64\n", lengthVar) + nn, err = this.iprintf("var %s tape.Tag\n", elementTagVar) n += nn; if err != nil { return n, err } nn, err = this.iprintf("%s, nn, err = decoder.ReadTag()\n", elementTagVar) n += nn; if err != nil { return n, err } @@ -681,7 +681,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st nn, err = this.iprintf("for index := range int(%s) {\n", lengthVar) n += nn; if err != nil { return n, err } this.push() - nn, err = this.generateDecodeValue(typ.Element, "", "(*this)[index]", elementTagVar) + nn, err = this.generateDecodeValue(typ.Element, "", "(&(*this)[index])", elementTagVar) n += nn; if err != nil { return n, err } this.pop() nn, err = this.iprintf("}\n") @@ -758,7 +758,7 @@ func (this *Generator) generateDecodeBranch(hash [16]byte, typ Type, typeName st // decode payload nn, err = this.generateDecodeValue( field.Type, "", - fmt.Sprintf("&(this.%s)", field.Name), fieldTagVar) + fmt.Sprintf("(&(this.%s))", field.Name), fieldTagVar) n += nn; if err != nil { return n, err } this.pop() } -- 2.50.1