tape: DecodeAny only returns an error when there is one

internal/testutil/testutil.go

@@ -0,0 +1,89 @@
+package testutil
+
+import "fmt"
+import "slices"
+import "strings"
+
+// Snake lets you compare blocks of data where the ordering of certain parts may
+// be swapped every which way. It is designed for comparing the encoding of
+// maps where the ordering of individual elements is inconsistent.
+//
+// The snake is divided into sectors, which hold a number of variations. For a
+// sector to be satisfied by some data, some ordering of it must match the data
+// exactly. for the snake to be satisfied by some data, its sectors must match
+// the data in order, but the internal ordering of each sector doesn't matter.
+type Snake []    []     []byte
+//         snake sector variation
+
+// S returns a new snake.
+func S(data ...byte) Snake {
+	return (Snake { }).Add(data...)
+}
+
+// AddVar returns a new snake with the given sector added on to it. Successive
+// calls of this method can be chained together to create a big ass snake.
+func (sn Snake) AddVar(sector ...[]byte) Snake {
+	slice := make(Snake, len(sn) + 1)
+	copy(slice, sn)
+	slice[len(slice) - 1] = sector
+	return slice
+}
+
+// Add is like AddVar, but adds a sector with only one variation, which means it
+// does not vary, hence why the method is called that.
+func (sn Snake) Add(data ...byte) Snake {
+	return sn.AddVar(data)
+}
+
+// Check determines if the data satisfies the snake.
+func (sn Snake) Check(data []byte) (ok bool, n int) {
+	left := data
+	variations := map[int] []byte { }
+	for _, sector := range sn {
+		clear(variations)
+		for key, variation := range sector {
+			variations[key] = variation
+		}
+		for len(variations) > 0 {
+			found := false
+			for key, variation := range variations {
+				if len(left) < len(variation) { continue }
+				if !slices.Equal(left[:len(variation)], variation) { continue }
+				n += len(variation)
+				left = data[n:]
+				delete(variations, key)
+				found = true
+			}
+			if !found { return false, n }
+		}
+	}
+	if n < len(data) {
+		return false, n
+	}
+	return true, n
+}
+
+func (sn Snake) String() string {
+	out := strings.Builder { }
+	for index, sector := range sn {
+		if index > 0 { out.WriteString(" : ") }
+		out.WriteRune('[')
+		for index, variation := range sector {
+			if index > 0 { out.WriteString(" / ") }
+			for _, byt := range variation {
+				fmt.Fprintf(&out, "%02x", byt)
+			}
+		}
+		out.WriteRune(']')
+	}
+	return out.String()
+}
+
+// HexBytes formats bytes into a hexadecimal string.
+func HexBytes(data []byte) string {
+	out := strings.Builder { }
+	for _, byt := range data {
+		fmt.Fprintf(&out, "%02x", byt)
+	}
+	return out.String()
+}

internal/testutil/testutil_test.go

@@ -0,0 +1,66 @@
+package testutil
+
+import "testing"
+
+func TestSnakeA(test *testing.T) {
+	snake := S(1, 6).AddVar(
+		[]byte { 1 },
+		[]byte { 2 },
+		[]byte { 3 },
+		[]byte { 4 },
+		[]byte { 5 },
+	).Add(9)
+
+	test.Log(snake)
+
+	ok, n := snake.Check([]byte { 1, 6, 1, 2, 3, 4, 5, 9 })
+	if !ok { test.Fatal("false negative:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 5, 4, 3, 2, 1, 9 })
+	if !ok { test.Fatal("false negative:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 3, 1, 4, 2, 5, 9 })
+	if !ok { test.Fatal("false negative:", n) }
+	
+	ok, n = snake.Check([]byte { 1, 6, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 1, 2, 3, 4, 5, 6, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 0, 2, 3, 4, 5, 6, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 7, 1, 4, 2, 5, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 7, 3, 1, 4, 2, 5, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 7, 3, 1, 4, 2, 5, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 1, 2, 3, 4, 5, 9, 10})
+	if ok { test.Fatal("false positive:", n) }
+}
+
+func TestSnakeB(test *testing.T) {
+	snake := S(1, 6).AddVar(
+		[]byte { 1 },
+		[]byte { 2 },
+	).Add(9).AddVar(
+		[]byte { 3, 2 },
+		[]byte { 0 },
+		[]byte { 1, 1, 2, 3 },
+	)
+
+	test.Log(snake)
+
+	ok, n := snake.Check([]byte { 1, 6, 1, 2, 9, 3, 2, 0, 1, 1, 2, 3})
+	if !ok { test.Fatal("false negative:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 2, 1, 9, 0, 1, 1, 2, 3, 3, 2})
+	if !ok { test.Fatal("false negative:", n) }
+	
+	ok, n = snake.Check([]byte { 1, 6, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 1, 2, 9 })
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 9, 3, 2, 0, 1, 1, 2, 3})
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 2, 9, 0, 1, 1, 2, 3, 3, 2})
+	if ok { test.Fatal("false positive:", n) }
+	ok, n = snake.Check([]byte { 1, 6, 1, 2, 9, 3, 2, 1, 1, 2, 3})
+	if ok { test.Fatal("false positive:", n) }
+}

tape/dynamic.go

@@ -1,5 +1,11 @@
 package tape
 
+// dont smoke reflection, kids!!!!!!!!!
+// totally reflectric, reflectrified, etc. this is probably souper slow but
+// certainly no slower than the built in json encoder i'd imagine.
+// TODO: add support for struct tags: `tape:"0000"`, tape:"0001"` so they can get
+// transformed into tables with a defined schema
+
 import "fmt"
 import "reflect"
 
@@ -14,6 +20,7 @@ import "reflect"
 //   - []<supported type>
 //   - map[uint16]<supported type>
 func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
+	// TODO use reflection for all of this to ignore type names
 	// primitives
 	switch value := value.(type) {
 	case int:    return encoder.WriteInt32(int32(value))
@@ -34,6 +41,7 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 		}
 		nn, err := encoder.Write(value)
 		n += nn; if err != nil { return n, err }
+		return n, nil
 	}
 
 	// aggregates
@@ -41,21 +49,124 @@ func EncodeAny(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	switch reflectType.Kind() {
 	case reflect.Slice:
 		return encodeAnySlice(encoder, value, tag)
-	case reflect.Array:
+	// case reflect.Array:
-		return encodeAnySlice(encoder, reflect.ValueOf(value).Slice(0, reflectType.Len()).Interface(), tag)
+		// return encodeAnySlice(encoder, reflect.ValueOf(value).Slice(0, reflectType.Len()).Interface(), tag)
 	case reflect.Map:
 		if reflectType.Key() == reflect.TypeOf(uint16(0)) {
 			return encodeAnyMap(encoder, value, tag)
 		}
-		return 0, fmt.Errorf("cannot encode map key %T, key must be uint16", value)
+		return n, fmt.Errorf("cannot encode map key %T, key must be uint16", value)
 	}
-	return 0, fmt.Errorf("cannot encode type %T", value)
+	return n, fmt.Errorf("cannot encode type %T", value)
+}
+
+// DecodeAny decodes data and places it into destination, which must be a
+// pointer to a supported type. See [EncodeAny] for a list of supported types.
+func DecodeAny(decoder *Decoder, destination any, tag Tag) (n int, err error) {
+	return decodeAny(decoder, reflect.ValueOf(destination), tag)
+}
+
+func decodeAny(decoder *Decoder, destination reflect.Value, tag Tag) (n int, err error) {
+	errWrongDestinationType := func(expected string) error {
+		return fmt.Errorf(
+			"expected &%s destination, not %v",
+			expected, destination)
+	}
+	
+	if destination.Kind() != reflect.Pointer {
+		return n, errWrongDestinationType("pointer")
+	}
+	
+	switch tag.WithoutCN() {
+	case SI:
+		// SI: (none)
+		err = setIntPtr(destination, uint64(tag.CN()))
+		if err != nil { return n, err }
+	case LI:
+		// LI: <value: IntN>
+		nn, err := decodeAndSetIntPtr(decoder, destination, tag.CN() - 1)
+		n += nn; if err != nil { return n, err }
+	case FP:
+		// FP: <value: FloatN>
+		nn, err := decodeAndSetFloatPtr(decoder, destination, tag.CN() - 1)
+		n += nn; if err != nil { return n, err }
+	case SBA:
+		// SBA: <data: U8>*
+		destination, err := asByteArrayPtr(destination)
+		if err != nil { return n, err }
+		buffer := make([]byte, tag.CN())
+		nn, err := decoder.Read(buffer)
+		n += nn; if err != nil { return n, err }
+		*destination = buffer
+	case LBA:
+		// LBA: <length: UN> <data: U8>*
+		destination, err := asByteArrayPtr(destination)
+		if err != nil { return n, err }
+		length, nn, err := decoder.ReadUintN(tag.CN() - 1)
+		n += nn; if err != nil { return n, err }
+		buffer := make([]byte, length)
+		nn, err = decoder.Read(buffer)
+		n += nn; if err != nil { return n, err }
+		*destination = buffer
+	case OTA:
+		// OTA: <length: UN> <elementTag: tape.Tag> <values>*
+		length, nn, err := decoder.ReadUintN(tag.CN() - 1)
+		n += nn; if err != nil { return n, err }
+		oneTag, nn, err := decoder.ReadTag()
+		n += nn; if err != nil { return n, err }
+		var slice reflect.Value
+		needSet := false
+		elem := destination.Elem()
+		if elem.Kind() == reflect.Struct && elem.Type().Name() == "unknownSlicePlaceholder" {
+			needSet = true
+			slice, err = skeletonValueSlice(oneTag, int(length))
+			if err != nil { return n, err }
+		} else {
+			slice = elem
+			if slice.Kind() != reflect.Slice {
+				return n, errWrongDestinationType("slice")
+			}
+			slice.SetLen(int(length))
+		}
+		for index := range length {
+			nn, err := decodeAny(decoder, slice.Index(int(index)), oneTag)
+			n += nn; if err != nil { return n, err }
+		}
+		if needSet {
+			destination.Elem().Set(slice)
+		}
+	case KTV:
+		// KTV: <length: UN> (<key: U16> <tag: Tag> <value>)*
+		table := destination.Elem()
+		var dummyMap map[uint16] any
+		if table.Type() != reflect.TypeOf(dummyMap) {
+			return n, errWrongDestinationType("map[uint16] any")
+		}
+		length, nn, err := decoder.ReadUintN(tag.CN() - 1)
+		n += nn; if err != nil { return n, err }
+		table.Clear()
+		for _ = range length {
+			key, nn, err := decoder.ReadUint16()
+			n += nn; if err != nil { return n, err }
+			itemTag, nn, err := decoder.ReadTag()
+			n += nn; if err != nil { return n, err }
+			value, err := skeletonValue(itemTag)
+			if err != nil { return n, err }
+			nn, err = decodeAny(decoder, value.Elem(), itemTag)
+			n += nn; if err != nil { return n, err }
+			table.SetMapIndex(reflect.ValueOf(key), value)
+		}
+	default:
+		return n, fmt.Errorf("unknown TN %d", tag.TN())
+	}
+	return n, nil
 }
 
 // TagAny returns the correct tag for an "any" value. Returns an error if the
 // underlying type is unsupported. See [EncodeAny] for a list of supported
 // types.
 func TagAny(value any) (Tag, error) {
+	// TODO use reflection for all of this to ignore type names
 	// primitives
 	switch value := value.(type) {
 	case int,   uint:   return LI.WithCN(3), nil
@@ -70,11 +181,11 @@ func TagAny(value any) (Tag, error) {
 	// aggregates
 	reflectType := reflect.TypeOf(value)
 	switch reflectType.Kind() {
-	case reflect.Slice: return OTA.WithCN(reflect.ValueOf(value).Len()), nil
+	case reflect.Slice: return OTA.WithCN(IntBytes(uint64(reflect.ValueOf(value).Len())) - 1), nil
 	case reflect.Array: return OTA.WithCN(reflectType.Len()), nil
 	case reflect.Map:
 		if reflectType.Key() == reflect.TypeOf(uint16(0)) {
-			return OTA.WithCN(reflect.ValueOf(value).Len()), nil
+			return KTV.WithCN(IntBytes(uint64(reflect.ValueOf(value).Len())) - 1), nil
 		}
 		return 0, fmt.Errorf("cannot encode map key %T, key must be uint16", value)
 	}
@@ -89,7 +200,7 @@ func encodeAnySlice(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	reflectType := reflect.TypeOf(value)
 	oneTag, err := TagAny(reflect.Zero(reflectType.Elem()).Interface())
 	if err != nil { return n, err }
-	for index := 0; index <= reflectValue.Len(); index += 1 {
+	for index := 0; index < reflectValue.Len(); index += 1 {
 		item := reflectValue.Index(index).Interface()
 		itemTag, err := TagAny(item)
 		if err != nil { return n, err }
@@ -98,7 +209,7 @@ func encodeAnySlice(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	if oneTag.Is(SBA) { oneTag += 1 << 5 }
 	nn, err = encoder.WriteUint8(uint8(oneTag))
 	n += nn; if err != nil { return n, err }
-	for index := 0; index <= reflectValue.Len(); index += 1 {
+	for index := 0; index < reflectValue.Len(); index += 1 {
 		item := reflectValue.Index(index).Interface()
 		nn, err = EncodeAny(encoder, item, oneTag)
 		n += nn; if err != nil { return n, err }
@@ -118,7 +229,7 @@ func encodeAnyMap(encoder *Encoder, value any, tag Tag) (n int, err error) {
 		nn, err = encoder.WriteUint16(key)
 		n += nn; if err != nil { return n, err }
 		itemTag, err := TagAny(value)
-		n += nn; if err != nil { return n, err }
+		if err != nil { return n, err }
 		nn, err = encoder.WriteUint8(uint8(itemTag))
 		n += nn; if err != nil { return n, err }
 		nn, err = EncodeAny(encoder, value, itemTag)
@@ -126,3 +237,114 @@ func encodeAnyMap(encoder *Encoder, value any, tag Tag) (n int, err error) {
 	}
 	return n, nil
 }
+
+func setIntPtr(destination reflect.Value, value uint64) error {
+	elem := destination.Elem()
+	if !elem.CanInt() {
+		return fmt.Errorf("cannot assign integer to %T", elem.Interface())
+	}
+	elem.Set(reflect.ValueOf(value).Convert(elem.Type()))
+	return nil
+}
+
+func setFloatPtr(destination reflect.Value, value float64) error {
+	elem := destination.Elem()
+	if !elem.CanFloat() {
+		return fmt.Errorf("cannot assign float to %T", elem.Interface())
+	}
+	elem.Set(reflect.ValueOf(value).Convert(elem.Type()))
+	return nil
+}
+
+func decodeAndSetIntPtr(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) {
+	value, nn, err := decoder.ReadUintN(bytes)
+	n += nn; if err != nil { return n, err }
+	return n, setIntPtr(destination, value)
+}
+
+func decodeAndSetFloatPtr(decoder *Decoder, destination reflect.Value, bytes int) (n int, err error) {
+	switch bytes {
+	case 8:
+		value, nn, err := decoder.ReadFloat64()
+		n += nn; if err != nil { return n, err }
+		return n, setFloatPtr(destination, float64(value))
+	case 4:
+		value, nn, err := decoder.ReadFloat32()
+		n += nn; if err != nil { return n, err }
+		return n, setFloatPtr(destination, float64(value))
+	}
+	return n, fmt.Errorf("cannot decode float%d", bytes * 8)
+}
+
+func asByteArrayPtr(value reflect.Value) (*[]byte, error) {
+	typ := value.Type()
+	if typ.Kind() != reflect.Pointer {
+		return nil, fmt.Errorf("cannot convert %T to pointer", value)
+	}
+	if typ.Elem().Kind() != reflect.Slice {
+		return nil, fmt.Errorf("cannot convert %T to slice pointer", value)
+	}
+	if typ.Elem().Elem() != reflect.TypeOf(byte(0)) {
+		return nil, fmt.Errorf("cannot convert %T to *[]byte", value)
+	}
+	return value.Convert(reflect.PtrTo(reflect.SliceOf(reflect.TypeOf(byte(0))))).Interface().(*[]byte), nil
+}
+
+func skeletonValue(tag Tag) (reflect.Value, error) {
+	switch tag.WithoutCN() {
+	case SI:
+		value := uint8(0)
+		return reflect.ValueOf(&value), nil
+	case LI:
+		switch tag.CN() {
+		case 0: value := uint8(0); return reflect.ValueOf(&value), nil
+		case 1: value := uint16(0); return reflect.ValueOf(&value), nil
+		case 3: value := uint32(0); return reflect.ValueOf(&value), nil
+		case 7: value := uint64(0); return reflect.ValueOf(&value), nil
+		}
+		return reflect.Value { }, fmt.Errorf("unknown CN %d for LI", tag.CN())
+	case FP:
+		switch tag.CN() {
+		case 3: value := float32(0); return reflect.ValueOf(&value), nil
+		case 7: value := float64(0); return reflect.ValueOf(&value), nil
+		}
+		return reflect.Value { }, fmt.Errorf("unknown CN %d for FP", tag.CN())
+	case SBA: value := []byte { }; return reflect.ValueOf(&value), nil
+	case LBA: value := []byte { }; return reflect.ValueOf(&value), nil
+	case OTA: value := unknownSlicePlaceholder { }; return reflect.ValueOf(&value), nil
+	case KTV: value := map[uint16] any { }; return reflect.ValueOf(&value), nil
+	}
+	return reflect.Value { }, fmt.Errorf("unknown TN %d", tag.TN())
+}
+
+func skeletonValueSlice(tag Tag, length int) (reflect.Value, error) {
+	switch tag.WithoutCN() {
+	case SI:
+		value := make([]uint8, length)
+		return reflect.ValueOf(&value), nil
+	case LI:
+		switch tag.CN() {
+		case 0: value := make([]uint8, length); return reflect.ValueOf(&value), nil
+		case 1: value := make([]uint16, length); return reflect.ValueOf(&value), nil
+		case 3: value := make([]uint32, length); return reflect.ValueOf(&value), nil
+		case 7: value := make([]uint64, length); return reflect.ValueOf(&value), nil
+		}
+		return reflect.Value { }, fmt.Errorf("unknown CN %d for LI OTA", tag.CN())
+	case FP:
+		switch tag.CN() {
+		case 3: value := make([]float32, length); return reflect.ValueOf(&value), nil
+		case 7: value := make([]float64, length); return reflect.ValueOf(&value), nil
+		}
+		return reflect.Value { }, fmt.Errorf("unknown CN %d for FP OTA", tag.CN())
+	case SBA: value := make([][]byte, length); return reflect.ValueOf(&value), nil
+	case LBA: value := make([][]byte, length); return reflect.ValueOf(&value), nil
+	case OTA: value := make([]any, length); return reflect.ValueOf(&value), nil
+	case KTV: value := make([]map[uint16] any, length); return reflect.ValueOf(&value), nil
+	}
+	return reflect.Value { }, fmt.Errorf("unknown TN %d", tag.TN())
+}
+
+// unknownSlicePlaceholder is inserted by skeletonValue and informs the program
+// that the destination for the slice needs to be generated based on the item
+// tag in the OTA.
+type unknownSlicePlaceholder struct { }

tape/encode.go

@@ -82,7 +82,7 @@ func (this *Encoder) WriteIntN(value int64, bytes int) (n int, err error) {
 	return this.WriteUintN(uint64(value), bytes)
 }
 
-// for below functions, increase buffers if go somehow gets support for over 64
+// for Write/ReadUintN, increase buffers if go somehow gets support for over 64
 // bit integers. we could also make an expanding int type in goutil to use here,
 // or maybe there is one in the stdlib. keep the int64 versions as well though
 // because its ergonomic.
@@ -92,7 +92,7 @@ func (this *Encoder) WriteUintN(value uint64, bytes int) (n int, err error) {
 	// TODO: don't make multiple write calls (without allocating)
 	buffer := [1]byte { }
 	for bytesLeft := bytes; bytesLeft > 0; bytesLeft -- {
-		buffer[0] = byte(buffer[0]) >> ((bytesLeft - 1) * 8)
+		buffer[0] = byte(value) >> ((bytesLeft - 1) * 8)
 		nn, err := this.Write(buffer[:])
 		n += nn; if err != nil { return n, err }
 	}

tape/tag.go

@@ -1,16 +1,18 @@
 package tape
 
+import "fmt"
+
 type Tag byte; const (
-	SI      Tag = 0 << 5  // Small integer
+	SI      Tag = 0 << 5 // Small integer
-	LI      Tag = 1 << 5  // Large integer
+	LI      Tag = 1 << 5 // Large integer
-	FP      Tag = 2 << 5  // Floating point
+	FP      Tag = 2 << 5 // Floating point
-	SBA     Tag = 3 << 5  // Small byte array
+	SBA     Tag = 3 << 5 // Small byte array
-	LBA     Tag = 4 << 5  // Large byte array
+	LBA     Tag = 4 << 5 // Large byte array
-	OTA     Tag = 5 << 5  // One-tag array
+	OTA     Tag = 5 << 5 // One-tag array
-	KTV     Tag = 6 << 5  // Key-tag-value table
+	KTV     Tag = 6 << 5 // Key-tag-value table
-	TNMask  Tag = 0xE0    // The entire TN bitfield
+	TNMask  Tag = 0xE0   // The entire TN bitfield
-	CNMask  Tag = 0x20    // The entire CN bitfield
+	CNMask  Tag = 0x1F   // The entire CN bitfield
-	CNLimit Tag = 32      // All valid CNs are < CNLimit
+	CNLimit Tag = 32     // All valid CNs are < CNLimit
 )
 
 func (tag Tag) TN() int {
@@ -33,6 +35,20 @@ func (tag Tag) Is(other Tag) bool {
 	return tag.TN() == other.TN()
 }
 
+func (tag Tag) String() string {
+	tn := fmt.Sprint(tag.TN())
+	switch tag.WithoutCN() {
+	case SI:  tn = "SI"
+	case LI:  tn = "LI"
+	case FP:  tn = "FP"
+	case SBA: tn = "SBA"
+	case LBA: tn = "LBA"
+	case OTA: tn = "OTA"
+	case KTV: tn = "KTV"
+	}
+	return fmt.Sprintf("%s:%d", tn, tag.CN())
+}
+
 // BufferTag returns the appropriate tag for a buffer.
 func BufferTag(value []byte) Tag {
 	return bufferLenTag(len(value))