// Binary TLV (Type-Length-Value) encoding for DynamoDB items // Replaces JSON storage with efficient binary format // Format: [attribute_count][name_len][name][type_tag][value_len][value]... package dynamodb import "core:bytes" import "core:slice" // Type tags for binary encoding (1 byte each) Type_Tag :: enum u8 { // Scalar types String = 0x01, // S Number = 0x02, // N (stored as string) Binary = 0x03, // B (base64 string) Boolean = 0x04, // BOOL Null = 0x05, // NULL // Set types String_Set = 0x10, // SS Number_Set = 0x11, // NS Binary_Set = 0x12, // BS // Complex types List = 0x20, // L Map = 0x21, // M } // ============================================================================ // Encoding (Item → Binary) // ============================================================================ // Encode an Item to binary TLV format // Format: [attribute_count:varint][attributes...] // Each attribute: [name_len:varint][name:bytes][type_tag:u8][value_encoded:bytes] encode :: proc(item: Item) -> ([]byte, bool) { buf: bytes.Buffer bytes.buffer_init_allocator(&buf, 0, 1024, context.allocator) defer bytes.buffer_destroy(&buf) // Write attribute count encode_varint(&buf, len(item)) // Collect and sort keys for deterministic encoding keys := make([dynamic]string, context.temp_allocator) for key in item { append(&keys, key) } slice.sort_by(keys[:], proc(a, b: string) -> bool { return a < b }) // Encode each attribute for key in keys { value := item[key] // Write attribute name encode_varint(&buf, len(key)) bytes.buffer_write_string(&buf, key) // Encode attribute value ok := encode_attribute_value(&buf, value) if !ok { return nil, false } } return bytes.buffer_to_bytes(&buf), true } // Encode an AttributeValue to binary format encode_attribute_value :: proc(buf: ^bytes.Buffer, attr: Attribute_Value) -> bool { switch v in attr { case String: bytes.buffer_write_byte(buf, u8(Type_Tag.String)) encode_varint(buf, len(v)) bytes.buffer_write_string(buf, string(v)) case DDB_Number: bytes.buffer_write_byte(buf, u8(Type_Tag.Number)) // Store as string in item encoding num_str := format_ddb_number(v) encode_varint(buf, len(num_str)) bytes.buffer_write_string(buf, num_str) case Number: bytes.buffer_write_byte(buf, u8(Type_Tag.Number)) encode_varint(buf, len(v)) bytes.buffer_write_string(buf, string(v)) case Binary: bytes.buffer_write_byte(buf, u8(Type_Tag.Binary)) encode_varint(buf, len(v)) bytes.buffer_write_string(buf, string(v)) case Bool: bytes.buffer_write_byte(buf, u8(Type_Tag.Boolean)) bytes.buffer_write_byte(buf, 1 if bool(v) else 0) case Null: bytes.buffer_write_byte(buf, u8(Type_Tag.Null)) // NULL has no value bytes case DDB_Number_Set: bytes.buffer_write_byte(buf, u8(Type_Tag.Number_Set)) // Use Number_Set tag, not DDB_Number_Set encode_varint(buf, len(v)) for num in v { // Format the DDB_Number to a string num_str := format_ddb_number(num) encode_varint(buf, len(num_str)) bytes.buffer_write_string(buf, num_str) } case String_Set: bytes.buffer_write_byte(buf, u8(Type_Tag.String_Set)) encode_varint(buf, len(v)) for s in v { encode_varint(buf, len(s)) bytes.buffer_write_string(buf, s) } case Number_Set: bytes.buffer_write_byte(buf, u8(Type_Tag.Number_Set)) encode_varint(buf, len(v)) for n in v { encode_varint(buf, len(n)) bytes.buffer_write_string(buf, n) } case Binary_Set: bytes.buffer_write_byte(buf, u8(Type_Tag.Binary_Set)) encode_varint(buf, len(v)) for b in v { encode_varint(buf, len(b)) bytes.buffer_write_string(buf, b) } case List: bytes.buffer_write_byte(buf, u8(Type_Tag.List)) encode_varint(buf, len(v)) for item in v { ok := encode_attribute_value(buf, item) if !ok { return false } } case Map: bytes.buffer_write_byte(buf, u8(Type_Tag.Map)) encode_varint(buf, len(v)) // Collect and sort keys for deterministic encoding keys := make([dynamic]string, context.temp_allocator) for key in v { append(&keys, key) } slice.sort_by(keys[:], proc(a, b: string) -> bool { return a < b }) // Encode each map entry for key in keys { value := v[key] encode_varint(buf, len(key)) bytes.buffer_write_string(buf, key) ok := encode_attribute_value(buf, value) if !ok { return false } } } return true } // ============================================================================ // Decoding (Binary → Item) // ============================================================================ // Binary decoder helper Binary_Decoder :: struct { data: []byte, pos: int, } decoder_read_byte :: proc(decoder: ^Binary_Decoder) -> (u8, bool) { if decoder.pos >= len(decoder.data) { return 0, false } byte := decoder.data[decoder.pos] decoder.pos += 1 return byte, true } decoder_read_bytes :: proc(decoder: ^Binary_Decoder, length: int) -> ([]byte, bool) { if decoder.pos + length > len(decoder.data) { return nil, false } bytes := decoder.data[decoder.pos:decoder.pos + length] decoder.pos += length return bytes, true } decoder_read_varint :: proc(decoder: ^Binary_Decoder) -> (int, bool) { result: int = 0 shift: uint = 0 for decoder.pos < len(decoder.data) { byte := decoder.data[decoder.pos] decoder.pos += 1 result |= int(byte & 0x7F) << shift if (byte & 0x80) == 0 { return result, true } shift += 7 if shift >= 64 { return 0, false // Varint overflow } } return 0, false // Unexpected end of data } // Decode binary TLV format back into an Item decode :: proc(data: []byte) -> (Item, bool) { decoder := Binary_Decoder{data = data, pos = 0} attr_count, count_ok := decoder_read_varint(&decoder) if !count_ok { return {}, false } item := make(Item) for _ in 0.. (Attribute_Value, bool) { type_byte, type_ok := decoder_read_byte(decoder) if !type_ok { return nil, false } type_tag := Type_Tag(type_byte) switch type_tag { case .String: length, len_ok := decoder_read_varint(decoder) if !len_ok { return nil, false } data, data_ok := decoder_read_bytes(decoder, length) if !data_ok { return nil, false } str := string(data) owned := transmute(string)slice.clone(transmute([]byte)str) return String(owned), true case .Number: length, len_ok := decoder_read_varint(decoder) if !len_ok { return nil, false } data, data_ok := decoder_read_bytes(decoder, length) if !data_ok { return nil, false } num_str := string(data) // Parse into DDB_Number ddb_num, num_ok := parse_ddb_number(num_str) if !num_ok { return nil, false } return ddb_num, true case .Binary: length, len_ok := decoder_read_varint(decoder) if !len_ok { return nil, false } data, data_ok := decoder_read_bytes(decoder, length) if !data_ok { return nil, false } str := string(data) owned := transmute(string)slice.clone(transmute([]byte)str) return Binary(owned), true case .Boolean: byte, byte_ok := decoder_read_byte(decoder) if !byte_ok { return nil, false } return Bool(byte != 0), true case .Null: return Null(true), true case .String_Set: count, count_ok := decoder_read_varint(decoder) if !count_ok { return nil, false } strings := make([]string, count) for i in 0..>= 7 if v == 0 { bytes.buffer_write_byte(buf, byte) return } else { bytes.buffer_write_byte(buf, byte | 0x80) } } }