consolidate
This commit is contained in:
2
Makefile
2
Makefile
@@ -14,7 +14,7 @@ SHIM_HDRS := $(SHIM_DIR)/rocksdb_shim.h
|
||||
|
||||
CXX := g++
|
||||
AR := ar
|
||||
CXXFLAGS := -O2 -fPIC -std=c++17 $(INCLUDE_PATH)
|
||||
CXXFLAGS := -O2 -fPIC -std=c++20 $(INCLUDE_PATH)
|
||||
|
||||
|
||||
# RocksDB and compression libraries
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
// Binary TLV (Type-Length-Value) encoding for DynamoDB items
|
||||
// Replaces JSON storage with efficient binary format
|
||||
// Format: [attribute_count][name_len][name][type_tag][value_len][value]...
|
||||
package item_codec
|
||||
package dynamodb
|
||||
|
||||
import "core:bytes"
|
||||
import "core:encoding/varint"
|
||||
import "core:slice"
|
||||
import "../dynamodb"
|
||||
|
||||
// Type tags for binary encoding (1 byte each)
|
||||
Type_Tag :: enum u8 {
|
||||
@@ -34,7 +32,7 @@ Type_Tag :: enum u8 {
|
||||
// Encode an Item to binary TLV format
|
||||
// Format: [attribute_count:varint][attributes...]
|
||||
// Each attribute: [name_len:varint][name:bytes][type_tag:u8][value_encoded:bytes]
|
||||
encode :: proc(item: dynamodb.Item) -> ([]byte, bool) {
|
||||
encode :: proc(item: Item) -> ([]byte, bool) {
|
||||
buf: bytes.Buffer
|
||||
bytes.buffer_init_allocator(&buf, 0, 1024, context.allocator)
|
||||
defer bytes.buffer_destroy(&buf)
|
||||
@@ -71,32 +69,32 @@ encode :: proc(item: dynamodb.Item) -> ([]byte, bool) {
|
||||
}
|
||||
|
||||
// Encode an AttributeValue to binary format
|
||||
encode_attribute_value :: proc(buf: ^bytes.Buffer, attr: dynamodb.Attribute_Value) -> bool {
|
||||
encode_attribute_value :: proc(buf: ^bytes.Buffer, attr: Attribute_Value) -> bool {
|
||||
switch v in attr {
|
||||
case dynamodb.String:
|
||||
case String:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.String))
|
||||
encode_varint(buf, len(v))
|
||||
bytes.buffer_write_string(buf, string(v))
|
||||
|
||||
case dynamodb.Number:
|
||||
case Number:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.Number))
|
||||
encode_varint(buf, len(v))
|
||||
bytes.buffer_write_string(buf, string(v))
|
||||
|
||||
case dynamodb.Binary:
|
||||
case Binary:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.Binary))
|
||||
encode_varint(buf, len(v))
|
||||
bytes.buffer_write_string(buf, string(v))
|
||||
|
||||
case dynamodb.Bool:
|
||||
case Bool:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.Boolean))
|
||||
bytes.buffer_write_byte(buf, 1 if bool(v) else 0)
|
||||
|
||||
case dynamodb.Null:
|
||||
case Null:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.Null))
|
||||
// NULL has no value bytes
|
||||
|
||||
case dynamodb.String_Set:
|
||||
case String_Set:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.String_Set))
|
||||
encode_varint(buf, len(v))
|
||||
for s in v {
|
||||
@@ -104,7 +102,7 @@ encode_attribute_value :: proc(buf: ^bytes.Buffer, attr: dynamodb.Attribute_Valu
|
||||
bytes.buffer_write_string(buf, s)
|
||||
}
|
||||
|
||||
case dynamodb.Number_Set:
|
||||
case Number_Set:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.Number_Set))
|
||||
encode_varint(buf, len(v))
|
||||
for n in v {
|
||||
@@ -112,7 +110,7 @@ encode_attribute_value :: proc(buf: ^bytes.Buffer, attr: dynamodb.Attribute_Valu
|
||||
bytes.buffer_write_string(buf, n)
|
||||
}
|
||||
|
||||
case dynamodb.Binary_Set:
|
||||
case Binary_Set:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.Binary_Set))
|
||||
encode_varint(buf, len(v))
|
||||
for b in v {
|
||||
@@ -120,7 +118,7 @@ encode_attribute_value :: proc(buf: ^bytes.Buffer, attr: dynamodb.Attribute_Valu
|
||||
bytes.buffer_write_string(buf, b)
|
||||
}
|
||||
|
||||
case dynamodb.List:
|
||||
case List:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.List))
|
||||
encode_varint(buf, len(v))
|
||||
for item in v {
|
||||
@@ -130,7 +128,7 @@ encode_attribute_value :: proc(buf: ^bytes.Buffer, attr: dynamodb.Attribute_Valu
|
||||
}
|
||||
}
|
||||
|
||||
case dynamodb.Map:
|
||||
case Map:
|
||||
bytes.buffer_write_byte(buf, u8(Type_Tag.Map))
|
||||
encode_varint(buf, len(v))
|
||||
|
||||
@@ -169,10 +167,6 @@ Binary_Decoder :: struct {
|
||||
pos: int,
|
||||
}
|
||||
|
||||
decoder_init :: proc(data: []byte) -> Binary_Decoder {
|
||||
return Binary_Decoder{data = data, pos = 0}
|
||||
}
|
||||
|
||||
decoder_read_byte :: proc(decoder: ^Binary_Decoder) -> (u8, bool) {
|
||||
if decoder.pos >= len(decoder.data) {
|
||||
return 0, false
|
||||
@@ -217,28 +211,28 @@ decoder_read_varint :: proc(decoder: ^Binary_Decoder) -> (int, bool) {
|
||||
}
|
||||
|
||||
// Decode binary TLV format back into an Item
|
||||
decode :: proc(data: []byte) -> (dynamodb.Item, bool) {
|
||||
decoder := decoder_init(data)
|
||||
decode :: proc(data: []byte) -> (Item, bool) {
|
||||
decoder := Binary_Decoder{data = data, pos = 0}
|
||||
|
||||
attr_count, count_ok := decoder_read_varint(&decoder)
|
||||
if !count_ok {
|
||||
return {}, false
|
||||
}
|
||||
|
||||
item := make(dynamodb.Item)
|
||||
item := make(Item)
|
||||
|
||||
for i in 0..<attr_count {
|
||||
for _ in 0..<attr_count {
|
||||
// Read attribute name
|
||||
name_len, name_len_ok := decoder_read_varint(&decoder)
|
||||
if !name_len_ok {
|
||||
// Cleanup on error
|
||||
dynamodb.item_destroy(&item)
|
||||
item_destroy(&item)
|
||||
return {}, false
|
||||
}
|
||||
|
||||
name_bytes, name_ok := decoder_read_bytes(&decoder, name_len)
|
||||
if !name_ok {
|
||||
dynamodb.item_destroy(&item)
|
||||
item_destroy(&item)
|
||||
return {}, false
|
||||
}
|
||||
|
||||
@@ -249,7 +243,7 @@ decode :: proc(data: []byte) -> (dynamodb.Item, bool) {
|
||||
value, value_ok := decode_attribute_value(&decoder)
|
||||
if !value_ok {
|
||||
delete(owned_name)
|
||||
dynamodb.item_destroy(&item)
|
||||
item_destroy(&item)
|
||||
return {}, false
|
||||
}
|
||||
|
||||
@@ -260,7 +254,7 @@ decode :: proc(data: []byte) -> (dynamodb.Item, bool) {
|
||||
}
|
||||
|
||||
// Decode an AttributeValue from binary format
|
||||
decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_Value, bool) {
|
||||
decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (Attribute_Value, bool) {
|
||||
type_byte, type_ok := decoder_read_byte(decoder)
|
||||
if !type_ok {
|
||||
return nil, false
|
||||
@@ -282,7 +276,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
|
||||
str := string(data)
|
||||
owned := transmute(string)slice.clone(transmute([]byte)str)
|
||||
return dynamodb.String(owned), true
|
||||
return String(owned), true
|
||||
|
||||
case .Number:
|
||||
length, len_ok := decoder_read_varint(decoder)
|
||||
@@ -297,7 +291,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
|
||||
str := string(data)
|
||||
owned := transmute(string)slice.clone(transmute([]byte)str)
|
||||
return dynamodb.Number(owned), true
|
||||
return Number(owned), true
|
||||
|
||||
case .Binary:
|
||||
length, len_ok := decoder_read_varint(decoder)
|
||||
@@ -312,7 +306,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
|
||||
str := string(data)
|
||||
owned := transmute(string)slice.clone(transmute([]byte)str)
|
||||
return dynamodb.Binary(owned), true
|
||||
return Binary(owned), true
|
||||
|
||||
case .Boolean:
|
||||
byte, byte_ok := decoder_read_byte(decoder)
|
||||
@@ -320,10 +314,10 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return dynamodb.Bool(byte != 0), true
|
||||
return Bool(byte != 0), true
|
||||
|
||||
case .Null:
|
||||
return dynamodb.Null(true), true
|
||||
return Null(true), true
|
||||
|
||||
case .String_Set:
|
||||
count, count_ok := decoder_read_varint(decoder)
|
||||
@@ -357,7 +351,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
strings[i] = transmute(string)slice.clone(transmute([]byte)str)
|
||||
}
|
||||
|
||||
return dynamodb.String_Set(strings), true
|
||||
return String_Set(strings), true
|
||||
|
||||
case .Number_Set:
|
||||
count, count_ok := decoder_read_varint(decoder)
|
||||
@@ -390,7 +384,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
numbers[i] = transmute(string)slice.clone(transmute([]byte)str)
|
||||
}
|
||||
|
||||
return dynamodb.Number_Set(numbers), true
|
||||
return Number_Set(numbers), true
|
||||
|
||||
case .Binary_Set:
|
||||
count, count_ok := decoder_read_varint(decoder)
|
||||
@@ -423,7 +417,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
binaries[i] = transmute(string)slice.clone(transmute([]byte)str)
|
||||
}
|
||||
|
||||
return dynamodb.Binary_Set(binaries), true
|
||||
return Binary_Set(binaries), true
|
||||
|
||||
case .List:
|
||||
count, count_ok := decoder_read_varint(decoder)
|
||||
@@ -431,7 +425,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
return nil, false
|
||||
}
|
||||
|
||||
list := make([]dynamodb.Attribute_Value, count)
|
||||
list := make([]Attribute_Value, count)
|
||||
|
||||
for i in 0..<count {
|
||||
value, value_ok := decode_attribute_value(decoder)
|
||||
@@ -439,7 +433,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
// Cleanup on error
|
||||
for j in 0..<i {
|
||||
item := list[j]
|
||||
dynamodb.attr_value_destroy(&item)
|
||||
attr_value_destroy(&item)
|
||||
}
|
||||
delete(list)
|
||||
return nil, false
|
||||
@@ -448,7 +442,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
list[i] = value
|
||||
}
|
||||
|
||||
return dynamodb.List(list), true
|
||||
return List(list), true
|
||||
|
||||
case .Map:
|
||||
count, count_ok := decoder_read_varint(decoder)
|
||||
@@ -456,9 +450,9 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
return nil, false
|
||||
}
|
||||
|
||||
attr_map := make(map[string]dynamodb.Attribute_Value)
|
||||
attr_map := make(map[string]Attribute_Value)
|
||||
|
||||
for i in 0..<count {
|
||||
for _ in 0..<count {
|
||||
// Read key
|
||||
key_len, key_len_ok := decoder_read_varint(decoder)
|
||||
if !key_len_ok {
|
||||
@@ -466,7 +460,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
for k, v in attr_map {
|
||||
delete(k)
|
||||
v_copy := v
|
||||
dynamodb.attr_value_destroy(&v_copy)
|
||||
attr_value_destroy(&v_copy)
|
||||
}
|
||||
delete(attr_map)
|
||||
return nil, false
|
||||
@@ -477,7 +471,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
for k, v in attr_map {
|
||||
delete(k)
|
||||
v_copy := v
|
||||
dynamodb.attr_value_destroy(&v_copy)
|
||||
attr_value_destroy(&v_copy)
|
||||
}
|
||||
delete(attr_map)
|
||||
return nil, false
|
||||
@@ -493,7 +487,7 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
for k, v in attr_map {
|
||||
delete(k)
|
||||
v_copy := v
|
||||
dynamodb.attr_value_destroy(&v_copy)
|
||||
attr_value_destroy(&v_copy)
|
||||
}
|
||||
delete(attr_map)
|
||||
return nil, false
|
||||
@@ -502,14 +496,14 @@ decode_attribute_value :: proc(decoder: ^Binary_Decoder) -> (dynamodb.Attribute_
|
||||
attr_map[owned_key] = value
|
||||
}
|
||||
|
||||
return dynamodb.Map(attr_map), true
|
||||
return Map(attr_map), true
|
||||
}
|
||||
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Varint Encoding
|
||||
// Varint Encoding (Encodes a varint length prefix)
|
||||
// ============================================================================
|
||||
|
||||
encode_varint :: proc(buf: ^bytes.Buffer, value: int) {
|
||||
@@ -4,7 +4,6 @@ package dynamodb
|
||||
|
||||
import "core:encoding/json"
|
||||
import "core:fmt"
|
||||
import "core:mem"
|
||||
import "core:slice"
|
||||
import "core:strings"
|
||||
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
package key_codec
|
||||
package dynamodb
|
||||
|
||||
import "core:bytes"
|
||||
import "core:encoding/varint"
|
||||
import "core:mem"
|
||||
|
||||
// Entity type prefix bytes for namespacing
|
||||
Entity_Type :: enum u8 {
|
||||
@@ -12,28 +10,52 @@ Entity_Type :: enum u8 {
|
||||
LSI = 0x04, // Local secondary index
|
||||
}
|
||||
|
||||
// Encode a varint length prefix
|
||||
encode_varint :: proc(buf: ^bytes.Buffer, value: int) {
|
||||
temp: [10]byte
|
||||
n := varint.encode_u64(temp[:], u64(value))
|
||||
bytes.buffer_write(buf, temp[:n])
|
||||
}
|
||||
|
||||
// Decode a varint length prefix
|
||||
// Decode a varint length prefix from a byte slice.
|
||||
// Reads starting at data[offset^] and advances offset^ past the varint on success.
|
||||
decode_varint :: proc(data: []byte, offset: ^int) -> (value: int, ok: bool) {
|
||||
if offset^ >= len(data) {
|
||||
i := offset^
|
||||
if i < 0 || i >= len(data) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
val, n := varint.decode_u64(data[offset^:])
|
||||
if n <= 0 {
|
||||
u: u64 = 0
|
||||
shift: u32 = 0
|
||||
|
||||
for {
|
||||
if i >= len(data) {
|
||||
return 0, false // truncated
|
||||
}
|
||||
|
||||
b := data[i]
|
||||
i += 1
|
||||
|
||||
u |= u64(b & 0x7F) << shift
|
||||
|
||||
if (b & 0x80) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
shift += 7
|
||||
if shift >= 64 {
|
||||
return 0, false // malformed / overflow
|
||||
}
|
||||
}
|
||||
|
||||
// ensure it fits in int on this platform
|
||||
max_int := int((~uint(0)) >> 1)
|
||||
if u > u64(max_int) {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
offset^ += n
|
||||
return int(val), true
|
||||
offset^ = i
|
||||
return int(u), true
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// Build metadata key: [meta][table_name]
|
||||
build_meta_key :: proc(table_name: string) -> []byte {
|
||||
buf: bytes.Buffer
|
||||
@@ -170,10 +192,6 @@ Key_Decoder :: struct {
|
||||
pos: int,
|
||||
}
|
||||
|
||||
decoder_init :: proc(data: []byte) -> Key_Decoder {
|
||||
return Key_Decoder{data = data, pos = 0}
|
||||
}
|
||||
|
||||
decoder_read_entity_type :: proc(decoder: ^Key_Decoder) -> (Entity_Type, bool) {
|
||||
if decoder.pos >= len(decoder.data) {
|
||||
return .Meta, false
|
||||
@@ -228,7 +246,7 @@ Decoded_Data_Key :: struct {
|
||||
}
|
||||
|
||||
decode_data_key :: proc(key: []byte) -> (result: Decoded_Data_Key, ok: bool) {
|
||||
decoder := decoder_init(key)
|
||||
decoder := Key_Decoder{data = key, pos = 0}
|
||||
|
||||
// Read and verify entity type
|
||||
entity_type := decoder_read_entity_type(&decoder) or_return
|
||||
@@ -1,15 +1,12 @@
|
||||
// Storage engine mapping DynamoDB operations to RocksDB
|
||||
package dynamodb
|
||||
|
||||
import "core:encoding/json"
|
||||
import "core:fmt"
|
||||
import "core:mem"
|
||||
import "core:slice"
|
||||
import "core:strings"
|
||||
import "core:sync"
|
||||
import "core:time"
|
||||
import "../key_codec"
|
||||
import "../item_codec"
|
||||
import "../rocksdb"
|
||||
|
||||
Storage_Error :: enum {
|
||||
@@ -218,12 +215,12 @@ serialize_table_metadata :: proc(metadata: ^Table_Metadata) -> ([]byte, bool) {
|
||||
meta_item["CreationDateTime"] = Number(fmt.aprint(metadata.creation_date_time))
|
||||
|
||||
// Encode to binary
|
||||
return item_codec.encode(meta_item)
|
||||
return encode(meta_item)
|
||||
}
|
||||
|
||||
// Deserialize table metadata from binary format
|
||||
deserialize_table_metadata :: proc(data: []byte, allocator: mem.Allocator) -> (Table_Metadata, bool) {
|
||||
meta_item, ok := item_codec.decode(data)
|
||||
meta_item, ok := decode(data)
|
||||
if !ok {
|
||||
return {}, false
|
||||
}
|
||||
@@ -239,7 +236,7 @@ deserialize_table_metadata :: proc(data: []byte, allocator: mem.Allocator) -> (T
|
||||
|
||||
// Get table metadata
|
||||
get_table_metadata :: proc(engine: ^Storage_Engine, table_name: string) -> (Table_Metadata, Storage_Error) {
|
||||
meta_key := key_codec.build_meta_key(table_name)
|
||||
meta_key := build_meta_key(table_name)
|
||||
defer delete(meta_key)
|
||||
|
||||
value, get_err := rocksdb.db_get(&engine.db, meta_key)
|
||||
@@ -275,7 +272,7 @@ create_table :: proc(
|
||||
defer sync.rw_mutex_unlock(table_lock)
|
||||
|
||||
// Check if table already exists
|
||||
meta_key := key_codec.build_meta_key(table_name)
|
||||
meta_key := build_meta_key(table_name)
|
||||
defer delete(meta_key)
|
||||
|
||||
existing, get_err := rocksdb.db_get(&engine.db, meta_key)
|
||||
@@ -340,7 +337,7 @@ delete_table :: proc(engine: ^Storage_Engine, table_name: string) -> Storage_Err
|
||||
defer sync.rw_mutex_unlock(table_lock)
|
||||
|
||||
// Check table exists
|
||||
meta_key := key_codec.build_meta_key(table_name)
|
||||
meta_key := build_meta_key(table_name)
|
||||
defer delete(meta_key)
|
||||
|
||||
existing, get_err := rocksdb.db_get(&engine.db, meta_key)
|
||||
@@ -403,11 +400,11 @@ put_item :: proc(engine: ^Storage_Engine, table_name: string, item: Item) -> Sto
|
||||
}
|
||||
|
||||
// Build storage key
|
||||
storage_key := key_codec.build_data_key(table_name, key_values.pk, key_values.sk)
|
||||
storage_key := build_data_key(table_name, key_values.pk, key_values.sk)
|
||||
defer delete(storage_key)
|
||||
|
||||
// Encode item
|
||||
encoded_item, encode_ok := item_codec.encode(item)
|
||||
encoded_item, encode_ok := encode(item)
|
||||
if !encode_ok {
|
||||
return .Serialization_Error
|
||||
}
|
||||
@@ -456,7 +453,7 @@ get_item :: proc(engine: ^Storage_Engine, table_name: string, key: Item) -> (May
|
||||
}
|
||||
|
||||
// Build storage key
|
||||
storage_key := key_codec.build_data_key(table_name, key_values.pk, key_values.sk)
|
||||
storage_key := build_data_key(table_name, key_values.pk, key_values.sk)
|
||||
defer delete(storage_key)
|
||||
|
||||
// Get from RocksDB
|
||||
@@ -470,7 +467,7 @@ get_item :: proc(engine: ^Storage_Engine, table_name: string, key: Item) -> (May
|
||||
defer delete(value)
|
||||
|
||||
// Decode item
|
||||
item, decode_ok := item_codec.decode(value)
|
||||
item, decode_ok := decode(value)
|
||||
if !decode_ok {
|
||||
return nil, .Serialization_Error
|
||||
}
|
||||
@@ -512,7 +509,7 @@ delete_item :: proc(engine: ^Storage_Engine, table_name: string, key: Item) -> S
|
||||
}
|
||||
|
||||
// Build storage key
|
||||
storage_key := key_codec.build_data_key(table_name, key_values.pk, key_values.sk)
|
||||
storage_key := build_data_key(table_name, key_values.pk, key_values.sk)
|
||||
defer delete(storage_key)
|
||||
|
||||
// Delete from RocksDB
|
||||
|
||||
@@ -113,7 +113,7 @@ Key_Values :: struct {
|
||||
key_get_values :: proc(key: ^Key) -> (Key_Values, bool) {
|
||||
pk_bytes: []byte
|
||||
|
||||
switch v in key.pk {
|
||||
#partial switch v in key.pk {
|
||||
case String:
|
||||
pk_bytes = transmute([]byte)string(v)
|
||||
case Number:
|
||||
@@ -121,12 +121,13 @@ key_get_values :: proc(key: ^Key) -> (Key_Values, bool) {
|
||||
case Binary:
|
||||
pk_bytes = transmute([]byte)string(v)
|
||||
case:
|
||||
// Keys should only be scalar types (S, N, or B)
|
||||
return {}, false
|
||||
}
|
||||
|
||||
sk_bytes: Maybe([]byte)
|
||||
if sk, ok := key.sk.?; ok {
|
||||
switch v in sk {
|
||||
#partial switch v in sk {
|
||||
case String:
|
||||
sk_bytes = transmute([]byte)string(v)
|
||||
case Number:
|
||||
@@ -134,6 +135,7 @@ key_get_values :: proc(key: ^Key) -> (Key_Values, bool) {
|
||||
case Binary:
|
||||
sk_bytes = transmute([]byte)string(v)
|
||||
case:
|
||||
// Keys should only be scalar types
|
||||
return {}, false
|
||||
}
|
||||
}
|
||||
@@ -410,30 +412,35 @@ attr_value_destroy :: proc(attr: ^Attribute_Value) {
|
||||
for s in v {
|
||||
delete(s)
|
||||
}
|
||||
delete([]string(v))
|
||||
slice := v
|
||||
delete(slice)
|
||||
case Number_Set:
|
||||
for n in v {
|
||||
delete(n)
|
||||
}
|
||||
delete([]string(v))
|
||||
slice := v
|
||||
delete(slice)
|
||||
case Binary_Set:
|
||||
for b in v {
|
||||
delete(b)
|
||||
}
|
||||
delete([]string(v))
|
||||
slice := v
|
||||
delete(slice)
|
||||
case List:
|
||||
for item in v {
|
||||
item_copy := item
|
||||
attr_value_destroy(&item_copy)
|
||||
}
|
||||
delete([]Attribute_Value(v))
|
||||
list := v
|
||||
delete(list)
|
||||
case Map:
|
||||
for key, val in v {
|
||||
delete(key)
|
||||
val_copy := val
|
||||
attr_value_destroy(&val_copy)
|
||||
}
|
||||
delete(map[string]Attribute_Value(v))
|
||||
m := v
|
||||
delete(m)
|
||||
case Bool, Null:
|
||||
// Nothing to free
|
||||
}
|
||||
|
||||
636
main.odin
636
main.odin
@@ -1,11 +1,12 @@
|
||||
package main
|
||||
|
||||
import "core:encoding/json"
|
||||
import "core:fmt"
|
||||
import "core:mem"
|
||||
import "core:os"
|
||||
import "core:strconv"
|
||||
//import "core:strings" // I know we'll use in future but because we're not right now, compiler is complaining
|
||||
import "rocksdb"
|
||||
import "core:strings"
|
||||
import "dynamodb"
|
||||
|
||||
Config :: struct {
|
||||
host: string,
|
||||
@@ -25,12 +26,12 @@ main :: proc() {
|
||||
os.make_directory(config.data_dir)
|
||||
|
||||
// Initialize storage engine
|
||||
db, err := rocksdb.db_open(config.data_dir, true)
|
||||
engine, err := dynamodb.storage_engine_init(context.allocator, config.data_dir)
|
||||
if err != .None {
|
||||
fmt.eprintln("Failed to initialize storage:", err)
|
||||
os.exit(1)
|
||||
}
|
||||
defer rocksdb.db_close(&db)
|
||||
defer dynamodb.storage_engine_destroy(engine)
|
||||
|
||||
fmt.printfln("Storage engine initialized at %s", config.data_dir)
|
||||
fmt.printfln("Starting DynamoDB-compatible server on %s:%d", config.host, config.port)
|
||||
@@ -38,13 +39,12 @@ main :: proc() {
|
||||
// Create HTTP server
|
||||
server_config := default_server_config()
|
||||
|
||||
// For now, use a simple echo handler until we implement the full DynamoDB handler
|
||||
server, server_ok := server_init(
|
||||
context.allocator,
|
||||
config.host,
|
||||
config.port,
|
||||
handle_http_request,
|
||||
&db,
|
||||
handle_dynamodb_request,
|
||||
engine,
|
||||
server_config,
|
||||
)
|
||||
|
||||
@@ -63,30 +63,628 @@ main :: proc() {
|
||||
}
|
||||
}
|
||||
|
||||
// Temporary HTTP request handler
|
||||
// TODO: Replace with full DynamoDB handler once dynamodb/handler.odin is implemented
|
||||
handle_http_request :: proc(ctx: rawptr, request: ^HTTP_Request, request_alloc: mem.Allocator) -> HTTP_Response {
|
||||
//db := cast(^rocksdb.DB)ctx // I know we'll use in future but because we're not right now, compiler is complaining
|
||||
// DynamoDB request handler - called for each HTTP request with request-scoped arena allocator
|
||||
handle_dynamodb_request :: proc(ctx: rawptr, request: ^HTTP_Request, request_alloc: mem.Allocator) -> HTTP_Response {
|
||||
engine := cast(^dynamodb.Storage_Engine)ctx
|
||||
|
||||
// All allocations in this function use the request arena automatically
|
||||
response := response_init(request_alloc)
|
||||
response_add_header(&response, "Content-Type", "application/x-amz-json-1.0")
|
||||
response_add_header(&response, "x-amzn-RequestId", "local-request-id")
|
||||
|
||||
// Get X-Amz-Target header
|
||||
// Get X-Amz-Target header to determine operation
|
||||
target := request_get_header(request, "X-Amz-Target")
|
||||
if target == nil {
|
||||
return make_error_response(&response, .ValidationException, "Missing X-Amz-Target header")
|
||||
}
|
||||
|
||||
if t, ok := target.?; ok {
|
||||
// Echo back the operation for now
|
||||
body := fmt.aprintf("{{\"operation\":\"%s\",\"status\":\"not_implemented\"}}", t)
|
||||
response_set_body(&response, transmute([]byte)body)
|
||||
} else {
|
||||
response_set_status(&response, .Bad_Request)
|
||||
response_set_body(&response, transmute([]byte)string("{\"error\":\"Missing X-Amz-Target header\"}"))
|
||||
operation := dynamodb.operation_from_target(target.?)
|
||||
|
||||
// Route to appropriate handler
|
||||
#partial switch operation {
|
||||
case .CreateTable:
|
||||
handle_create_table(engine, request, &response)
|
||||
case .DeleteTable:
|
||||
handle_delete_table(engine, request, &response)
|
||||
case .DescribeTable:
|
||||
handle_describe_table(engine, request, &response)
|
||||
case .ListTables:
|
||||
handle_list_tables(engine, request, &response)
|
||||
case .PutItem:
|
||||
handle_put_item(engine, request, &response)
|
||||
case .GetItem:
|
||||
handle_get_item(engine, request, &response)
|
||||
case .DeleteItem:
|
||||
handle_delete_item(engine, request, &response)
|
||||
case .Query:
|
||||
handle_query(engine, request, &response)
|
||||
case .Scan:
|
||||
handle_scan(engine, request, &response)
|
||||
case .Unknown:
|
||||
return make_error_response(&response, .ValidationException, "Unknown operation")
|
||||
case:
|
||||
return make_error_response(&response, .ValidationException, "Operation not implemented")
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Table Operations
|
||||
// ============================================================================
|
||||
|
||||
handle_create_table :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
// Parse JSON body
|
||||
data, parse_err := json.parse(request.body, allocator = context.allocator)
|
||||
if parse_err != nil {
|
||||
make_error_response(response, .ValidationException, "Invalid JSON")
|
||||
return
|
||||
}
|
||||
defer json.destroy_value(data)
|
||||
|
||||
root, ok := data.(json.Object)
|
||||
if !ok {
|
||||
make_error_response(response, .ValidationException, "Request must be an object")
|
||||
return
|
||||
}
|
||||
|
||||
// Extract TableName
|
||||
table_name_val, found := root["TableName"]
|
||||
if !found {
|
||||
make_error_response(response, .ValidationException, "Missing TableName")
|
||||
return
|
||||
}
|
||||
|
||||
table_name, name_ok := table_name_val.(json.String)
|
||||
if !name_ok {
|
||||
make_error_response(response, .ValidationException, "TableName must be a string")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse KeySchema
|
||||
key_schema, ks_err := parse_key_schema(root)
|
||||
if ks_err != .None {
|
||||
msg := key_schema_error_message(ks_err)
|
||||
make_error_response(response, .ValidationException, msg)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse AttributeDefinitions
|
||||
attr_defs, ad_err := parse_attribute_definitions(root)
|
||||
if ad_err != .None {
|
||||
msg := attribute_definitions_error_message(ad_err)
|
||||
make_error_response(response, .ValidationException, msg)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate that key attributes are defined
|
||||
if !validate_key_attributes_defined(key_schema, attr_defs) {
|
||||
make_error_response(response, .ValidationException, "Key attribute not defined in AttributeDefinitions")
|
||||
return
|
||||
}
|
||||
|
||||
// Create the table
|
||||
desc, create_err := dynamodb.create_table(engine, string(table_name), key_schema, attr_defs)
|
||||
if create_err != .None {
|
||||
#partial switch create_err {
|
||||
case .Table_Already_Exists:
|
||||
make_error_response(response, .ResourceInUseException, "Table already exists")
|
||||
case:
|
||||
make_error_response(response, .InternalServerError, "Failed to create table")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Build response
|
||||
resp_body := fmt.aprintf(
|
||||
`{{"TableDescription":{{"TableName":"%s","TableStatus":"%s","CreationDateTime":%d}}}}`,
|
||||
desc.table_name,
|
||||
dynamodb.table_status_to_string(desc.table_status),
|
||||
desc.creation_date_time,
|
||||
)
|
||||
|
||||
response_set_body(response, transmute([]byte)resp_body)
|
||||
}
|
||||
|
||||
handle_delete_table :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
table_name, ok := dynamodb.parse_table_name(request.body)
|
||||
if !ok {
|
||||
make_error_response(response, .ValidationException, "Invalid request or missing TableName")
|
||||
return
|
||||
}
|
||||
|
||||
err := dynamodb.delete_table(engine, table_name)
|
||||
if err != .None {
|
||||
#partial switch err {
|
||||
case .Table_Not_Found:
|
||||
make_error_response(response, .ResourceNotFoundException, "Table not found")
|
||||
case:
|
||||
make_error_response(response, .InternalServerError, "Failed to delete table")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
resp_body := fmt.aprintf(`{{"TableDescription":{{"TableName":"%s","TableStatus":"DELETING"}}}}`, table_name)
|
||||
response_set_body(response, transmute([]byte)resp_body)
|
||||
}
|
||||
|
||||
handle_describe_table :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
table_name, ok := dynamodb.parse_table_name(request.body)
|
||||
if !ok {
|
||||
make_error_response(response, .ValidationException, "Invalid request or missing TableName")
|
||||
return
|
||||
}
|
||||
|
||||
metadata, err := dynamodb.get_table_metadata(engine, table_name)
|
||||
if err != .None {
|
||||
#partial switch err {
|
||||
case .Table_Not_Found:
|
||||
make_error_response(response, .ResourceNotFoundException, "Table not found")
|
||||
case:
|
||||
make_error_response(response, .InternalServerError, "Failed to describe table")
|
||||
}
|
||||
return
|
||||
}
|
||||
defer dynamodb.table_metadata_destroy(&metadata, context.allocator)
|
||||
|
||||
// Build response with key schema
|
||||
builder := strings.builder_make()
|
||||
strings.write_string(&builder, `{"Table":{"TableName":"`)
|
||||
strings.write_string(&builder, metadata.table_name)
|
||||
strings.write_string(&builder, `","TableStatus":"`)
|
||||
strings.write_string(&builder, dynamodb.table_status_to_string(metadata.table_status))
|
||||
strings.write_string(&builder, `","CreationDateTime":`)
|
||||
fmt.sbprintf(&builder, "%d", metadata.creation_date_time)
|
||||
strings.write_string(&builder, `,"KeySchema":[`)
|
||||
|
||||
for ks, i in metadata.key_schema {
|
||||
if i > 0 do strings.write_string(&builder, ",")
|
||||
fmt.sbprintf(&builder, `{"AttributeName":"%s","KeyType":"%s"}`,
|
||||
ks.attribute_name, dynamodb.key_type_to_string(ks.key_type))
|
||||
}
|
||||
|
||||
strings.write_string(&builder, `],"AttributeDefinitions":[`)
|
||||
|
||||
for ad, i in metadata.attribute_definitions {
|
||||
if i > 0 do strings.write_string(&builder, ",")
|
||||
fmt.sbprintf(&builder, `{"AttributeName":"%s","AttributeType":"%s"}`,
|
||||
ad.attribute_name, dynamodb.scalar_type_to_string(ad.attribute_type))
|
||||
}
|
||||
|
||||
strings.write_string(&builder, `]}}`)
|
||||
|
||||
resp_body := strings.to_string(builder)
|
||||
response_set_body(response, transmute([]byte)resp_body)
|
||||
}
|
||||
|
||||
handle_list_tables :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
_ = request // Not using request body for ListTables
|
||||
|
||||
tables := dynamodb.list_tables(engine)
|
||||
// list_tables returns []string which may be empty, not an error
|
||||
|
||||
// Build response
|
||||
builder := strings.builder_make()
|
||||
strings.write_string(&builder, `{"TableNames":[`)
|
||||
|
||||
for table, i in tables {
|
||||
if i > 0 do strings.write_string(&builder, ",")
|
||||
fmt.sbprintf(&builder, `"%s"`, table)
|
||||
}
|
||||
|
||||
strings.write_string(&builder, `]}`)
|
||||
|
||||
resp_body := strings.to_string(builder)
|
||||
response_set_body(response, transmute([]byte)resp_body)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Item Operations
|
||||
// ============================================================================
|
||||
|
||||
handle_put_item :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
table_name, ok := dynamodb.parse_table_name(request.body)
|
||||
if !ok {
|
||||
make_error_response(response, .ValidationException, "Invalid request or missing TableName")
|
||||
return
|
||||
}
|
||||
|
||||
item, item_ok := dynamodb.parse_item_from_request(request.body)
|
||||
if !item_ok {
|
||||
make_error_response(response, .ValidationException, "Invalid or missing Item")
|
||||
return
|
||||
}
|
||||
defer dynamodb.item_destroy(&item)
|
||||
|
||||
err := dynamodb.put_item(engine, table_name, item)
|
||||
if err != .None {
|
||||
#partial switch err {
|
||||
case .Table_Not_Found:
|
||||
make_error_response(response, .ResourceNotFoundException, "Table not found")
|
||||
case .Missing_Key_Attribute:
|
||||
make_error_response(response, .ValidationException, "Item missing required key attribute")
|
||||
case .Invalid_Key:
|
||||
make_error_response(response, .ValidationException, "Invalid key format")
|
||||
case:
|
||||
make_error_response(response, .InternalServerError, "Failed to put item")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
response_set_body(response, transmute([]byte)string("{}"))
|
||||
}
|
||||
|
||||
handle_get_item :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
table_name, ok := dynamodb.parse_table_name(request.body)
|
||||
if !ok {
|
||||
make_error_response(response, .ValidationException, "Invalid request or missing TableName")
|
||||
return
|
||||
}
|
||||
|
||||
key, key_ok := dynamodb.parse_key_from_request(request.body)
|
||||
if !key_ok {
|
||||
make_error_response(response, .ValidationException, "Invalid or missing Key")
|
||||
return
|
||||
}
|
||||
defer dynamodb.item_destroy(&key)
|
||||
|
||||
item, err := dynamodb.get_item(engine, table_name, key)
|
||||
if err != .None {
|
||||
#partial switch err {
|
||||
case .Table_Not_Found:
|
||||
make_error_response(response, .ResourceNotFoundException, "Table not found")
|
||||
case .Missing_Key_Attribute:
|
||||
make_error_response(response, .ValidationException, "Key missing required attributes")
|
||||
case .Invalid_Key:
|
||||
make_error_response(response, .ValidationException, "Invalid key format")
|
||||
case:
|
||||
make_error_response(response, .InternalServerError, "Failed to get item")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if item_val, has_item := item.?; has_item {
|
||||
defer dynamodb.item_destroy(&item_val)
|
||||
item_json := dynamodb.serialize_item(item_val)
|
||||
resp := fmt.aprintf(`{"Item":%s}`, item_json)
|
||||
response_set_body(response, transmute([]byte)resp)
|
||||
} else {
|
||||
response_set_body(response, transmute([]byte)string("{}"))
|
||||
}
|
||||
}
|
||||
|
||||
handle_delete_item :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
table_name, ok := dynamodb.parse_table_name(request.body)
|
||||
if !ok {
|
||||
make_error_response(response, .ValidationException, "Invalid request or missing TableName")
|
||||
return
|
||||
}
|
||||
|
||||
key, key_ok := dynamodb.parse_key_from_request(request.body)
|
||||
if !key_ok {
|
||||
make_error_response(response, .ValidationException, "Invalid or missing Key")
|
||||
return
|
||||
}
|
||||
defer dynamodb.item_destroy(&key)
|
||||
|
||||
err := dynamodb.delete_item(engine, table_name, key)
|
||||
if err != .None {
|
||||
#partial switch err {
|
||||
case .Table_Not_Found:
|
||||
make_error_response(response, .ResourceNotFoundException, "Table not found")
|
||||
case .Missing_Key_Attribute:
|
||||
make_error_response(response, .ValidationException, "Key missing required attributes")
|
||||
case .Invalid_Key:
|
||||
make_error_response(response, .ValidationException, "Invalid key format")
|
||||
case:
|
||||
make_error_response(response, .InternalServerError, "Failed to delete item")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
response_set_body(response, transmute([]byte)string("{}"))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Query and Scan Operations
|
||||
// ============================================================================
|
||||
|
||||
handle_query :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
_ = engine
|
||||
_ = request
|
||||
// For now, return not implemented
|
||||
// TODO: Implement KeyConditionExpression parsing and query logic
|
||||
make_error_response(response, .ValidationException, "Query operation not yet implemented")
|
||||
}
|
||||
|
||||
handle_scan :: proc(engine: ^dynamodb.Storage_Engine, request: ^HTTP_Request, response: ^HTTP_Response) {
|
||||
_ = engine
|
||||
_ = request
|
||||
|
||||
// TODO: Implement scan operation in storage.odin
|
||||
make_error_response(response, .ValidationException, "Scan operation not yet implemented")
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Schema Parsing Helpers
|
||||
// ============================================================================
|
||||
|
||||
Key_Schema_Error :: enum {
|
||||
None,
|
||||
Missing_Key_Schema,
|
||||
Invalid_Key_Schema,
|
||||
No_Hash_Key,
|
||||
Multiple_Hash_Keys,
|
||||
Multiple_Range_Keys,
|
||||
Invalid_Key_Type,
|
||||
}
|
||||
|
||||
parse_key_schema :: proc(root: json.Object) -> ([]dynamodb.Key_Schema_Element, Key_Schema_Error) {
|
||||
key_schema_val, found := root["KeySchema"]
|
||||
if !found do return nil, .Missing_Key_Schema
|
||||
|
||||
key_schema_array, ok := key_schema_val.(json.Array)
|
||||
if !ok do return nil, .Invalid_Key_Schema
|
||||
|
||||
if len(key_schema_array) == 0 || len(key_schema_array) > 2 {
|
||||
return nil, .Invalid_Key_Schema
|
||||
}
|
||||
|
||||
key_schema := make([]dynamodb.Key_Schema_Element, len(key_schema_array))
|
||||
|
||||
hash_count := 0
|
||||
range_count := 0
|
||||
|
||||
for elem, i in key_schema_array {
|
||||
elem_obj, elem_ok := elem.(json.Object)
|
||||
if !elem_ok {
|
||||
// Cleanup
|
||||
for j in 0..<i {
|
||||
delete(key_schema[j].attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .Invalid_Key_Schema
|
||||
}
|
||||
|
||||
// Get AttributeName
|
||||
attr_name_val, attr_found := elem_obj["AttributeName"]
|
||||
if !attr_found {
|
||||
for j in 0..<i {
|
||||
delete(key_schema[j].attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .Invalid_Key_Schema
|
||||
}
|
||||
|
||||
attr_name, name_ok := attr_name_val.(json.String)
|
||||
if !name_ok {
|
||||
for j in 0..<i {
|
||||
delete(key_schema[j].attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .Invalid_Key_Schema
|
||||
}
|
||||
|
||||
// Get KeyType
|
||||
key_type_val, type_found := elem_obj["KeyType"]
|
||||
if !type_found {
|
||||
for j in 0..<i {
|
||||
delete(key_schema[j].attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .Invalid_Key_Schema
|
||||
}
|
||||
|
||||
key_type_str, type_ok := key_type_val.(json.String)
|
||||
if !type_ok {
|
||||
for j in 0..<i {
|
||||
delete(key_schema[j].attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .Invalid_Key_Schema
|
||||
}
|
||||
|
||||
key_type, kt_ok := dynamodb.key_type_from_string(string(key_type_str))
|
||||
if !kt_ok {
|
||||
for j in 0..<i {
|
||||
delete(key_schema[j].attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .Invalid_Key_Type
|
||||
}
|
||||
|
||||
// Count key types
|
||||
switch key_type {
|
||||
case .HASH: hash_count += 1
|
||||
case .RANGE: range_count += 1
|
||||
}
|
||||
|
||||
key_schema[i] = dynamodb.Key_Schema_Element{
|
||||
attribute_name = strings.clone(string(attr_name)),
|
||||
key_type = key_type,
|
||||
}
|
||||
}
|
||||
|
||||
// Validate key counts
|
||||
if hash_count == 0 {
|
||||
for ks in key_schema {
|
||||
delete(ks.attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .No_Hash_Key
|
||||
}
|
||||
|
||||
if hash_count > 1 {
|
||||
for ks in key_schema {
|
||||
delete(ks.attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .Multiple_Hash_Keys
|
||||
}
|
||||
|
||||
if range_count > 1 {
|
||||
for ks in key_schema {
|
||||
delete(ks.attribute_name)
|
||||
}
|
||||
delete(key_schema)
|
||||
return nil, .Multiple_Range_Keys
|
||||
}
|
||||
|
||||
return key_schema, .None
|
||||
}
|
||||
|
||||
key_schema_error_message :: proc(err: Key_Schema_Error) -> string {
|
||||
switch err {
|
||||
case .None: return ""
|
||||
case .Missing_Key_Schema: return "Missing KeySchema"
|
||||
case .Invalid_Key_Schema: return "Invalid KeySchema format"
|
||||
case .No_Hash_Key: return "KeySchema must contain exactly one HASH key"
|
||||
case .Multiple_Hash_Keys: return "KeySchema can only contain one HASH key"
|
||||
case .Multiple_Range_Keys: return "KeySchema can only contain one RANGE key"
|
||||
case .Invalid_Key_Type: return "Invalid KeyType (must be HASH or RANGE)"
|
||||
}
|
||||
return "Invalid KeySchema"
|
||||
}
|
||||
|
||||
Attribute_Definitions_Error :: enum {
|
||||
None,
|
||||
Missing_Attribute_Definitions,
|
||||
Invalid_Attribute_Definitions,
|
||||
Invalid_Attribute_Type,
|
||||
Duplicate_Attribute_Name,
|
||||
}
|
||||
|
||||
parse_attribute_definitions :: proc(root: json.Object) -> ([]dynamodb.Attribute_Definition, Attribute_Definitions_Error) {
|
||||
attr_defs_val, found := root["AttributeDefinitions"]
|
||||
if !found do return nil, .Missing_Attribute_Definitions
|
||||
|
||||
attr_defs_array, ok := attr_defs_val.(json.Array)
|
||||
if !ok do return nil, .Invalid_Attribute_Definitions
|
||||
|
||||
if len(attr_defs_array) == 0 {
|
||||
return nil, .Invalid_Attribute_Definitions
|
||||
}
|
||||
|
||||
attr_defs := make([]dynamodb.Attribute_Definition, len(attr_defs_array))
|
||||
seen_names := make(map[string]bool, allocator = context.temp_allocator)
|
||||
defer delete(seen_names)
|
||||
|
||||
for elem, i in attr_defs_array {
|
||||
elem_obj, elem_ok := elem.(json.Object)
|
||||
if !elem_ok {
|
||||
for j in 0..<i {
|
||||
delete(attr_defs[j].attribute_name)
|
||||
}
|
||||
delete(attr_defs)
|
||||
return nil, .Invalid_Attribute_Definitions
|
||||
}
|
||||
|
||||
// Get AttributeName
|
||||
attr_name_val, name_found := elem_obj["AttributeName"]
|
||||
if !name_found {
|
||||
for j in 0..<i {
|
||||
delete(attr_defs[j].attribute_name)
|
||||
}
|
||||
delete(attr_defs)
|
||||
return nil, .Invalid_Attribute_Definitions
|
||||
}
|
||||
|
||||
attr_name, name_ok := attr_name_val.(json.String)
|
||||
if !name_ok {
|
||||
for j in 0..<i {
|
||||
delete(attr_defs[j].attribute_name)
|
||||
}
|
||||
delete(attr_defs)
|
||||
return nil, .Invalid_Attribute_Definitions
|
||||
}
|
||||
|
||||
// Check for duplicates
|
||||
if string(attr_name) in seen_names {
|
||||
for j in 0..<i {
|
||||
delete(attr_defs[j].attribute_name)
|
||||
}
|
||||
delete(attr_defs)
|
||||
return nil, .Duplicate_Attribute_Name
|
||||
}
|
||||
seen_names[string(attr_name)] = true
|
||||
|
||||
// Get AttributeType
|
||||
attr_type_val, type_found := elem_obj["AttributeType"]
|
||||
if !type_found {
|
||||
for j in 0..<i {
|
||||
delete(attr_defs[j].attribute_name)
|
||||
}
|
||||
delete(attr_defs)
|
||||
return nil, .Invalid_Attribute_Definitions
|
||||
}
|
||||
|
||||
attr_type_str, type_ok := attr_type_val.(json.String)
|
||||
if !type_ok {
|
||||
for j in 0..<i {
|
||||
delete(attr_defs[j].attribute_name)
|
||||
}
|
||||
delete(attr_defs)
|
||||
return nil, .Invalid_Attribute_Definitions
|
||||
}
|
||||
|
||||
attr_type, at_ok := dynamodb.scalar_type_from_string(string(attr_type_str))
|
||||
if !at_ok {
|
||||
for j in 0..<i {
|
||||
delete(attr_defs[j].attribute_name)
|
||||
}
|
||||
delete(attr_defs)
|
||||
return nil, .Invalid_Attribute_Type
|
||||
}
|
||||
|
||||
attr_defs[i] = dynamodb.Attribute_Definition{
|
||||
attribute_name = strings.clone(string(attr_name)),
|
||||
attribute_type = attr_type,
|
||||
}
|
||||
}
|
||||
|
||||
return attr_defs, .None
|
||||
}
|
||||
|
||||
attribute_definitions_error_message :: proc(err: Attribute_Definitions_Error) -> string {
|
||||
switch err {
|
||||
case .None: return ""
|
||||
case .Missing_Attribute_Definitions: return "Missing AttributeDefinitions"
|
||||
case .Invalid_Attribute_Definitions: return "Invalid AttributeDefinitions format"
|
||||
case .Invalid_Attribute_Type: return "Invalid AttributeType (must be S, N, or B)"
|
||||
case .Duplicate_Attribute_Name: return "Duplicate attribute name in AttributeDefinitions"
|
||||
}
|
||||
return "Invalid AttributeDefinitions"
|
||||
}
|
||||
|
||||
validate_key_attributes_defined :: proc(key_schema: []dynamodb.Key_Schema_Element, attr_defs: []dynamodb.Attribute_Definition) -> bool {
|
||||
for ks in key_schema {
|
||||
found := false
|
||||
for ad in attr_defs {
|
||||
if ks.attribute_name == ad.attribute_name {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found do return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Error Response Helper
|
||||
// ============================================================================
|
||||
|
||||
make_error_response :: proc(response: ^HTTP_Response, err_type: dynamodb.DynamoDB_Error_Type, message: string) -> HTTP_Response {
|
||||
response_set_status(response, .Bad_Request)
|
||||
error_body := dynamodb.error_to_response(err_type, message)
|
||||
response_set_body(response, transmute([]byte)error_body)
|
||||
return response^
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Configuration
|
||||
// ============================================================================
|
||||
|
||||
parse_config :: proc() -> Config {
|
||||
config := Config{
|
||||
host = "0.0.0.0",
|
||||
|
||||
@@ -1,22 +1,167 @@
|
||||
// TODO: In order to use RocksDB's WAL replication helpers, we need to import the C++ library so we use this shim
|
||||
/**
|
||||
C++ shim implementation notes (the important bits)
|
||||
#include "rocksdb_shim.h"
|
||||
#include <rocksdb/db.h>
|
||||
#include <rocksdb/options.h>
|
||||
#include <rocksdb/slice.h>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
|
||||
In this rocksdb_shim.cc we'll need to use:
|
||||
// Internal structure wrapping rocksdb::DB
|
||||
struct jormun_db {
|
||||
rocksdb::DB* db;
|
||||
};
|
||||
|
||||
rocksdb::DB::Open(...)
|
||||
// Placeholder for WAL iterator (not implemented yet)
|
||||
struct jormun_wal_iter {
|
||||
// TODO: Implement with TransactionLogIterator when needed
|
||||
void* placeholder;
|
||||
};
|
||||
|
||||
db->GetLatestSequenceNumber()
|
||||
// Open database
|
||||
jormun_db* jormun_db_open(const char* path, int create_if_missing, char** err) {
|
||||
rocksdb::Options options;
|
||||
options.create_if_missing = create_if_missing != 0;
|
||||
|
||||
db->GetUpdatesSince(seq, &iter)
|
||||
rocksdb::DB* db_ptr = nullptr;
|
||||
rocksdb::Status status = rocksdb::DB::Open(options, path, &db_ptr);
|
||||
|
||||
from each TransactionLogIterator entry:
|
||||
if (!status.ok()) {
|
||||
if (err) {
|
||||
std::string error_msg = status.ToString();
|
||||
*err = strdup(error_msg.c_str());
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
get WriteBatch and serialize via WriteBatch::Data()
|
||||
jormun_db* jdb = new jormun_db;
|
||||
jdb->db = db_ptr;
|
||||
return jdb;
|
||||
}
|
||||
|
||||
apply via rocksdb::WriteBatch wb(data); db->Write(write_options, &wb);
|
||||
// Close database
|
||||
void jormun_db_close(jormun_db* db) {
|
||||
if (db) {
|
||||
delete db->db;
|
||||
delete db;
|
||||
}
|
||||
}
|
||||
|
||||
Also we must configure WAL retention so the followers don’t fall off the end. RocksDB warns the iterator can become invalid if WAL is cleared aggressively; typical controls are WAL TTL / size limit.
|
||||
// Put key-value pair
|
||||
void jormun_db_put(jormun_db* db,
|
||||
const void* key, size_t keylen,
|
||||
const void* val, size_t vallen,
|
||||
char** err) {
|
||||
if (!db || !db->db) {
|
||||
if (err) *err = strdup("Database is null");
|
||||
return;
|
||||
}
|
||||
|
||||
https://github.com/facebook/rocksdb/issues/1565
|
||||
*/
|
||||
rocksdb::WriteOptions write_options;
|
||||
rocksdb::Slice key_slice(static_cast<const char*>(key), keylen);
|
||||
rocksdb::Slice val_slice(static_cast<const char*>(val), vallen);
|
||||
|
||||
rocksdb::Status status = db->db->Put(write_options, key_slice, val_slice);
|
||||
|
||||
if (!status.ok() && err) {
|
||||
std::string error_msg = status.ToString();
|
||||
*err = strdup(error_msg.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
// Get value for key
|
||||
unsigned char* jormun_db_get(jormun_db* db,
|
||||
const void* key, size_t keylen,
|
||||
size_t* vallen,
|
||||
char** err) {
|
||||
if (!db || !db->db) {
|
||||
if (err) *err = strdup("Database is null");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
rocksdb::ReadOptions read_options;
|
||||
rocksdb::Slice key_slice(static_cast<const char*>(key), keylen);
|
||||
std::string value;
|
||||
|
||||
rocksdb::Status status = db->db->Get(read_options, key_slice, &value);
|
||||
|
||||
if (status.IsNotFound()) {
|
||||
*vallen = 0;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (!status.ok()) {
|
||||
if (err) {
|
||||
std::string error_msg = status.ToString();
|
||||
*err = strdup(error_msg.c_str());
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Allocate and copy value
|
||||
*vallen = value.size();
|
||||
unsigned char* result = static_cast<unsigned char*>(malloc(value.size()));
|
||||
if (result) {
|
||||
memcpy(result, value.data(), value.size());
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Free memory allocated by the shim
|
||||
void jormun_free(void* p) {
|
||||
free(p);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// WAL Replication Functions (Stubs for now - to be implemented)
|
||||
// ============================================================================
|
||||
|
||||
// Get latest sequence number
|
||||
uint64_t jormun_latest_sequence(jormun_db* db) {
|
||||
if (!db || !db->db) return 0;
|
||||
return db->db->GetLatestSequenceNumber();
|
||||
}
|
||||
|
||||
// Create WAL iterator (stub)
|
||||
jormun_wal_iter* jormun_wal_iter_create(jormun_db* db, uint64_t seq, char** err) {
|
||||
(void)db;
|
||||
(void)seq;
|
||||
if (err) {
|
||||
*err = strdup("WAL iteration not yet implemented");
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Destroy WAL iterator (stub)
|
||||
void jormun_wal_iter_destroy(jormun_wal_iter* it) {
|
||||
if (it) {
|
||||
delete it;
|
||||
}
|
||||
}
|
||||
|
||||
// Get next batch from WAL (stub)
|
||||
int jormun_wal_iter_next(jormun_wal_iter* it,
|
||||
uint64_t* batch_start_seq,
|
||||
unsigned char** out_data,
|
||||
size_t* out_len,
|
||||
char** err) {
|
||||
(void)it;
|
||||
(void)batch_start_seq;
|
||||
(void)out_data;
|
||||
(void)out_len;
|
||||
if (err) {
|
||||
*err = strdup("WAL iteration not yet implemented");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Apply write batch (stub)
|
||||
void jormun_apply_writebatch(jormun_db* db,
|
||||
const unsigned char* data, size_t len,
|
||||
char** err) {
|
||||
(void)db;
|
||||
(void)data;
|
||||
(void)len;
|
||||
if (err) {
|
||||
*err = strdup("WAL apply not yet implemented");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user