biondizzle/jormun-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

global secondary indexes

Browse Source
This commit is contained in:
biondizzle
2026-02-16 02:15:15 -05:00
parent 31e80ac572
commit 972e6ece5e
8 changed files with 1369 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
dynamodb/storage.odin
View File

@@ -84,7 +84,23 @@ table_metadata_destroy :: proc(metadata: ^Table_Metadata, allocator: mem.Allocat
}
delete(metadata.attribute_definitions, allocator)
// TODO: Free GSI/LSI if we implement them
// Free GSI definitions
if gsis, has_gsis := metadata.global_secondary_indexes.?; has_gsis {
for gsi in gsis {
delete(gsi.index_name, allocator)
for ks in gsi.key_schema {
delete(ks.attribute_name, allocator)
}
delete(gsi.key_schema, allocator)
if nka, has_nka := gsi.projection.non_key_attributes.?; has_nka {
for attr in nka {
delete(attr, allocator)
}
delete(nka, allocator)
}
}
delete(gsis, allocator)
}
}
// Get the partition key attribute name
@@ -187,7 +203,6 @@ remove_table_lock :: proc(engine: ^Storage_Engine, table_name: string) {
// Serialize table metadata to binary format
serialize_table_metadata :: proc(metadata: ^Table_Metadata) -> ([]byte, bool) {
// Create a temporary item to hold metadata
meta_item := make(Item, context.temp_allocator)
defer delete(meta_item)
@@ -200,7 +215,7 @@ serialize_table_metadata :: proc(metadata: ^Table_Metadata) -> ([]byte, bool) {
if i > 0 {
strings.write_string(&ks_builder, ",")
}
fmt.sbprintf(&ks_builder, `{"AttributeName":"%s","KeyType":"%s"}`,
fmt.sbprintf(&ks_builder, `{{"AttributeName":"%s","KeyType":"%s"}}`,
ks.attribute_name, key_type_to_string(ks.key_type))
}
strings.write_string(&ks_builder, "]")
@@ -216,7 +231,7 @@ serialize_table_metadata :: proc(metadata: ^Table_Metadata) -> ([]byte, bool) {
if i > 0 {
strings.write_string(&ad_builder, ",")
}
fmt.sbprintf(&ad_builder, `{"AttributeName":"%s","AttributeType":"%s"}`,
fmt.sbprintf(&ad_builder, `{{"AttributeName":"%s","AttributeType":"%s"}}`,
ad.attribute_name, scalar_type_to_string(ad.attribute_type))
}
strings.write_string(&ad_builder, "]")
@@ -227,6 +242,48 @@ serialize_table_metadata :: proc(metadata: ^Table_Metadata) -> ([]byte, bool) {
meta_item["TableStatus"] = String(strings.clone(table_status_to_string(metadata.table_status)))
meta_item["CreationDateTime"] = Number(fmt.aprint(metadata.creation_date_time))
// Encode GSI definitions as JSON string
if gsis, has_gsis := metadata.global_secondary_indexes.?; has_gsis && len(gsis) > 0 {
gsi_builder := strings.builder_make(context.temp_allocator)
defer strings.builder_destroy(&gsi_builder)
strings.write_string(&gsi_builder, "[")
for gsi, i in gsis {
if i > 0 {
strings.write_string(&gsi_builder, ",")
}
fmt.sbprintf(&gsi_builder, `{{"IndexName":"%s","KeySchema":[`, gsi.index_name)
for ks, j in gsi.key_schema {
if j > 0 {
strings.write_string(&gsi_builder, ",")
}
fmt.sbprintf(&gsi_builder, `{{"AttributeName":"%s","KeyType":"%s"}}`,
ks.attribute_name, key_type_to_string(ks.key_type))
}
strings.write_string(&gsi_builder, `],"Projection":{{"ProjectionType":"`)
switch gsi.projection.projection_type {
case .ALL: strings.write_string(&gsi_builder, "ALL")
case .KEYS_ONLY: strings.write_string(&gsi_builder, "KEYS_ONLY")
case .INCLUDE: strings.write_string(&gsi_builder, "INCLUDE")
}
strings.write_string(&gsi_builder, `"`)
if nka, has_nka := gsi.projection.non_key_attributes.?; has_nka && len(nka) > 0 {
strings.write_string(&gsi_builder, `,"NonKeyAttributes":[`)
for attr, k in nka {
if k > 0 {
strings.write_string(&gsi_builder, ",")
}
fmt.sbprintf(&gsi_builder, `"%s"`, attr)
}
strings.write_string(&gsi_builder, "]")
}
strings.write_string(&gsi_builder, "}}")
}
strings.write_string(&gsi_builder, "]")
meta_item["GlobalSecondaryIndexes"] = String(strings.clone(strings.to_string(gsi_builder)))
}
// Encode to binary
return encode(meta_item)
}
@@ -282,6 +339,17 @@ deserialize_table_metadata :: proc(data: []byte, allocator: mem.Allocator) -> (T
}
}
// Parse GlobalSecondaryIndexes from embedded JSON string
if gsi_val, gsi_found := meta_item["GlobalSecondaryIndexes"]; gsi_found {
#partial switch v in gsi_val {
case String:
gsis, gsi_ok := parse_gsis_json(string(v), allocator)
if gsi_ok && len(gsis) > 0 {
metadata.global_secondary_indexes = gsis
}
}
}
return metadata, true
}
@@ -463,6 +531,7 @@ create_table :: proc(
table_name: string,
key_schema: []Key_Schema_Element,
attribute_definitions: []Attribute_Definition,
gsis: Maybe([]Global_Secondary_Index) = nil,
) -> (Table_Description, Storage_Error) {
table_lock := get_or_create_table_lock(engine, table_name)
sync.rw_mutex_lock(table_lock)
@@ -500,6 +569,34 @@ create_table :: proc(
ad.attribute_name = strings.clone(ad.attribute_name, engine.allocator)
}
// Deep copy GSI definitions into engine allocator
if gsi_defs, has_gsis := gsis.?; has_gsis && len(gsi_defs) > 0 {
owned_gsis := make([]Global_Secondary_Index, len(gsi_defs), engine.allocator)
for gsi_def, i in gsi_defs {
owned_gsis[i] = Global_Secondary_Index{
index_name = strings.clone(gsi_def.index_name, engine.allocator),
key_schema = make([]Key_Schema_Element, len(gsi_def.key_schema), engine.allocator),
projection = Projection{
projection_type = gsi_def.projection.projection_type,
},
}
for ks, j in gsi_def.key_schema {
owned_gsis[i].key_schema[j] = Key_Schema_Element{
attribute_name = strings.clone(ks.attribute_name, engine.allocator),
key_type = ks.key_type,
}
}
if nka, has_nka := gsi_def.projection.non_key_attributes.?; has_nka {
owned_nka := make([]string, len(nka), engine.allocator)
for attr, k in nka {
owned_nka[k] = strings.clone(attr, engine.allocator)
}
owned_gsis[i].projection.non_key_attributes = owned_nka
}
}
metadata.global_secondary_indexes = owned_gsis
}
// Serialize and store
meta_value, serialize_ok := serialize_table_metadata(&metadata)
if !serialize_ok {
@@ -522,6 +619,7 @@ create_table :: proc(
creation_date_time = now,
item_count = 0,
table_size_bytes = 0,
global_secondary_indexes = gsis,
}
return desc, .None
@@ -565,7 +663,6 @@ delete_table :: proc(engine: ^Storage_Engine, table_name: string) -> Storage_Err
break
}
// Delete this item
err: cstring
rocksdb.rocksdb_delete(
engine.db.handle,
@@ -582,6 +679,41 @@ delete_table :: proc(engine: ^Storage_Engine, table_name: string) -> Storage_Err
}
}
// Delete all GSI entries for this table
gsi_table_prefix := build_gsi_table_prefix(table_name)
defer delete(gsi_table_prefix)
gsi_iter := rocksdb.rocksdb_create_iterator(engine.db.handle, engine.db.read_options)
if gsi_iter != nil {
defer rocksdb.rocksdb_iter_destroy(gsi_iter)
rocksdb.rocksdb_iter_seek(gsi_iter, raw_data(gsi_table_prefix), c.size_t(len(gsi_table_prefix)))
for rocksdb.rocksdb_iter_valid(gsi_iter) != 0 {
key_len: c.size_t
key_ptr := rocksdb.rocksdb_iter_key(gsi_iter, &key_len)
key_bytes := key_ptr[:key_len]
if !has_prefix(key_bytes, gsi_table_prefix) {
break
}
err: cstring
rocksdb.rocksdb_delete(
engine.db.handle,
engine.db.write_options,
raw_data(key_bytes),
c.size_t(len(key_bytes)),
&err,
)
if err != nil {
rocksdb.rocksdb_free(rawptr(err))
}
rocksdb.rocksdb_iter_next(gsi_iter)
}
}
// Delete metadata
del_err := rocksdb.db_delete(&engine.db, meta_key)
if del_err != .None {
@@ -639,6 +771,17 @@ put_item :: proc(engine: ^Storage_Engine, table_name: string, item: Item) -> Sto
storage_key := build_data_key(table_name, key_values.pk, key_values.sk)
defer delete(storage_key)
// --- GSI cleanup: delete OLD GSI entries if item already exists ---
existing_value, existing_err := rocksdb.db_get(&engine.db, storage_key)
if existing_err == .None && existing_value != nil {
defer delete(existing_value)
old_item, decode_ok := decode(existing_value)
if decode_ok {
defer item_destroy(&old_item)
gsi_delete_entries(engine, table_name, old_item, &metadata)
}
}
// Encode item
encoded_item, encode_ok := encode(item)
if !encode_ok {
@@ -652,6 +795,12 @@ put_item :: proc(engine: ^Storage_Engine, table_name: string, item: Item) -> Sto
return .RocksDB_Error
}
// --- GSI maintenance: write NEW GSI entries ---
gsi_err := gsi_write_entries(engine, table_name, item, &metadata)
if gsi_err != .None {
return gsi_err
}
return .None
}
@@ -748,6 +897,17 @@ delete_item :: proc(engine: ^Storage_Engine, table_name: string, key: Item) -> S
storage_key := build_data_key(table_name, key_values.pk, key_values.sk)
defer delete(storage_key)
// --- GSI cleanup: read existing item to know which GSI entries to remove ---
existing_value, existing_err := rocksdb.db_get(&engine.db, storage_key)
if existing_err == .None && existing_value != nil {
defer delete(existing_value)
old_item, decode_ok := decode(existing_value)
if decode_ok {
defer item_destroy(&old_item)
gsi_delete_entries(engine, table_name, old_item, &metadata)
}
}
// Delete from RocksDB
del_err := rocksdb.db_delete(&engine.db, storage_key)
if del_err != .None {