From 7bbbe19ec0af6da4a357c8266bc768c267ff1f47 Mon Sep 17 00:00:00 2001 From: Yang Lian Date: Wed, 8 Sep 2021 14:35:28 +0800 Subject: [PATCH 1/9] add hash index to speed up searching --- docs/configuration.md | 5 +- inc/fdb_def.h | 26 +++++ src/fdb_kvdb.c | 229 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 258 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index fc30fc1..db6fce0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -38,4 +38,7 @@ The print function macro defines the configuration. When it is not configured by ## FDB_DEBUG_ENABLE -Enable debugging information output. When this configuration is closed, the system will not output logs for debugging. \ No newline at end of file +Enable debugging information output. When this configuration is closed, the system will not output logs for debugging. + +## FDB_KV_CACHE_HASHINDEX_ENHANCEMENT +Enable the enhancement of the KV cache, it will take more RAM but improve the search speed, it will create a hash-base index in memory. It will take 4-bytes for a key-value entry. \ No newline at end of file diff --git a/inc/fdb_def.h b/inc/fdb_def.h index 6989281..15bbd7a 100644 --- a/inc/fdb_def.h +++ b/inc/fdb_def.h @@ -37,6 +37,10 @@ extern "C" { #if (FDB_KV_CACHE_TABLE_SIZE > 0) && (FDB_SECTOR_CACHE_TABLE_SIZE > 0) #define FDB_KV_USING_CACHE +/* the KV cache use hashmap enhancement, it will take more RAM but improve the search speed */ +//#define FDB_KV_CACHE_HASH_ENHANCEMENT +#else +#undef FDB_KV_CACHE_HASH_ENHANCEMENT #endif #if defined(FDB_USING_FILE_LIBC_MODE) || defined(FDB_USING_FILE_POSIX_MODE) @@ -76,6 +80,7 @@ if (!(EXPR)) \ #define FDB_KVDB_CTRL_SET_FILE_MODE 0x09 /**< set file mode control command */ #define FDB_KVDB_CTRL_SET_MAX_SIZE 0x0A /**< set database max size in file mode control command */ #define FDB_KVDB_CTRL_SET_NOT_FORMAT 0x0B /**< set database NOT format mode control command */ +#define FDB_KVDB_CTRL_SET_HASH_OPS 0x0C /**< set hash operations if hash enhancement is enabled */ #define FDB_TSDB_CTRL_SET_SEC_SIZE 0x00 /**< set sector size control command */ #define FDB_TSDB_CTRL_GET_SEC_SIZE 0x01 /**< get sector size control command */ @@ -281,6 +286,21 @@ struct fdb_db { void *user_data; }; +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT +/* memory ops */ +struct fdb_cache_hash_enhancement_ops +{ + /* init call when need init, please return the size of memory that allocated */ + int (*init)(uint32_t default_value); + /* read the offset, return a 4-bytes data */ + uint32_t (*read)(long offset); + /* write a 4-bytes data */ + int (*write)(long offset, const uint32_t addr); +}; + +typedef struct fdb_cache_hash_enhancement_ops *fdb_cache_hash_enhancement_ops_t; +#endif + /* KVDB structure */ struct fdb_kvdb { struct fdb_db parent; /**< inherit from fdb_db */ @@ -296,6 +316,12 @@ struct fdb_kvdb { struct kv_cache_node kv_cache_table[FDB_KV_CACHE_TABLE_SIZE]; /* sector cache table, it caching the sector info which status is current using */ struct sector_cache_node sector_cache_table[FDB_SECTOR_CACHE_TABLE_SIZE]; +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + /* KV hash enhancement */ + size_t kv_cache_enm_total_size; + size_t kv_cache_enm_size; + fdb_cache_hash_enhancement_ops_t kv_cache_enm_ops; +#endif #endif /* FDB_KV_USING_CACHE */ #ifdef FDB_KV_AUTO_UPDATE diff --git a/src/fdb_kvdb.c b/src/fdb_kvdb.c index 59ed9f1..a60a227 100644 --- a/src/fdb_kvdb.c +++ b/src/fdb_kvdb.c @@ -167,8 +167,178 @@ static bool get_sector_from_cache(fdb_kvdb_t db, uint32_t sec_addr, uint32_t *em return false; } +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT +/* use hashmap implement */ +#define HASHMAP_MAX_CHAIN_LENGTH (8) + +static uint32_t kv_hash_int( + fdb_kvdb_t db, + const char *const keystring, + const size_t len) +{ + uint32_t key = fdb_calc_crc32(0, keystring, len); + + /* Robert Jenkins' 32 bit Mix Function */ + key += (key << 12); + key ^= (key >> 22); + key += (key << 4); + key ^= (key >> 9); + key += (key << 10); + key ^= (key >> 2); + key += (key << 7); + key ^= (key >> 12); + + /* Knuth's Multiplicative Method */ + key = (key >> 3) * 2654435761; + + return key % db->kv_cache_enm_total_size; +} + +static int kv_hash_compare(fdb_kvdb_t db, uint32_t curr, const char *const key, const unsigned len) +{ + char saved_name[FDB_KV_NAME_MAX]; + uint32_t addr; + + addr = db->kv_cache_enm_ops->read(curr); + + /* read the KV name in flash */ + _fdb_flash_read((fdb_db_t)db, addr + KV_HDR_DATA_SIZE, (uint32_t *)saved_name, FDB_KV_NAME_MAX); + if (!strncmp(key, saved_name, len)) { + return true; + } + return false; +} + + +static int kv_hash_find(fdb_kvdb_t db, const char *const key, const size_t len, + uint32_t *const out_index) +{ + unsigned int start, curr; + unsigned int i; + int total_in_use; + + /* If full, return immediately */ + if (db->kv_cache_enm_size >= db->kv_cache_enm_total_size) { + return 0; + } + + /* Find the best index */ + curr = start = kv_hash_int(db, key, len); + + /* First linear probe to check if we've already insert the element */ + total_in_use = 0; + + for (i = 0; i < HASHMAP_MAX_CHAIN_LENGTH; i++) { + bool in_use = (db->kv_cache_enm_ops->read(curr) != FDB_DATA_UNUSED); + if (in_use) { + total_in_use++; + } + + if (in_use && kv_hash_compare(db, curr, key, len)) { + *out_index = curr; + return 2; + } + + curr = (curr + 1) % db->kv_cache_enm_total_size; + } + + curr = start; + + /* Second linear probe to actually insert our element (only if there was + at least one empty entry) */ + if (HASHMAP_MAX_CHAIN_LENGTH > total_in_use) { + for (i = 0; i < HASHMAP_MAX_CHAIN_LENGTH; i++) { + if (db->kv_cache_enm_ops->read(curr) == FDB_DATA_UNUSED) { + *out_index = curr; + return 1; + } + + curr = (curr + 1) % db->kv_cache_enm_total_size; + } + } + + return 0; +} + + +static bool kv_hash_get(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t *addr) +{ + size_t curr; + size_t i; + + /* Find data location */ + curr = kv_hash_int(db, name, name_len); + + /* Linear probing, if necessary */ + for (i = 0; i < HASHMAP_MAX_CHAIN_LENGTH; i++) { + if (db->kv_cache_enm_ops->read(curr) != FDB_DATA_UNUSED) { + /* read the KV name in flash */ + if (kv_hash_compare(db, curr, name, name_len)) { + *addr = db->kv_cache_enm_ops->read(curr); + return true; + } + } + + curr = (curr + 1) % db->kv_cache_enm_total_size; + } + + return false; +} + +static int kv_hash_put(fdb_kvdb_t db, const char *const key, + const unsigned len, uint32_t addr) { + uint32_t index; + int ret = kv_hash_find(db, key, len, &index); + /* Find a place to put our value. */ + + if(ret) { + db->kv_cache_enm_ops->write(index, addr); + + switch(ret) { + case 1: + /* new insert */ + db->kv_cache_enm_size++; + break; + case 2: + /* update */ + if (addr!= FDB_DATA_UNUSED) { + // delete + db->kv_cache_enm_size--; + } + break; + } + } + return ret; +} + +static bool hash_index_cb(fdb_kv_t kv, void *arg1, void *arg2) +{ + bool value_is_str = true, print_value = false; + size_t *using_size = arg1; + fdb_kvdb_t db = arg2; + + if (kv->crc_is_ok) { + /* calculate the total using flash size */ + *using_size += kv->len; + /* check KV */ + if (kv->status == FDB_KV_WRITE) { + kv_hash_put(db, kv->name, kv->name_len,kv->addr.start); + } + } + + return false; +} +#endif + static void update_kv_cache(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t addr) { +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + if (kv_hash_put(db, name, name_len, addr)) { + return; + } + + { +#endif size_t i, empty_index = FDB_KV_CACHE_TABLE_SIZE, min_activity_index = FDB_KV_CACHE_TABLE_SIZE; uint16_t name_crc = (uint16_t) (fdb_calc_crc32(0, name, name_len) >> 16), min_activity = 0xFFFF; @@ -206,6 +376,9 @@ static void update_kv_cache(fdb_kvdb_t db, const char *name, size_t name_len, ui db->kv_cache_table[min_activity_index].name_crc = name_crc; db->kv_cache_table[min_activity_index].active = 0; } +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + } +#endif } /* @@ -213,6 +386,13 @@ static void update_kv_cache(fdb_kvdb_t db, const char *name, size_t name_len, ui */ static bool get_kv_from_cache(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t *addr) { +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + if (kv_hash_get(db, name, name_len, addr)) { + return true; + } + // new stack + { +#endif size_t i; uint16_t name_crc = (uint16_t) (fdb_calc_crc32(0, name, name_len) >> 16); @@ -232,7 +412,9 @@ static bool get_kv_from_cache(fdb_kvdb_t db, const char *name, size_t name_len, } } } - +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + } +#endif return false; } #endif /* FDB_KV_USING_CACHE */ @@ -1219,6 +1401,29 @@ static fdb_err_t set_kv(fdb_kvdb_t db, const char *key, const void *value_buf, s return result; } +static void kv_update_hash_index(fdb_kvdb_t db) +{ + struct fdb_kv kv; + size_t using_size = 0; + + if (!db_init_ok(db)) { + FDB_INFO("Error: KV (%s) isn't initialize OK.\n", db_name(db)); + return; + } + + /* lock the KV cache */ + db_lock(db); + + kv_iterator(db, &kv, &using_size, db, hash_index_cb); + + FDB_PRINT("\nmode: next generation\n"); + FDB_PRINT("size: %u/%u bytes.\n", (uint32_t)using_size + ((SECTOR_NUM - FDB_GC_EMPTY_SEC_THRESHOLD) * SECTOR_HDR_DATA_SIZE), + db_max_size(db) - db_sec_size(db) * FDB_GC_EMPTY_SEC_THRESHOLD); + + /* unlock the KV cache */ + db_unlock(db); +} + /** * Set a blob KV. If it blob value is NULL, delete it. * If not find it in flash, then create it. @@ -1578,6 +1783,13 @@ void fdb_kvdb_control(fdb_kvdb_t db, int cmd, void *arg) FDB_ASSERT(db->parent.init_ok == false); db->parent.not_formatable = *(bool *)arg; break; + case FDB_KVDB_CTRL_SET_HASH_OPS: +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + /* this change MUST before database initialization */ + FDB_ASSERT(db->parent.init_ok == false); + db->kv_cache_enm_ops = (fdb_cache_hash_enhancement_ops_t)arg; +#endif + break; } } @@ -1628,6 +1840,17 @@ fdb_err_t fdb_kvdb_init(fdb_kvdb_t db, const char *name, const char *part_name, for (i = 0; i < FDB_KV_CACHE_TABLE_SIZE; i++) { db->kv_cache_table[i].addr = FDB_DATA_UNUSED; } + +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + { + size_t size = 0; + db->kv_cache_enm_size = 0; + db->kv_cache_enm_total_size = 0; + + size = db->kv_cache_enm_ops->init(FDB_DATA_UNUSED); + db->kv_cache_enm_total_size = size >> 2; + } +#endif #endif /* FDB_KV_USING_CACHE */ FDB_DEBUG("KVDB size is %u bytes.\n", db_max_size(db)); @@ -1644,6 +1867,10 @@ fdb_err_t fdb_kvdb_init(fdb_kvdb_t db, const char *name, const char *part_name, _fdb_init_finish((fdb_db_t)db, result); +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + kv_update_hash_index(db); +#endif + return result; } From e3c7f55c09fd0f67c8b0aefd7d77181b726a47bf Mon Sep 17 00:00:00 2001 From: Yang Lian Date: Wed, 8 Sep 2021 14:45:10 +0800 Subject: [PATCH 2/9] comment it --- inc/fdb_def.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/inc/fdb_def.h b/inc/fdb_def.h index 15bbd7a..35e9983 100644 --- a/inc/fdb_def.h +++ b/inc/fdb_def.h @@ -287,7 +287,11 @@ struct fdb_db { }; #ifdef FDB_KV_CACHE_HASH_ENHANCEMENT -/* memory ops */ +/* memory ops, + In some complicated situations, the memory may use page-management, + so the functions designed to switch pages before accessing, + or meet other prerequisites. + */ struct fdb_cache_hash_enhancement_ops { /* init call when need init, please return the size of memory that allocated */ From d3a4923fabbfd46f305f1ec952852f04688e3c08 Mon Sep 17 00:00:00 2001 From: Yang Lian Date: Wed, 8 Sep 2021 14:46:13 +0800 Subject: [PATCH 3/9] Add hash index --- demos/linux/applications/main.c | 39 +++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/demos/linux/applications/main.c b/demos/linux/applications/main.c index 61f375a..89664e8 100644 --- a/demos/linux/applications/main.c +++ b/demos/linux/applications/main.c @@ -12,15 +12,17 @@ #define FDB_LOG_TAG "[main]" +static struct fdb_cache_hash_enhancement_ops hash_ops; + static pthread_mutex_t kv_locker, ts_locker; static uint32_t boot_count = 0; static time_t boot_time[10] = {0, 1, 2, 3}; /* default KV nodes */ static struct fdb_default_kv_node default_kv_table[] = { - {"username", "armink", 0}, /* string KV */ - {"password", "123456", 0}, /* string KV */ - {"boot_count", &boot_count, sizeof(boot_count)}, /* int type KV */ - {"boot_time", &boot_time, sizeof(boot_time)}, /* int array type KV */ + {"username", "armink", 0}, /* string KV */ + {"password", "123456", 0}, /* string KV */ + {"boot_count", &boot_count, sizeof(boot_count)}, /* int type KV */ + {"boot_time", &boot_time, sizeof(boot_time)}, /* int array type KV */ }; /* KVDB object */ static struct fdb_kvdb kvdb = { 0 }; @@ -49,12 +51,39 @@ static fdb_time_t get_time(void) return time(NULL); } +#include +#include +static uint32_t *hash_memory; + +static int hash_init(uint32_t default_value) +{ + size_t size = 4096 * 4; + hash_memory = (uint32_t *)malloc(size); + memset(hash_memory, default_value, size); + return size; +} + +static uint32_t hash_read(long offset) +{ + return hash_memory[offset]; +} + +static int hash_write(long offset, const uint32_t addr) +{ + hash_memory[offset] = addr; + return 1; +} + int main(void) { fdb_err_t result; bool file_mode = true; uint32_t sec_size = 4096, db_size = sec_size * 4; + hash_ops.init = hash_init; + hash_ops.read = hash_read; + hash_ops.write = hash_write; + #ifdef FDB_USING_KVDB { /* KVDB Sample */ struct fdb_default_kv default_kv; @@ -72,6 +101,8 @@ int main(void) fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_FILE_MODE, &file_mode); /* create database directory */ mkdir("fdb_kvdb1", 0777); + + fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_HASH_OPS, &hash_ops); /* Key-Value database initialization * * &kvdb: database object From 9de97c99cc106c652531d66364e9dd018d5f5b68 Mon Sep 17 00:00:00 2001 From: Yang Lian Date: Thu, 9 Sep 2021 10:34:13 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E5=88=A0=E9=99=A4=E4=BA=86=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E7=9A=84=E6=9B=B4=E6=96=B0=E7=9A=84=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E6=8F=90=E5=8D=87=E5=88=9D=E5=A7=8B=E5=8C=96=E9=80=9F?= =?UTF-8?q?=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _fdb_kv_load() 函数里面本来就会更新缓存，刚开始没有注意到 这次删除了重复更新Hash的代码 --- src/fdb_kvdb.c | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/src/fdb_kvdb.c b/src/fdb_kvdb.c index a60a227..f273c73 100644 --- a/src/fdb_kvdb.c +++ b/src/fdb_kvdb.c @@ -310,24 +310,6 @@ static int kv_hash_put(fdb_kvdb_t db, const char *const key, } return ret; } - -static bool hash_index_cb(fdb_kv_t kv, void *arg1, void *arg2) -{ - bool value_is_str = true, print_value = false; - size_t *using_size = arg1; - fdb_kvdb_t db = arg2; - - if (kv->crc_is_ok) { - /* calculate the total using flash size */ - *using_size += kv->len; - /* check KV */ - if (kv->status == FDB_KV_WRITE) { - kv_hash_put(db, kv->name, kv->name_len,kv->addr.start); - } - } - - return false; -} #endif static void update_kv_cache(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t addr) @@ -1401,29 +1383,6 @@ static fdb_err_t set_kv(fdb_kvdb_t db, const char *key, const void *value_buf, s return result; } -static void kv_update_hash_index(fdb_kvdb_t db) -{ - struct fdb_kv kv; - size_t using_size = 0; - - if (!db_init_ok(db)) { - FDB_INFO("Error: KV (%s) isn't initialize OK.\n", db_name(db)); - return; - } - - /* lock the KV cache */ - db_lock(db); - - kv_iterator(db, &kv, &using_size, db, hash_index_cb); - - FDB_PRINT("\nmode: next generation\n"); - FDB_PRINT("size: %u/%u bytes.\n", (uint32_t)using_size + ((SECTOR_NUM - FDB_GC_EMPTY_SEC_THRESHOLD) * SECTOR_HDR_DATA_SIZE), - db_max_size(db) - db_sec_size(db) * FDB_GC_EMPTY_SEC_THRESHOLD); - - /* unlock the KV cache */ - db_unlock(db); -} - /** * Set a blob KV. If it blob value is NULL, delete it. * If not find it in flash, then create it. @@ -1867,10 +1826,6 @@ fdb_err_t fdb_kvdb_init(fdb_kvdb_t db, const char *name, const char *part_name, _fdb_init_finish((fdb_db_t)db, result); -#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT - kv_update_hash_index(db); -#endif - return result; } From 12ea0a237f29fb650d8d3b3bc9a463df1fcb69d4 Mon Sep 17 00:00:00 2001 From: Yang Lian Date: Thu, 9 Sep 2021 10:34:52 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20hash=20enhancement=20?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E7=9A=84=E6=B5=8B=E8=AF=95=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- demos/linux/applications/main.c | 31 +++++++++++++++++-- samples/kvdb_benchmark_sample.c | 54 +++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 samples/kvdb_benchmark_sample.c diff --git a/demos/linux/applications/main.c b/demos/linux/applications/main.c index 89664e8..72c18b7 100644 --- a/demos/linux/applications/main.c +++ b/demos/linux/applications/main.c @@ -9,10 +9,13 @@ #include #include #include +#include #define FDB_LOG_TAG "[main]" +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT static struct fdb_cache_hash_enhancement_ops hash_ops; +#endif static pthread_mutex_t kv_locker, ts_locker; static uint32_t boot_count = 0; @@ -34,6 +37,7 @@ static int counts = 0; extern void kvdb_basic_sample(fdb_kvdb_t kvdb); extern void kvdb_type_string_sample(fdb_kvdb_t kvdb); extern void kvdb_type_blob_sample(fdb_kvdb_t kvdb); +extern void kvdb_bench_sample(fdb_kvdb_t kvdb); extern void tsdb_sample(fdb_tsdb_t tsdb); static void lock(fdb_db_t db) @@ -51,38 +55,51 @@ static fdb_time_t get_time(void) return time(NULL); } +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT #include #include static uint32_t *hash_memory; +/* return the size of the memory block in byte, + the memory block is composed of uint32_t + The block size MUST BE a multiple of four.*/ static int hash_init(uint32_t default_value) { - size_t size = 4096 * 4; + size_t elements_cnt = 4096; + size_t size = 4 * elements_cnt; + hash_memory = (uint32_t *)malloc(size); memset(hash_memory, default_value, size); + return size; } +/* return an element of the memory block */ static uint32_t hash_read(long offset) { return hash_memory[offset]; } +/* write an element to the memory block */ static int hash_write(long offset, const uint32_t addr) { hash_memory[offset] = addr; return 1; } +#endif + int main(void) { fdb_err_t result; bool file_mode = true; - uint32_t sec_size = 4096, db_size = sec_size * 4; + uint32_t sec_size = 4096, db_size = sec_size * 1024; +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT hash_ops.init = hash_init; hash_ops.read = hash_read; hash_ops.write = hash_write; +#endif #ifdef FDB_USING_KVDB { /* KVDB Sample */ @@ -101,8 +118,9 @@ int main(void) fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_FILE_MODE, &file_mode); /* create database directory */ mkdir("fdb_kvdb1", 0777); - +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_HASH_OPS, &hash_ops); +#endif /* Key-Value database initialization * * &kvdb: database object @@ -124,6 +142,13 @@ int main(void) kvdb_type_string_sample(&kvdb); /* run blob KV samples */ kvdb_type_blob_sample(&kvdb); + + /* run bench test */ + { + clock_t stick = clock(); + kvdb_bench_sample(&kvdb); + FDB_INFO("\nBench time: %d\n\n",clock() - stick); + } } #endif /* FDB_USING_KVDB */ diff --git a/samples/kvdb_benchmark_sample.c b/samples/kvdb_benchmark_sample.c new file mode 100644 index 0000000..76d5dcd --- /dev/null +++ b/samples/kvdb_benchmark_sample.c @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2020, LianYang, + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @file + * @brief benchmark KV samples. + * + * write some KV entries and read it out + */ + +#include +#include + +#ifdef FDB_USING_KVDB + +#define FDB_LOG_TAG "[sample][kvdb][bench]" + +#define ROUND (200) + +void kvdb_bench_sample(fdb_kvdb_t kvdb) +{ + struct fdb_blob blob; + char bench_key[FDB_KV_NAME_MAX]; + char bench_value[FDB_KV_NAME_MAX]; + + FDB_INFO("==================== kvdb_bench_sample ====================\n"); + + if (fdb_kv_get(kvdb, "1")==NULL) + { /* SET the KV value */ + for (int i = 0; i Date: Thu, 9 Sep 2021 10:36:25 +0800 Subject: [PATCH 6/9] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20hash=20enhancement=20m?= =?UTF-8?q?acro?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- demos/linux/applications/fdb_cfg.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/demos/linux/applications/fdb_cfg.h b/demos/linux/applications/fdb_cfg.h index 73a65ef..eec2e13 100644 --- a/demos/linux/applications/fdb_cfg.h +++ b/demos/linux/applications/fdb_cfg.h @@ -25,6 +25,7 @@ /* Using file storage mode by POSIX file API, like open/read/write/close */ #define FDB_USING_FILE_POSIX_MODE +//#define FDB_USING_FILE_LIBC_MODE /* log print macro. default EF_PRINT macro is printf() */ /* #define FDB_PRINT(...) my_printf(__VA_ARGS__) */ @@ -32,4 +33,7 @@ /* print debug information */ #define FDB_DEBUG_ENABLE +/* enable hash enhancement, open it can speed up searhing */ +//#define FDB_KV_CACHE_HASH_ENHANCEMENT + #endif /* _FDB_CFG_H_ */ From c2ddea0dd4bd9c47a47b805cdba2d21090d40f17 Mon Sep 17 00:00:00 2001 From: Yang Lian Date: Thu, 9 Sep 2021 11:12:34 +0800 Subject: [PATCH 7/9] =?UTF-8?q?=E5=88=A0=E9=99=A4=E4=BE=A6=E6=B5=8B?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/fdb_kvdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fdb_kvdb.c b/src/fdb_kvdb.c index f273c73..8865f6a 100644 --- a/src/fdb_kvdb.c +++ b/src/fdb_kvdb.c @@ -301,7 +301,7 @@ static int kv_hash_put(fdb_kvdb_t db, const char *const key, break; case 2: /* update */ - if (addr!= FDB_DATA_UNUSED) { + if (addr == FDB_DATA_UNUSED) { // delete db->kv_cache_enm_size--; } From 57cfef858cb4201a956c8a2c7b541ae04f2c9280 Mon Sep 17 00:00:00 2001 From: Yang Lian Date: Sat, 11 Sep 2021 16:51:41 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E5=9C=A8=E6=9E=84=E5=BB=BA=E7=B4=A2?= =?UTF-8?q?=E5=BC=95(build=20index)=E7=9A=84=E6=97=B6=E5=80=99=EF=BC=8C?= =?UTF-8?q?=E5=81=87=E8=AE=BE=E6=B2=A1=E6=9C=89=E9=87=8D=E5=A4=8D=E7=9A=84?= =?UTF-8?q?=E9=94=AE=E5=80=BC=EF=BC=8C=E6=8F=90=E5=8D=87=E5=90=AF=E5=8A=A8?= =?UTF-8?q?=E9=80=9F=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/fdb_kvdb.c | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/fdb_kvdb.c b/src/fdb_kvdb.c index 8865f6a..6adae4c 100644 --- a/src/fdb_kvdb.c +++ b/src/fdb_kvdb.c @@ -211,7 +211,7 @@ static int kv_hash_compare(fdb_kvdb_t db, uint32_t curr, const char *const key, static int kv_hash_find(fdb_kvdb_t db, const char *const key, const size_t len, - uint32_t *const out_index) + uint32_t *const out_index, bool build) { unsigned int start, curr; unsigned int i; @@ -234,9 +234,13 @@ static int kv_hash_find(fdb_kvdb_t db, const char *const key, const size_t len, total_in_use++; } - if (in_use && kv_hash_compare(db, curr, key, len)) { - *out_index = curr; - return 2; + if (!build) + { + /* ignore update when build */ + if (in_use && kv_hash_compare(db, curr, key, len)) { + *out_index = curr; + return 2; + } } curr = (curr + 1) % db->kv_cache_enm_total_size; @@ -286,9 +290,9 @@ static bool kv_hash_get(fdb_kvdb_t db, const char *name, size_t name_len, uint32 } static int kv_hash_put(fdb_kvdb_t db, const char *const key, - const unsigned len, uint32_t addr) { + const unsigned len, uint32_t addr, bool build) { uint32_t index; - int ret = kv_hash_find(db, key, len, &index); + int ret = kv_hash_find(db, key, len, &index, build); /* Find a place to put our value. */ if(ret) { @@ -312,10 +316,13 @@ static int kv_hash_put(fdb_kvdb_t db, const char *const key, } #endif -static void update_kv_cache(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t addr) +#define _UNUSED_PARAM(x) +#define update_kv_cache(db, name, name_len, addr) update_kv_cache_impl(db, name, name_len, addr, false) + +static void update_kv_cache_impl(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t addr, bool build_index) { #ifdef FDB_KV_CACHE_HASH_ENHANCEMENT - if (kv_hash_put(db, name, name_len, addr)) { + if (kv_hash_put(db, name, name_len, addr, build_index)) { return; } @@ -324,6 +331,8 @@ static void update_kv_cache(fdb_kvdb_t db, const char *name, size_t name_len, ui size_t i, empty_index = FDB_KV_CACHE_TABLE_SIZE, min_activity_index = FDB_KV_CACHE_TABLE_SIZE; uint16_t name_crc = (uint16_t) (fdb_calc_crc32(0, name, name_len) >> 16), min_activity = 0xFFFF; + _UNUSED_PARAM(build); + for (i = 0; i < FDB_KV_CACHE_TABLE_SIZE; i++) { if (addr != FDB_DATA_UNUSED) { /* update the KV address in cache */ @@ -1642,7 +1651,7 @@ static bool check_and_recovery_kv_cb(fdb_kv_t kv, void *arg1, void *arg2) return true; } else if (kv->crc_is_ok && kv->status == FDB_KV_WRITE) { /* update the cache when first load */ - update_kv_cache(db, kv->name, kv->name_len, kv->addr.start); + update_kv_cache_impl(db, kv->name, kv->name_len, kv->addr.start, true); } return false; From 9c26f73304bae6a3d538a5459d250bd3c2292358 Mon Sep 17 00:00:00 2001 From: Yang Lian Date: Mon, 13 Sep 2021 11:00:01 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=E6=9B=BF=E6=8D=A2=E4=BA=86=20HASH=20?= =?UTF-8?q?=E7=AE=97=E6=B3=95=20=E5=A2=9E=E5=8A=A0=E4=BA=86=E5=86=B2?= =?UTF-8?q?=E7=AA=81=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- inc/fdb_def.h | 2 ++ src/fdb_kvdb.c | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/inc/fdb_def.h b/inc/fdb_def.h index 35e9983..3903b6c 100644 --- a/inc/fdb_def.h +++ b/inc/fdb_def.h @@ -322,6 +322,8 @@ struct fdb_kvdb { struct sector_cache_node sector_cache_table[FDB_SECTOR_CACHE_TABLE_SIZE]; #ifdef FDB_KV_CACHE_HASH_ENHANCEMENT /* KV hash enhancement */ + bool kv_cache_enm_collisions; + bool kv_cache_table_collisions; size_t kv_cache_enm_total_size; size_t kv_cache_enm_size; fdb_cache_hash_enhancement_ops_t kv_cache_enm_ops; diff --git a/src/fdb_kvdb.c b/src/fdb_kvdb.c index 6adae4c..3d02c8d 100644 --- a/src/fdb_kvdb.c +++ b/src/fdb_kvdb.c @@ -171,6 +171,7 @@ static bool get_sector_from_cache(fdb_kvdb_t db, uint32_t sec_addr, uint32_t *em /* use hashmap implement */ #define HASHMAP_MAX_CHAIN_LENGTH (8) +#if 0 static uint32_t kv_hash_int( fdb_kvdb_t db, const char *const keystring, @@ -194,6 +195,37 @@ static uint32_t kv_hash_int( return key % db->kv_cache_enm_total_size; } +#else + +/* + * 32 bit magic FNV-0 and FNV-1 prime + */ +#define FNV_32_PRIME ((uint32_t)0x01000193) + +static uint32_t kv_hash_int( + fdb_kvdb_t db, + const char *const keystring, + const size_t len) +{ + uint32_t hval = 0x811c9dc5; + unsigned char *bp = (unsigned char *)keystring; /* start of buffer */ + unsigned char *be = bp + len; /* beyond end of buffer */ + + /* + * FNV-1 hash each octet in the buffer + */ + while (bp < be) { + hval *= FNV_32_PRIME; + /* xor the bottom with the current octet */ + hval ^= (uint32_t)*bp++; + } + + /* return our new hash value */ + return hval % db->kv_cache_enm_total_size; +} + +#endif + static int kv_hash_compare(fdb_kvdb_t db, uint32_t curr, const char *const key, const unsigned len) { char saved_name[FDB_KV_NAME_MAX]; @@ -219,6 +251,8 @@ static int kv_hash_find(fdb_kvdb_t db, const char *const key, const size_t len, /* If full, return immediately */ if (db->kv_cache_enm_size >= db->kv_cache_enm_total_size) { + /* collisions ? */ + db->kv_cache_enm_collisions = true; return 0; } @@ -261,6 +295,8 @@ static int kv_hash_find(fdb_kvdb_t db, const char *const key, const size_t len, } } + /* collisions ? */ + db->kv_cache_enm_collisions = true; return 0; } @@ -369,6 +405,11 @@ static void update_kv_cache_impl(fdb_kvdb_t db, const char *name, size_t name_le } #ifdef FDB_KV_CACHE_HASH_ENHANCEMENT } + + if(!build_index) { + /* collisions */ + db->kv_cache_table_collisions = true; + } #endif } @@ -713,7 +754,13 @@ static bool find_kv(fdb_kvdb_t db, const char *key, fdb_kv_t kv) } #endif /* FDB_KV_USING_CACHE */ +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + if (db->kv_cache_enm_collisions && db->kv_cache_enm_total_size) { +#endif find_ok = find_kv_no_cache(db, key, kv); +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + } +#endif #ifdef FDB_KV_USING_CACHE if (find_ok) { @@ -1812,6 +1859,8 @@ fdb_err_t fdb_kvdb_init(fdb_kvdb_t db, const char *name, const char *part_name, #ifdef FDB_KV_CACHE_HASH_ENHANCEMENT { size_t size = 0; + db->kv_cache_enm_collisions = false; + db->kv_cache_table_collisions = false; db->kv_cache_enm_size = 0; db->kv_cache_enm_total_size = 0;