diff --git a/demos/linux/applications/fdb_cfg.h b/demos/linux/applications/fdb_cfg.h index 73a65ef..eec2e13 100644 --- a/demos/linux/applications/fdb_cfg.h +++ b/demos/linux/applications/fdb_cfg.h @@ -25,6 +25,7 @@ /* Using file storage mode by POSIX file API, like open/read/write/close */ #define FDB_USING_FILE_POSIX_MODE +//#define FDB_USING_FILE_LIBC_MODE /* log print macro. default EF_PRINT macro is printf() */ /* #define FDB_PRINT(...) my_printf(__VA_ARGS__) */ @@ -32,4 +33,7 @@ /* print debug information */ #define FDB_DEBUG_ENABLE +/* enable hash enhancement, open it can speed up searhing */ +//#define FDB_KV_CACHE_HASH_ENHANCEMENT + #endif /* _FDB_CFG_H_ */ diff --git a/demos/linux/applications/main.c b/demos/linux/applications/main.c index 61f375a..72c18b7 100644 --- a/demos/linux/applications/main.c +++ b/demos/linux/applications/main.c @@ -9,18 +9,23 @@ #include #include #include +#include #define FDB_LOG_TAG "[main]" +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT +static struct fdb_cache_hash_enhancement_ops hash_ops; +#endif + static pthread_mutex_t kv_locker, ts_locker; static uint32_t boot_count = 0; static time_t boot_time[10] = {0, 1, 2, 3}; /* default KV nodes */ static struct fdb_default_kv_node default_kv_table[] = { - {"username", "armink", 0}, /* string KV */ - {"password", "123456", 0}, /* string KV */ - {"boot_count", &boot_count, sizeof(boot_count)}, /* int type KV */ - {"boot_time", &boot_time, sizeof(boot_time)}, /* int array type KV */ + {"username", "armink", 0}, /* string KV */ + {"password", "123456", 0}, /* string KV */ + {"boot_count", &boot_count, sizeof(boot_count)}, /* int type KV */ + {"boot_time", &boot_time, sizeof(boot_time)}, /* int array type KV */ }; /* KVDB object */ static struct fdb_kvdb kvdb = { 0 }; @@ -32,6 +37,7 @@ static int counts = 0; extern void kvdb_basic_sample(fdb_kvdb_t kvdb); extern void kvdb_type_string_sample(fdb_kvdb_t kvdb); extern void kvdb_type_blob_sample(fdb_kvdb_t kvdb); +extern void kvdb_bench_sample(fdb_kvdb_t kvdb); extern void tsdb_sample(fdb_tsdb_t tsdb); static void lock(fdb_db_t db) @@ -49,11 +55,51 @@ static fdb_time_t get_time(void) return time(NULL); } +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT +#include +#include +static uint32_t *hash_memory; + +/* return the size of the memory block in byte, + the memory block is composed of uint32_t + The block size MUST BE a multiple of four.*/ +static int hash_init(uint32_t default_value) +{ + size_t elements_cnt = 4096; + size_t size = 4 * elements_cnt; + + hash_memory = (uint32_t *)malloc(size); + memset(hash_memory, default_value, size); + + return size; +} + +/* return an element of the memory block */ +static uint32_t hash_read(long offset) +{ + return hash_memory[offset]; +} + +/* write an element to the memory block */ +static int hash_write(long offset, const uint32_t addr) +{ + hash_memory[offset] = addr; + return 1; +} +#endif + + int main(void) { fdb_err_t result; bool file_mode = true; - uint32_t sec_size = 4096, db_size = sec_size * 4; + uint32_t sec_size = 4096, db_size = sec_size * 1024; + +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + hash_ops.init = hash_init; + hash_ops.read = hash_read; + hash_ops.write = hash_write; +#endif #ifdef FDB_USING_KVDB { /* KVDB Sample */ @@ -72,6 +118,9 @@ int main(void) fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_FILE_MODE, &file_mode); /* create database directory */ mkdir("fdb_kvdb1", 0777); +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_HASH_OPS, &hash_ops); +#endif /* Key-Value database initialization * * &kvdb: database object @@ -93,6 +142,13 @@ int main(void) kvdb_type_string_sample(&kvdb); /* run blob KV samples */ kvdb_type_blob_sample(&kvdb); + + /* run bench test */ + { + clock_t stick = clock(); + kvdb_bench_sample(&kvdb); + FDB_INFO("\nBench time: %d\n\n",clock() - stick); + } } #endif /* FDB_USING_KVDB */ diff --git a/docs/configuration.md b/docs/configuration.md index fc30fc1..db6fce0 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -38,4 +38,7 @@ The print function macro defines the configuration. When it is not configured by ## FDB_DEBUG_ENABLE -Enable debugging information output. When this configuration is closed, the system will not output logs for debugging. \ No newline at end of file +Enable debugging information output. When this configuration is closed, the system will not output logs for debugging. + +## FDB_KV_CACHE_HASHINDEX_ENHANCEMENT +Enable the enhancement of the KV cache, it will take more RAM but improve the search speed, it will create a hash-base index in memory. It will take 4-bytes for a key-value entry. \ No newline at end of file diff --git a/inc/fdb_def.h b/inc/fdb_def.h index 6989281..3903b6c 100644 --- a/inc/fdb_def.h +++ b/inc/fdb_def.h @@ -37,6 +37,10 @@ extern "C" { #if (FDB_KV_CACHE_TABLE_SIZE > 0) && (FDB_SECTOR_CACHE_TABLE_SIZE > 0) #define FDB_KV_USING_CACHE +/* the KV cache use hashmap enhancement, it will take more RAM but improve the search speed */ +//#define FDB_KV_CACHE_HASH_ENHANCEMENT +#else +#undef FDB_KV_CACHE_HASH_ENHANCEMENT #endif #if defined(FDB_USING_FILE_LIBC_MODE) || defined(FDB_USING_FILE_POSIX_MODE) @@ -76,6 +80,7 @@ if (!(EXPR)) \ #define FDB_KVDB_CTRL_SET_FILE_MODE 0x09 /**< set file mode control command */ #define FDB_KVDB_CTRL_SET_MAX_SIZE 0x0A /**< set database max size in file mode control command */ #define FDB_KVDB_CTRL_SET_NOT_FORMAT 0x0B /**< set database NOT format mode control command */ +#define FDB_KVDB_CTRL_SET_HASH_OPS 0x0C /**< set hash operations if hash enhancement is enabled */ #define FDB_TSDB_CTRL_SET_SEC_SIZE 0x00 /**< set sector size control command */ #define FDB_TSDB_CTRL_GET_SEC_SIZE 0x01 /**< get sector size control command */ @@ -281,6 +286,25 @@ struct fdb_db { void *user_data; }; +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT +/* memory ops, + In some complicated situations, the memory may use page-management, + so the functions designed to switch pages before accessing, + or meet other prerequisites. + */ +struct fdb_cache_hash_enhancement_ops +{ + /* init call when need init, please return the size of memory that allocated */ + int (*init)(uint32_t default_value); + /* read the offset, return a 4-bytes data */ + uint32_t (*read)(long offset); + /* write a 4-bytes data */ + int (*write)(long offset, const uint32_t addr); +}; + +typedef struct fdb_cache_hash_enhancement_ops *fdb_cache_hash_enhancement_ops_t; +#endif + /* KVDB structure */ struct fdb_kvdb { struct fdb_db parent; /**< inherit from fdb_db */ @@ -296,6 +320,14 @@ struct fdb_kvdb { struct kv_cache_node kv_cache_table[FDB_KV_CACHE_TABLE_SIZE]; /* sector cache table, it caching the sector info which status is current using */ struct sector_cache_node sector_cache_table[FDB_SECTOR_CACHE_TABLE_SIZE]; +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + /* KV hash enhancement */ + bool kv_cache_enm_collisions; + bool kv_cache_table_collisions; + size_t kv_cache_enm_total_size; + size_t kv_cache_enm_size; + fdb_cache_hash_enhancement_ops_t kv_cache_enm_ops; +#endif #endif /* FDB_KV_USING_CACHE */ #ifdef FDB_KV_AUTO_UPDATE diff --git a/samples/kvdb_benchmark_sample.c b/samples/kvdb_benchmark_sample.c new file mode 100644 index 0000000..76d5dcd --- /dev/null +++ b/samples/kvdb_benchmark_sample.c @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2020, LianYang, + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * @file + * @brief benchmark KV samples. + * + * write some KV entries and read it out + */ + +#include +#include + +#ifdef FDB_USING_KVDB + +#define FDB_LOG_TAG "[sample][kvdb][bench]" + +#define ROUND (200) + +void kvdb_bench_sample(fdb_kvdb_t kvdb) +{ + struct fdb_blob blob; + char bench_key[FDB_KV_NAME_MAX]; + char bench_value[FDB_KV_NAME_MAX]; + + FDB_INFO("==================== kvdb_bench_sample ====================\n"); + + if (fdb_kv_get(kvdb, "1")==NULL) + { /* SET the KV value */ + for (int i = 0; i> 22); + key += (key << 4); + key ^= (key >> 9); + key += (key << 10); + key ^= (key >> 2); + key += (key << 7); + key ^= (key >> 12); + + /* Knuth's Multiplicative Method */ + key = (key >> 3) * 2654435761; + + return key % db->kv_cache_enm_total_size; +} + +#else + +/* + * 32 bit magic FNV-0 and FNV-1 prime + */ +#define FNV_32_PRIME ((uint32_t)0x01000193) + +static uint32_t kv_hash_int( + fdb_kvdb_t db, + const char *const keystring, + const size_t len) +{ + uint32_t hval = 0x811c9dc5; + unsigned char *bp = (unsigned char *)keystring; /* start of buffer */ + unsigned char *be = bp + len; /* beyond end of buffer */ + + /* + * FNV-1 hash each octet in the buffer + */ + while (bp < be) { + hval *= FNV_32_PRIME; + /* xor the bottom with the current octet */ + hval ^= (uint32_t)*bp++; + } + + /* return our new hash value */ + return hval % db->kv_cache_enm_total_size; +} + +#endif + +static int kv_hash_compare(fdb_kvdb_t db, uint32_t curr, const char *const key, const unsigned len) +{ + char saved_name[FDB_KV_NAME_MAX]; + uint32_t addr; + + addr = db->kv_cache_enm_ops->read(curr); + + /* read the KV name in flash */ + _fdb_flash_read((fdb_db_t)db, addr + KV_HDR_DATA_SIZE, (uint32_t *)saved_name, FDB_KV_NAME_MAX); + if (!strncmp(key, saved_name, len)) { + return true; + } + return false; +} + + +static int kv_hash_find(fdb_kvdb_t db, const char *const key, const size_t len, + uint32_t *const out_index, bool build) +{ + unsigned int start, curr; + unsigned int i; + int total_in_use; + + /* If full, return immediately */ + if (db->kv_cache_enm_size >= db->kv_cache_enm_total_size) { + /* collisions ? */ + db->kv_cache_enm_collisions = true; + return 0; + } + + /* Find the best index */ + curr = start = kv_hash_int(db, key, len); + + /* First linear probe to check if we've already insert the element */ + total_in_use = 0; + + for (i = 0; i < HASHMAP_MAX_CHAIN_LENGTH; i++) { + bool in_use = (db->kv_cache_enm_ops->read(curr) != FDB_DATA_UNUSED); + if (in_use) { + total_in_use++; + } + + if (!build) + { + /* ignore update when build */ + if (in_use && kv_hash_compare(db, curr, key, len)) { + *out_index = curr; + return 2; + } + } + + curr = (curr + 1) % db->kv_cache_enm_total_size; + } + + curr = start; + + /* Second linear probe to actually insert our element (only if there was + at least one empty entry) */ + if (HASHMAP_MAX_CHAIN_LENGTH > total_in_use) { + for (i = 0; i < HASHMAP_MAX_CHAIN_LENGTH; i++) { + if (db->kv_cache_enm_ops->read(curr) == FDB_DATA_UNUSED) { + *out_index = curr; + return 1; + } + + curr = (curr + 1) % db->kv_cache_enm_total_size; + } + } + + /* collisions ? */ + db->kv_cache_enm_collisions = true; + return 0; +} + + +static bool kv_hash_get(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t *addr) +{ + size_t curr; + size_t i; + + /* Find data location */ + curr = kv_hash_int(db, name, name_len); + + /* Linear probing, if necessary */ + for (i = 0; i < HASHMAP_MAX_CHAIN_LENGTH; i++) { + if (db->kv_cache_enm_ops->read(curr) != FDB_DATA_UNUSED) { + /* read the KV name in flash */ + if (kv_hash_compare(db, curr, name, name_len)) { + *addr = db->kv_cache_enm_ops->read(curr); + return true; + } + } + + curr = (curr + 1) % db->kv_cache_enm_total_size; + } + + return false; +} + +static int kv_hash_put(fdb_kvdb_t db, const char *const key, + const unsigned len, uint32_t addr, bool build) { + uint32_t index; + int ret = kv_hash_find(db, key, len, &index, build); + /* Find a place to put our value. */ + + if(ret) { + db->kv_cache_enm_ops->write(index, addr); + + switch(ret) { + case 1: + /* new insert */ + db->kv_cache_enm_size++; + break; + case 2: + /* update */ + if (addr == FDB_DATA_UNUSED) { + // delete + db->kv_cache_enm_size--; + } + break; + } + } + return ret; +} +#endif + +#define _UNUSED_PARAM(x) +#define update_kv_cache(db, name, name_len, addr) update_kv_cache_impl(db, name, name_len, addr, false) + +static void update_kv_cache_impl(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t addr, bool build_index) +{ +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + if (kv_hash_put(db, name, name_len, addr, build_index)) { + return; + } + + { +#endif size_t i, empty_index = FDB_KV_CACHE_TABLE_SIZE, min_activity_index = FDB_KV_CACHE_TABLE_SIZE; uint16_t name_crc = (uint16_t) (fdb_calc_crc32(0, name, name_len) >> 16), min_activity = 0xFFFF; + _UNUSED_PARAM(build); + for (i = 0; i < FDB_KV_CACHE_TABLE_SIZE; i++) { if (addr != FDB_DATA_UNUSED) { /* update the KV address in cache */ @@ -206,6 +403,14 @@ static void update_kv_cache(fdb_kvdb_t db, const char *name, size_t name_len, ui db->kv_cache_table[min_activity_index].name_crc = name_crc; db->kv_cache_table[min_activity_index].active = 0; } +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + } + + if(!build_index) { + /* collisions */ + db->kv_cache_table_collisions = true; + } +#endif } /* @@ -213,6 +418,13 @@ static void update_kv_cache(fdb_kvdb_t db, const char *name, size_t name_len, ui */ static bool get_kv_from_cache(fdb_kvdb_t db, const char *name, size_t name_len, uint32_t *addr) { +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + if (kv_hash_get(db, name, name_len, addr)) { + return true; + } + // new stack + { +#endif size_t i; uint16_t name_crc = (uint16_t) (fdb_calc_crc32(0, name, name_len) >> 16); @@ -232,7 +444,9 @@ static bool get_kv_from_cache(fdb_kvdb_t db, const char *name, size_t name_len, } } } - +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + } +#endif return false; } #endif /* FDB_KV_USING_CACHE */ @@ -540,7 +754,13 @@ static bool find_kv(fdb_kvdb_t db, const char *key, fdb_kv_t kv) } #endif /* FDB_KV_USING_CACHE */ +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + if (db->kv_cache_enm_collisions && db->kv_cache_enm_total_size) { +#endif find_ok = find_kv_no_cache(db, key, kv); +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + } +#endif #ifdef FDB_KV_USING_CACHE if (find_ok) { @@ -1478,7 +1698,7 @@ static bool check_and_recovery_kv_cb(fdb_kv_t kv, void *arg1, void *arg2) return true; } else if (kv->crc_is_ok && kv->status == FDB_KV_WRITE) { /* update the cache when first load */ - update_kv_cache(db, kv->name, kv->name_len, kv->addr.start); + update_kv_cache_impl(db, kv->name, kv->name_len, kv->addr.start, true); } return false; @@ -1578,6 +1798,13 @@ void fdb_kvdb_control(fdb_kvdb_t db, int cmd, void *arg) FDB_ASSERT(db->parent.init_ok == false); db->parent.not_formatable = *(bool *)arg; break; + case FDB_KVDB_CTRL_SET_HASH_OPS: +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + /* this change MUST before database initialization */ + FDB_ASSERT(db->parent.init_ok == false); + db->kv_cache_enm_ops = (fdb_cache_hash_enhancement_ops_t)arg; +#endif + break; } } @@ -1628,6 +1855,19 @@ fdb_err_t fdb_kvdb_init(fdb_kvdb_t db, const char *name, const char *part_name, for (i = 0; i < FDB_KV_CACHE_TABLE_SIZE; i++) { db->kv_cache_table[i].addr = FDB_DATA_UNUSED; } + +#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT + { + size_t size = 0; + db->kv_cache_enm_collisions = false; + db->kv_cache_table_collisions = false; + db->kv_cache_enm_size = 0; + db->kv_cache_enm_total_size = 0; + + size = db->kv_cache_enm_ops->init(FDB_DATA_UNUSED); + db->kv_cache_enm_total_size = size >> 2; + } +#endif #endif /* FDB_KV_USING_CACHE */ FDB_DEBUG("KVDB size is %u bytes.\n", db_max_size(db));