From 0fb447a4496b448cf00248c10c8dd43478bfa32c Mon Sep 17 00:00:00 2001 From: EzraZigenbine Date: Mon, 15 Dec 2025 17:19:14 +1000 Subject: [PATCH 1/7] TSDB feature: Added FDB_TSDB_FIXED_BLOB_SIZE compile-time option to reduce Flash usage in fixed-size blob scenarios. When enabled, removes log_len and log_addr fields from log_idx_data structure, saving 8 bytes per TSL entry. The log address is instead calculated at runtime based on the TSL's position within the sector. This optimization is ideal for applications logging fixed-size data (e.g., single temperature as float + timestamp) where the 8-byte overhead per entry becomes significant with large numbers of entries. Changes: - Conditionally exclude log_len/log_addr from log_idx_data struct - Calculate log address dynamically in read_tsl() when enabled - Add strict size validation in tsl_append() - Maintain backward compatibility via conditional compilation --- inc/fdb_cfg_template.h | 5 +++++ src/fdb_tsdb.c | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/inc/fdb_cfg_template.h b/inc/fdb_cfg_template.h index c2de3c3..b9902da 100644 --- a/inc/fdb_cfg_template.h +++ b/inc/fdb_cfg_template.h @@ -23,6 +23,11 @@ /* using TSDB (Time series database) feature */ #define FDB_USING_TSDB +/* Use fixed-size blobs in TSDB to save flash overhead (8 bytes per entry). + * Define this to the fixed blob size in bytes when all TSL entries are the same size. + * Ideal for logging fixed-size sensor data (e.g., float + timestamp). */ +/* #define FDB_TSDB_FIXED_BLOB_SIZE 4 */ + /* Using FAL storage mode */ #define FDB_USING_FAL_MODE diff --git a/src/fdb_tsdb.c b/src/fdb_tsdb.c index 2d66c92..0ef0822 100644 --- a/src/fdb_tsdb.c +++ b/src/fdb_tsdb.c @@ -94,8 +94,10 @@ typedef struct sector_hdr_data *sector_hdr_data_t; struct log_idx_data { uint8_t status_table[TSL_STATUS_TABLE_SIZE]; /**< node status, @see fdb_tsl_status_t */ fdb_time_t time; /**< node timestamp */ +#ifndef FDB_TSDB_FIXED_BLOB_SIZE uint32_t log_len; /**< node total length (header + name + value), must align by FDB_WRITE_GRAN */ uint32_t log_addr; /**< node address */ +#endif }; typedef struct log_idx_data *log_idx_data_t; @@ -114,6 +116,10 @@ struct check_sec_hdr_cb_args { static fdb_err_t read_tsl(fdb_tsdb_t db, fdb_tsl_t tsl) { struct log_idx_data idx; +#ifdef FDB_TSDB_FIXED_BLOB_SIZE + uint32_t tsl_index_in_sector; + uint32_t sector_addr; +#endif /* read TSL index raw data */ _fdb_flash_read((fdb_db_t)db, tsl->addr.index, (uint32_t *) &idx, sizeof(struct log_idx_data)); tsl->status = (fdb_tsl_status_t) _fdb_get_status(idx.status_table, FDB_TSL_STATUS_NUM); @@ -122,9 +128,17 @@ static fdb_err_t read_tsl(fdb_tsdb_t db, fdb_tsl_t tsl) tsl->addr.log = FDB_DATA_UNUSED; tsl->time = 0; } else { +#ifdef FDB_TSDB_FIXED_BLOB_SIZE + tsl->log_len = FDB_TSDB_FIXED_BLOB_SIZE; + sector_addr = FDB_ALIGN_DOWN(tsl->addr.index, db_sec_size(db)); + tsl_index_in_sector = (tsl->addr.index - sector_addr - SECTOR_HDR_DATA_SIZE) / LOG_IDX_DATA_SIZE; + tsl->addr.log = sector_addr + db_sec_size(db) - (tsl_index_in_sector + 1) * FDB_WG_ALIGN(FDB_TSDB_FIXED_BLOB_SIZE); + tsl->time = idx.time; +#else tsl->log_len = idx.log_len; tsl->addr.log = idx.log_addr; tsl->time = idx.time; +#endif } return FDB_NO_ERR; @@ -306,16 +320,25 @@ static fdb_err_t write_tsl(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t time) fdb_err_t result = FDB_NO_ERR; struct log_idx_data idx; uint32_t idx_addr = db->cur_sec.empty_idx; - +#ifdef FDB_TSDB_FIXED_BLOB_SIZE + uint32_t log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(FDB_TSDB_FIXED_BLOB_SIZE); +#else idx.log_len = blob->size; - idx.time = time; idx.log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(idx.log_len); +#endif + + idx.time = time; + /* write the status will by write granularity */ _FDB_WRITE_STATUS(db, idx_addr, idx.status_table, FDB_TSL_STATUS_NUM, FDB_TSL_PRE_WRITE, false); /* write other index info */ FLASH_WRITE(db, idx_addr + LOG_IDX_TS_OFFSET, &idx.time, sizeof(struct log_idx_data) - LOG_IDX_TS_OFFSET, false); /* write blob data */ +#ifdef FDB_TSDB_FIXED_BLOB_SIZE + FLASH_WRITE(db, log_addr, blob->buf, blob->size, false); +#else FLASH_WRITE(db, idx.log_addr, blob->buf, blob->size, false); +#endif /* write the status will by write granularity */ _FDB_WRITE_STATUS(db, idx_addr, idx.status_table, FDB_TSL_STATUS_NUM, FDB_TSL_WRITE, true); @@ -388,6 +411,12 @@ static fdb_err_t tsl_append(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t *timestam fdb_err_t result = FDB_NO_ERR; fdb_time_t cur_time = timestamp == NULL ? db->get_time() : *timestamp; +#ifdef FDB_TSDB_FIXED_BLOB_SIZE + if(blob->size != FDB_TSDB_FIXED_BLOB_SIZE) { + FDB_INFO("Error: blob size (%zu) must equal FDB_TSDB_FIXED_BLOB_SIZE (%d)\n", blob->size, FDB_TSDB_FIXED_BLOB_SIZE); + return FDB_WRITE_ERR; + } +#else /* check the append length, MUST less than the db->max_len */ if(blob->size > db->max_len) { @@ -395,6 +424,7 @@ static fdb_err_t tsl_append(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t *timestam (intmax_t)blob->size, (intmax_t)(db->max_len)); return FDB_WRITE_ERR; } +#endif /* check the current timestamp, MUST more than the last save timestamp */ if (cur_time <= db->last_time) { From a32bcba16ba9df2c6b78bfa7fd439df1cc003764 Mon Sep 17 00:00:00 2001 From: EzraZigenbine Date: Tue, 16 Dec 2025 20:09:25 +1000 Subject: [PATCH 2/7] Fix inconsistent indentation noted in PR review --- src/fdb_tsdb.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/fdb_tsdb.c b/src/fdb_tsdb.c index 0ef0822..c485906 100644 --- a/src/fdb_tsdb.c +++ b/src/fdb_tsdb.c @@ -117,8 +117,8 @@ static fdb_err_t read_tsl(fdb_tsdb_t db, fdb_tsl_t tsl) { struct log_idx_data idx; #ifdef FDB_TSDB_FIXED_BLOB_SIZE - uint32_t tsl_index_in_sector; - uint32_t sector_addr; + uint32_t tsl_index_in_sector; + uint32_t sector_addr; #endif /* read TSL index raw data */ _fdb_flash_read((fdb_db_t)db, tsl->addr.index, (uint32_t *) &idx, sizeof(struct log_idx_data)); @@ -129,11 +129,11 @@ static fdb_err_t read_tsl(fdb_tsdb_t db, fdb_tsl_t tsl) tsl->time = 0; } else { #ifdef FDB_TSDB_FIXED_BLOB_SIZE - tsl->log_len = FDB_TSDB_FIXED_BLOB_SIZE; - sector_addr = FDB_ALIGN_DOWN(tsl->addr.index, db_sec_size(db)); - tsl_index_in_sector = (tsl->addr.index - sector_addr - SECTOR_HDR_DATA_SIZE) / LOG_IDX_DATA_SIZE; - tsl->addr.log = sector_addr + db_sec_size(db) - (tsl_index_in_sector + 1) * FDB_WG_ALIGN(FDB_TSDB_FIXED_BLOB_SIZE); - tsl->time = idx.time; + tsl->log_len = FDB_TSDB_FIXED_BLOB_SIZE; + sector_addr = FDB_ALIGN_DOWN(tsl->addr.index, db_sec_size(db)); + tsl_index_in_sector = (tsl->addr.index - sector_addr - SECTOR_HDR_DATA_SIZE) / LOG_IDX_DATA_SIZE; + tsl->addr.log = sector_addr + db_sec_size(db) - (tsl_index_in_sector + 1) * FDB_WG_ALIGN(FDB_TSDB_FIXED_BLOB_SIZE); + tsl->time = idx.time; #else tsl->log_len = idx.log_len; tsl->addr.log = idx.log_addr; @@ -321,7 +321,7 @@ static fdb_err_t write_tsl(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t time) struct log_idx_data idx; uint32_t idx_addr = db->cur_sec.empty_idx; #ifdef FDB_TSDB_FIXED_BLOB_SIZE - uint32_t log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(FDB_TSDB_FIXED_BLOB_SIZE); + uint32_t log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(FDB_TSDB_FIXED_BLOB_SIZE); #else idx.log_len = blob->size; idx.log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(idx.log_len); @@ -335,7 +335,7 @@ static fdb_err_t write_tsl(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t time) FLASH_WRITE(db, idx_addr + LOG_IDX_TS_OFFSET, &idx.time, sizeof(struct log_idx_data) - LOG_IDX_TS_OFFSET, false); /* write blob data */ #ifdef FDB_TSDB_FIXED_BLOB_SIZE - FLASH_WRITE(db, log_addr, blob->buf, blob->size, false); + FLASH_WRITE(db, log_addr, blob->buf, blob->size, false); #else FLASH_WRITE(db, idx.log_addr, blob->buf, blob->size, false); #endif @@ -412,10 +412,10 @@ static fdb_err_t tsl_append(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t *timestam fdb_time_t cur_time = timestamp == NULL ? db->get_time() : *timestamp; #ifdef FDB_TSDB_FIXED_BLOB_SIZE - if(blob->size != FDB_TSDB_FIXED_BLOB_SIZE) { - FDB_INFO("Error: blob size (%zu) must equal FDB_TSDB_FIXED_BLOB_SIZE (%d)\n", blob->size, FDB_TSDB_FIXED_BLOB_SIZE); - return FDB_WRITE_ERR; - } + if(blob->size != FDB_TSDB_FIXED_BLOB_SIZE) { + FDB_INFO("Error: blob size (%zu) must equal FDB_TSDB_FIXED_BLOB_SIZE (%d)\n", blob->size, FDB_TSDB_FIXED_BLOB_SIZE); + return FDB_WRITE_ERR; + } #else /* check the append length, MUST less than the db->max_len */ if(blob->size > db->max_len) From b2bd249f1864a4add26d9221d9991db282742a52 Mon Sep 17 00:00:00 2001 From: EzraZigenbine Date: Tue, 16 Dec 2025 20:21:29 +1000 Subject: [PATCH 3/7] Add warning that enabling this macro will cause incompatibility in Flash data format, as requested in PR review --- inc/fdb_cfg_template.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inc/fdb_cfg_template.h b/inc/fdb_cfg_template.h index b9902da..d54d8ca 100644 --- a/inc/fdb_cfg_template.h +++ b/inc/fdb_cfg_template.h @@ -25,7 +25,8 @@ /* Use fixed-size blobs in TSDB to save flash overhead (8 bytes per entry). * Define this to the fixed blob size in bytes when all TSL entries are the same size. - * Ideal for logging fixed-size sensor data (e.g., float + timestamp). */ + * Ideal for logging fixed-size sensor data (e.g., float + timestamp). + * Warning: If defined will be incompataible with variable blob flash store or if fix blob size is later changed */ /* #define FDB_TSDB_FIXED_BLOB_SIZE 4 */ /* Using FAL storage mode */ From 0d3899a94436047ae31a20322aaa077ae7a6cdd2 Mon Sep 17 00:00:00 2001 From: EzraZigenbine Date: Tue, 16 Dec 2025 21:02:07 +1000 Subject: [PATCH 4/7] Improve FLASH_WRITE reusability as requested in PR review --- src/fdb_tsdb.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/fdb_tsdb.c b/src/fdb_tsdb.c index c485906..45ff6e1 100644 --- a/src/fdb_tsdb.c +++ b/src/fdb_tsdb.c @@ -320,12 +320,7 @@ static fdb_err_t write_tsl(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t time) fdb_err_t result = FDB_NO_ERR; struct log_idx_data idx; uint32_t idx_addr = db->cur_sec.empty_idx; -#ifdef FDB_TSDB_FIXED_BLOB_SIZE - uint32_t log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(FDB_TSDB_FIXED_BLOB_SIZE); -#else - idx.log_len = blob->size; - idx.log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(idx.log_len); -#endif + uint32_t log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(blob->size); idx.time = time; @@ -334,11 +329,7 @@ static fdb_err_t write_tsl(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t time) /* write other index info */ FLASH_WRITE(db, idx_addr + LOG_IDX_TS_OFFSET, &idx.time, sizeof(struct log_idx_data) - LOG_IDX_TS_OFFSET, false); /* write blob data */ -#ifdef FDB_TSDB_FIXED_BLOB_SIZE FLASH_WRITE(db, log_addr, blob->buf, blob->size, false); -#else - FLASH_WRITE(db, idx.log_addr, blob->buf, blob->size, false); -#endif /* write the status will by write granularity */ _FDB_WRITE_STATUS(db, idx_addr, idx.status_table, FDB_TSL_STATUS_NUM, FDB_TSL_WRITE, true); From 6dcd4adaa11456be04bee83f7561032f1730c093 Mon Sep 17 00:00:00 2001 From: EzraZigenbine Date: Wed, 17 Dec 2025 19:23:29 +1000 Subject: [PATCH 5/7] tsdb: fix missing log address/size for variable blobs This regression was introduced by previous commit 0d3899a. --- src/fdb_tsdb.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fdb_tsdb.c b/src/fdb_tsdb.c index 45ff6e1..7d83c08 100644 --- a/src/fdb_tsdb.c +++ b/src/fdb_tsdb.c @@ -322,6 +322,11 @@ static fdb_err_t write_tsl(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t time) uint32_t idx_addr = db->cur_sec.empty_idx; uint32_t log_addr = db->cur_sec.empty_data - FDB_WG_ALIGN(blob->size); +#ifndef FDB_TSDB_FIXED_BLOB_SIZE + // variable-size blobs must store address and size in flash + idx.log_addr = log_addr; + idx.log_len = blob->size; +#endif idx.time = time; /* write the status will by write granularity */ From dc3588448d101f2e6d726419baeeb453b115666c Mon Sep 17 00:00:00 2001 From: EzraZigenbine Date: Wed, 17 Dec 2025 19:42:32 +1000 Subject: [PATCH 6/7] Move fixed-size variables to reduce #ifdef usage (pre-PR review) --- src/fdb_tsdb.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/fdb_tsdb.c b/src/fdb_tsdb.c index 7d83c08..97e5dbf 100644 --- a/src/fdb_tsdb.c +++ b/src/fdb_tsdb.c @@ -116,10 +116,7 @@ struct check_sec_hdr_cb_args { static fdb_err_t read_tsl(fdb_tsdb_t db, fdb_tsl_t tsl) { struct log_idx_data idx; -#ifdef FDB_TSDB_FIXED_BLOB_SIZE - uint32_t tsl_index_in_sector; - uint32_t sector_addr; -#endif + /* read TSL index raw data */ _fdb_flash_read((fdb_db_t)db, tsl->addr.index, (uint32_t *) &idx, sizeof(struct log_idx_data)); tsl->status = (fdb_tsl_status_t) _fdb_get_status(idx.status_table, FDB_TSL_STATUS_NUM); @@ -129,6 +126,8 @@ static fdb_err_t read_tsl(fdb_tsdb_t db, fdb_tsl_t tsl) tsl->time = 0; } else { #ifdef FDB_TSDB_FIXED_BLOB_SIZE + uint32_t tsl_index_in_sector; + uint32_t sector_addr; tsl->log_len = FDB_TSDB_FIXED_BLOB_SIZE; sector_addr = FDB_ALIGN_DOWN(tsl->addr.index, db_sec_size(db)); tsl_index_in_sector = (tsl->addr.index - sector_addr - SECTOR_HDR_DATA_SIZE) / LOG_IDX_DATA_SIZE; From cf3275c87a30495cdf9be38710e3c3c5b0362eb0 Mon Sep 17 00:00:00 2001 From: EzraZigenbine Date: Wed, 17 Dec 2025 19:48:18 +1000 Subject: [PATCH 7/7] fix spelling and formatting per PR review --- inc/fdb_cfg_template.h | 2 +- src/fdb_tsdb.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inc/fdb_cfg_template.h b/inc/fdb_cfg_template.h index d54d8ca..f8bde78 100644 --- a/inc/fdb_cfg_template.h +++ b/inc/fdb_cfg_template.h @@ -26,7 +26,7 @@ /* Use fixed-size blobs in TSDB to save flash overhead (8 bytes per entry). * Define this to the fixed blob size in bytes when all TSL entries are the same size. * Ideal for logging fixed-size sensor data (e.g., float + timestamp). - * Warning: If defined will be incompataible with variable blob flash store or if fix blob size is later changed */ + * Warning: If defined will be incompatible with variable blob flash store or if fixed blob size is later changed */ /* #define FDB_TSDB_FIXED_BLOB_SIZE 4 */ /* Using FAL storage mode */ diff --git a/src/fdb_tsdb.c b/src/fdb_tsdb.c index 97e5dbf..bc53375 100644 --- a/src/fdb_tsdb.c +++ b/src/fdb_tsdb.c @@ -407,7 +407,7 @@ static fdb_err_t tsl_append(fdb_tsdb_t db, fdb_blob_t blob, fdb_time_t *timestam fdb_time_t cur_time = timestamp == NULL ? db->get_time() : *timestamp; #ifdef FDB_TSDB_FIXED_BLOB_SIZE - if(blob->size != FDB_TSDB_FIXED_BLOB_SIZE) { + if (blob->size != FDB_TSDB_FIXED_BLOB_SIZE) { FDB_INFO("Error: blob size (%zu) must equal FDB_TSDB_FIXED_BLOB_SIZE (%d)\n", blob->size, FDB_TSDB_FIXED_BLOB_SIZE); return FDB_WRITE_ERR; }