From fd0f23dc023f65b4b4598a0a6d8ca593749b9d85 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:05:47 -0300 Subject: [PATCH 01/26] Improve build times --- CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 90670f83..61e3077c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -418,6 +418,15 @@ set(kth_database_sources_just_kth if (DB_NEW) set(kth_database_sources_just_kth ${kth_database_sources_just_kth} + src/databases/block_database.cpp + src/databases/internal_database.cpp + src/databases/header_database.cpp + src/databases/history_database.cpp + src/databases/reorg_database.cpp + src/databases/spend_database.cpp + src/databases/transaction_database.cpp + src/databases/transaction_unconfirmed_database.cpp + src/databases/utxo_database.cpp src/databases/utxo_entry.cpp ) endif() From bbb8e5cdd613d7ae36e3c72c7c8ad89b2f758faf Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:00 -0300 Subject: [PATCH 02/26] remove implementation files --- .../kth/database/databases/block_database.ipp | 280 ------------------ 1 file changed, 280 deletions(-) delete mode 100644 include/kth/database/databases/block_database.ipp diff --git a/include/kth/database/databases/block_database.ipp b/include/kth/database/databases/block_database.ipp deleted file mode 100644 index 9c79b7e4..00000000 --- a/include/kth/database/databases/block_database.ipp +++ /dev/null @@ -1,280 +0,0 @@ -// Copyright (c) 2016-2020 Knuth Project developers. -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef KTH_DATABASE_BLOCK_DATABASE_IPP_ -#define KTH_DATABASE_BLOCK_DATABASE_IPP_ - -namespace kth::database { - -/* -#if defined(KTH_DB_NEW_FULL) - -template -data_chunk internal_database_basis::serialize_txs(domain::chain::block const& block) { - data_chunk ret; - auto const& txs = block.transactions(); - ret.reserve(txs.size() * kth::hash_size); - - for (auto const& tx : txs) { - auto hash = tx.hash(); - ret.insert(ret.end(), hash.begin(), hash.end()); - } - return ret; -} - -#endif // defined(KTH_DB_NEW_FULL) -*/ - - -//public -template -std::pair internal_database_basis::get_block(hash_digest const& hash) const { - auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return {}; - } - - KTH_DB_val value; - if (kth_db_get(db_txn, dbi_block_header_by_hash_, &key, &value) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - // kth_db_txn_abort(db_txn); - return {}; - } - - // assert kth_db_get_size(value) == 4; - auto height = *static_cast(kth_db_get_data(value)); - - auto block = get_block(height, db_txn); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return {}; - } - - return {block, height}; -} - -//public -template -domain::chain::block internal_database_basis::get_block(uint32_t height) const { - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return domain::chain::block{}; - } - - auto block = get_block(height, db_txn); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return domain::chain::block{}; - } - - return block; -} - -template -domain::chain::block internal_database_basis::get_block(uint32_t height, KTH_DB_txn* db_txn) const { - - auto key = kth_db_make_value(sizeof(height), &height); - -#if defined(KTH_DB_NEW_BLOCKS) - - KTH_DB_val value; - - if (kth_db_get(db_txn, dbi_block_db_, &key, &value) != KTH_DB_SUCCESS) { - return domain::chain::block{}; - } - - auto data = db_value_to_data_chunk(value); - auto res = domain::create(data); - return res; - -#elif defined(KTH_DB_NEW_FULL) - - auto header = get_header(height, db_txn); - if ( ! header.is_valid()) { - return {}; - } - - domain::chain::transaction::list tx_list; - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_block_db_, &cursor) != KTH_DB_SUCCESS) { - return {}; - } - - KTH_DB_val value; - int rc; - if ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_SET)) == 0) { - - auto tx_id = *static_cast(kth_db_get_data(value));; - auto const entry = get_transaction(tx_id, db_txn); - - if ( ! entry.is_valid()) { - return {}; - } - - tx_list.push_back(std::move(entry.transaction())); - - while ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_NEXT_DUP)) == 0) { - auto tx_id = *static_cast(kth_db_get_data(value));; - auto const entry = get_transaction(tx_id, db_txn); - tx_list.push_back(std::move(entry.transaction())); - } - } - - kth_db_cursor_close(cursor); - - /*auto n = kth_db_get_size(value); - auto f = static_cast(kth_db_get_data(value)); - //precondition: mv_size es multiplo de 32 - - domain::chain::transaction::list tx_list; - tx_list.reserve(n / kth::hash_size); - - while (n != 0) { - hash_digest h; - std::copy(f, f + kth::hash_size, h.data()); - - auto tx_entry = get_transaction(h,max_uint32, db_txn); - - if ( ! tx_entry.is_valid() ) { - return domain::chain::block{}; - } - - auto const& tx = tx_entry.transaction(); - - tx_list.push_back(std::move(tx)); - - n -= kth::hash_size; - f += kth::hash_size; - }*/ - - return domain::chain::block{header, std::move(tx_list)}; - -#else - auto block = get_block_reorg(height, db_txn); - return block; -#endif //defined(KTH_DB_NEW_FULL) -} - - -#if defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) - -#if ! defined(KTH_DB_READONLY) - -#if defined(KTH_DB_NEW_BLOCKS) -template -result_code internal_database_basis::insert_block(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { -#elif defined(KTH_DB_NEW_FULL) -template -result_code internal_database_basis::insert_block(domain::chain::block const& block, uint32_t height, uint64_t tx_count, KTH_DB_txn* db_txn) { -#endif - -/*#if defined(KTH_DB_NEW_BLOCKS) - auto data = block.to_data(false); -#elif defined(KTH_DB_NEW_FULL) - auto data = serialize_txs(block); -#endif -*/ - -auto key = kth_db_make_value(sizeof(height), &height); - -#if defined(KTH_DB_NEW_BLOCKS) - - //TODO: store tx hash - auto data = block.to_data(false); - auto value = kth_db_make_value(data.size(), data.data()); - - auto res = kth_db_put(db_txn, dbi_block_db_, &key, &value, KTH_DB_APPEND); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key in Block DB [insert_block] ", res); - return result_code::duplicated_key; - } - - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error saving in Block DB [insert_block] ", res); - return result_code::other; - } - -#elif defined(KTH_DB_NEW_FULL) - - auto const& txs = block.transactions(); - - for (uint64_t i = tx_count; i -result_code internal_database_basis::remove_blocks_db(uint32_t height, KTH_DB_txn* db_txn) { - auto key = kth_db_make_value(sizeof(height), &height); - - #if defined(KTH_DB_NEW_BLOCKS) - - auto res = kth_db_del(db_txn, dbi_block_db_, &key, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting blocks DB in LMDB [remove_blocks_db] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting blocks DB in LMDB [remove_blocks_db] - kth_db_del: ", res); - return result_code::other; - } - -#elif defined(KTH_DB_NEW_FULL) - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_block_db_, &cursor) != KTH_DB_SUCCESS) { - return {}; - } - - KTH_DB_val value; - int rc; - if ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_SET)) == 0) { - - if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { - kth_db_cursor_close(cursor); - return result_code::other; - } - - while ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_NEXT_DUP)) == 0) { - if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { - kth_db_cursor_close(cursor); - return result_code::other; - } - } - } - - kth_db_cursor_close(cursor); - -#endif - - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) -#endif //defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) - - -} // namespace kth::database - -#endif // KTH_DATABASE_BLOCK_DATABASE_IPP_ From 978f52e8fd1677287cc9bc9a62d56cfcd510d352 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:07 -0300 Subject: [PATCH 03/26] remove implementation files --- .../database/databases/header_database.ipp | 99 ------------------- 1 file changed, 99 deletions(-) delete mode 100644 include/kth/database/databases/header_database.ipp diff --git a/include/kth/database/databases/header_database.ipp b/include/kth/database/databases/header_database.ipp deleted file mode 100644 index 9a9a8a16..00000000 --- a/include/kth/database/databases/header_database.ipp +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) 2016-2020 Knuth Project developers. -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef KTH_DATABASE_HEADER_DATABASE_IPP_ -#define KTH_DATABASE_HEADER_DATABASE_IPP_ - -#include - -namespace kth::database { - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::push_block_header(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { - - auto valuearr = block.header().to_data(true); //TODO(fernando): podría estar afuera de la DBTx - auto key = kth_db_make_value(sizeof(height), &height); - auto value = kth_db_make_value(valuearr.size(), valuearr.data()); - - auto res = kth_db_put(db_txn, dbi_block_header_, &key, &value, KTH_DB_APPEND); - if (res == KTH_DB_KEYEXIST) { - //TODO(fernando): El logging en general no está bueno que esté en la DbTx. - LOG_INFO(LOG_DATABASE, "Duplicate key inserting block header [push_block_header] ", res); //TODO(fernando): podría estar afuera de la DBTx. - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting block header [push_block_header] ", res); - return result_code::other; - } - - auto key_by_hash_arr = block.hash(); //TODO(fernando): podría estar afuera de la DBTx - auto key_by_hash = kth_db_make_value(key_by_hash_arr.size(), key_by_hash_arr.data()); - - res = kth_db_put(db_txn, dbi_block_header_by_hash_, &key_by_hash, &key, KTH_DB_NOOVERWRITE); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key inserting block header by hash [push_block_header] ", res); - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting block header by hash [push_block_header] ", res); - return result_code::other; - } - - return result_code::success; -} -#endif // ! defined(KTH_DB_READONLY) - - -template -domain::chain::header internal_database_basis::get_header(uint32_t height, KTH_DB_txn* db_txn) const { - auto key = kth_db_make_value(sizeof(height), &height); - KTH_DB_val value; - - if (kth_db_get(db_txn, dbi_block_header_, &key, &value) != KTH_DB_SUCCESS) { - return domain::chain::header{}; - } - - auto data = db_value_to_data_chunk(value); - auto res = domain::create(data); - return res; -} - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::remove_block_header(hash_digest const& hash, uint32_t height, KTH_DB_txn* db_txn) { - auto key = kth_db_make_value(sizeof(height), &height); - auto res = kth_db_del(db_txn, dbi_block_header_, &key, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting block header in LMDB [remove_block_header] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Erro deleting block header in LMDB [remove_block_header] - kth_db_del: ", res); - return result_code::other; - } - - auto key_hash = kth_db_make_value(hash.size(), const_cast(hash).data()); - - res = kth_db_del(db_txn, dbi_block_header_by_hash_, &key_hash, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting block header by hash in LMDB [remove_block_header] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Erro deleting block header by hash in LMDB [remove_block_header] - kth_db_del: ", res); - return result_code::other; - } - - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - - -} // namespace kth::database - -#endif // KTH_DATABASE_HEADER_DATABASE_IPP_ From 70d2a5cc7923726490fa8a6480a686009cee6fe5 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:11 -0300 Subject: [PATCH 04/26] remove implementation files --- .../database/databases/history_database.ipp | 384 ------------------ 1 file changed, 384 deletions(-) delete mode 100644 include/kth/database/databases/history_database.ipp diff --git a/include/kth/database/databases/history_database.ipp b/include/kth/database/databases/history_database.ipp deleted file mode 100644 index d40f3203..00000000 --- a/include/kth/database/databases/history_database.ipp +++ /dev/null @@ -1,384 +0,0 @@ -// Copyright (c) 2016-2020 Knuth Project developers. -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef KTH_DATABASE_HISTORY_DATABASE_IPP_ -#define KTH_DATABASE_HISTORY_DATABASE_IPP_ - -namespace kth::database { - -#if defined(KTH_DB_NEW_FULL) - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::insert_history_db(domain::wallet::payment_address const& address, data_chunk const& entry, KTH_DB_txn* db_txn) { - - auto key_arr = address.hash(); //TODO(fernando): should I take a reference? - auto key = kth_db_make_value(key_arr.size(), key_arr.data()); - auto value = kth_db_make_value(entry.size(), const_cast(entry).data()); - - auto res = kth_db_put(db_txn, dbi_history_db_, &key, &value, MDB_APPENDDUP); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key inserting history [insert_history_db] ", res); - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting history [insert_history_db] ", res); - return result_code::other; - } - - return result_code::success; -} - -template -result_code internal_database_basis::insert_input_history(domain::chain::input_point const& inpoint, uint32_t height, domain::chain::input const& input, KTH_DB_txn* db_txn) { - - auto const& prevout = input.previous_output(); - - if (prevout.validation.cache.is_valid()) { - - uint64_t history_count = get_history_count(db_txn); - - if (history_count == max_uint64) { - LOG_INFO(LOG_DATABASE, "Error getting history items count"); - return result_code::other; - } - - uint64_t id = history_count; - - // This results in a complete and unambiguous history for the - // address since standard outputs contain unambiguous address data. - for (auto const& address : prevout.validation.cache.addresses()) { - auto valuearr = history_entry::factory_to_data(id, inpoint, domain::chain::point_kind::spend, height, inpoint.index(), prevout.checksum()); - auto res = insert_history_db(address, valuearr, db_txn); - if (res != result_code::success) { - return res; - } - ++id; - } - } else { - //During an IBD with checkpoints some previous output info is missing. - //We can recover it by accessing the database - - //TODO (Mario) check if we can query UTXO - //TODO (Mario) requiere_confirmed = true ?? - - auto entry = get_utxo(prevout, db_txn); - - //auto entry = get_transaction(prevout.hash(), max_uint32, true, db_txn); - - if (entry.is_valid()) { - - //auto const& tx = entry.transaction(); - - //auto const& out_output = tx.outputs()[prevout.index()]; - - uint64_t history_count = get_history_count(db_txn); - if (history_count == max_uint64) { - LOG_INFO(LOG_DATABASE, "Error getting history items count"); - return result_code::other; - } - - uint64_t id = history_count; - - auto const& out_output = entry.output(); - for (auto const& address : out_output.addresses()) { - auto valuearr = history_entry::factory_to_data(id, inpoint, domain::chain::point_kind::spend, height, inpoint.index(), prevout.checksum()); - auto res = insert_history_db(address, valuearr, db_txn); - if (res != result_code::success) { - return res; - } - ++id; - } - } - else { - LOG_INFO(LOG_DATABASE, "Error finding UTXO for input history [insert_input_history]"); - } - } - - return result_code::success; -} - -template -result_code internal_database_basis::insert_output_history(hash_digest const& tx_hash,uint32_t height, uint32_t index, domain::chain::output const& output, KTH_DB_txn* db_txn ) { - - uint64_t history_count = get_history_count(db_txn); - if (history_count == max_uint64) { - LOG_INFO(LOG_DATABASE, "Error getting history items count"); - return result_code::other; - } - - uint64_t id = history_count; - - auto const outpoint = domain::chain::output_point {tx_hash, index}; - auto const value = output.value(); - - // Standard outputs contain unambiguous address data. - for (auto const& address : output.addresses()) { - auto valuearr = history_entry::factory_to_data(id, outpoint, domain::chain::point_kind::output, height, index, value); - auto res = insert_history_db(address, valuearr, db_txn); - if (res != result_code::success) { - return res; - } - ++id; - } - - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - - -template -// static -domain::chain::history_compact internal_database_basis::history_entry_to_history_compact(history_entry const& entry) { - return domain::chain::history_compact{entry.point_kind(), entry.point(), entry.height(), entry.value_or_checksum()}; -} - -template -domain::chain::history_compact::list internal_database_basis::get_history(short_hash const& key, size_t limit, size_t from_height) const { - - domain::chain::history_compact::list result; - - if (limit == 0) { - return result; - } - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return result; - } - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_history_db_, &cursor) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - return result; - } - - auto key_hash = kth_db_make_value(key.size(), const_cast(key).data()); - KTH_DB_val value; - int rc; - if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) { - - auto data = db_value_to_data_chunk(value); - auto entry = domain::create(data); - - if (from_height == 0 || entry.height() >= from_height) { - result.push_back(history_entry_to_history_compact(entry)); - } - - while ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_NEXT_DUP)) == 0) { - - if (limit > 0 && result.size() >= limit) { - break; - } - - auto data = db_value_to_data_chunk(value); - auto entry = domain::create(data); - - if (from_height == 0 || entry.height() >= from_height) { - result.push_back(history_entry_to_history_compact(entry)); - } - - } - } - - kth_db_cursor_close(cursor); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return result; - } - - return result; -} - -template -std::vector internal_database_basis::get_history_txns(short_hash const& key, size_t limit, size_t from_height) const { - - std::set temp; - std::vector result; - - // Stop once we reach the limit (if specified). - if (limit == 0) - return result; - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return result; - } - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_history_db_, &cursor) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - return result; - } - - auto key_hash = kth_db_make_value(key.size(), const_cast(key).data());; - KTH_DB_val value; - int rc; - if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) { - - auto data = db_value_to_data_chunk(value); - auto entry = domain::create(data); - - if (from_height == 0 || entry.height() >= from_height) { - // Avoid inserting the same tx - auto const & pair = temp.insert(entry.point().hash()); - if (pair.second){ - // Add valid txns to the result vector - result.push_back(*pair.first); - } - } - - while ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_NEXT_DUP)) == 0) { - - if (limit > 0 && result.size() >= limit) { - break; - } - - auto data = db_value_to_data_chunk(value); - auto entry = domain::create(data); - - if (from_height == 0 || entry.height() >= from_height) { - // Avoid inserting the same tx - auto const & pair = temp.insert(entry.point().hash()); - if (pair.second){ - // Add valid txns to the result vector - result.push_back(*pair.first); - } - } - - } - } - - kth_db_cursor_close(cursor); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return result; - } - - return result; -} - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::remove_transaction_history_db(domain::chain::transaction const& tx, size_t height, KTH_DB_txn* db_txn) { - - for (auto const& output: tx.outputs()) { - for (auto const& address : output.addresses()) { - auto res = remove_history_db(address, height, db_txn); - if (res != result_code::success) { - return res; - } - } - } - - for (auto const& input: tx.inputs()) { - - auto const& prevout = input.previous_output(); - - if (prevout.validation.cache.is_valid()) { - for (auto const& address : prevout.validation.cache.addresses()) { - auto res = remove_history_db(address, height, db_txn); - if (res != result_code::success) { - return res; - } - } - } - else { - - auto const& entry = get_transaction(prevout.hash(), max_uint32, db_txn); - - if (entry.is_valid()) { - - auto const& tx = entry.transaction(); - auto const& out_output = tx.outputs()[prevout.index()]; - - for (auto const& address : out_output.addresses()) { - - auto res = remove_history_db(address, height, db_txn); - if (res != result_code::success) { - return res; - } - } - } - - /* - for (auto const& address : input.addresses()) { - auto res = remove_history_db(address, height, db_txn); - if (res != result_code::success) { - return res; - } - }*/ - } - } - - return result_code::success; -} - -template -result_code internal_database_basis::remove_history_db(const short_hash& key, size_t height, KTH_DB_txn* db_txn) { - - KTH_DB_cursor* cursor; - - if (kth_db_cursor_open(db_txn, dbi_history_db_, &cursor) != KTH_DB_SUCCESS) { - return result_code::other; - } - - auto key_hash = kth_db_make_value(key.size(), const_cast(key).data());; - KTH_DB_val value; - int rc; - if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) { - - auto data = db_value_to_data_chunk(value); - auto entry = domain::create(data); - - if (entry.height() == height) { - - if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { - kth_db_cursor_close(cursor); - return result_code::other; - } - } - - while ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_NEXT_DUP)) == 0) { - - auto data = db_value_to_data_chunk(value); - auto entry = domain::create(data); - - if (entry.height() == height) { - if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { - kth_db_cursor_close(cursor); - return result_code::other; - } - } - } - } - - kth_db_cursor_close(cursor); - - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - - -template -uint64_t internal_database_basis::get_history_count(KTH_DB_txn* db_txn) const { - MDB_stat db_stats; - auto ret = mdb_stat(db_txn, dbi_history_db_, &db_stats); - if (ret != KTH_DB_SUCCESS) { - return max_uint64; - } - return db_stats.ms_entries; -} - -#endif //KTH_NEW_DB_FULL - -} // namespace kth::database - -#endif // KTH_DATABASE_HISTORY_DATABASE_IPP_ From 9d19fb0d61a05d66481c3cb56b549a05c23ade35 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:17 -0300 Subject: [PATCH 05/26] remove implementation files --- .../kth/database/databases/reorg_database.ipp | 275 ------------------ 1 file changed, 275 deletions(-) delete mode 100644 include/kth/database/databases/reorg_database.ipp diff --git a/include/kth/database/databases/reorg_database.ipp b/include/kth/database/databases/reorg_database.ipp deleted file mode 100644 index 2b4ff9c9..00000000 --- a/include/kth/database/databases/reorg_database.ipp +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) 2016-2020 Knuth Project developers. -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef KTH_DATABASE_REORG_DATABASE_HPP_ -#define KTH_DATABASE_REORG_DATABASE_HPP_ - -namespace kth::database { - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::insert_reorg_pool(uint32_t height, KTH_DB_val& key, KTH_DB_txn* db_txn) { - KTH_DB_val value; - //TODO: use cursors - auto res = kth_db_get(db_txn, dbi_utxo_, &key, &value); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found getting UTXO [insert_reorg_pool] ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error getting UTXO [insert_reorg_pool] ", res); - return result_code::other; - } - - res = kth_db_put(db_txn, dbi_reorg_pool_, &key, &value, KTH_DB_NOOVERWRITE); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key inserting in reorg pool [insert_reorg_pool] ", res); - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting in reorg pool [insert_reorg_pool] ", res); - return result_code::other; - } - - auto key_index = kth_db_make_value(sizeof(height), &height); //TODO(fernando): podría estar afuera de la DBTx - auto value_index = kth_db_make_value(kth_db_get_size(key), kth_db_get_data(key)); //TODO(fernando): podría estar afuera de la DBTx - res = kth_db_put(db_txn, dbi_reorg_index_, &key_index, &value_index, 0); - - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key inserting in reorg index [insert_reorg_pool] ", res); - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting in reorg index [insert_reorg_pool] ", res); - return result_code::other; - } - - return result_code::success; -} - -//TODO : remove this database in db_new_with_blocks and db_new_full -template -result_code internal_database_basis::push_block_reorg(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { - - auto valuearr = block.to_data(false); //TODO(fernando): podría estar afuera de la DBTx - auto key = kth_db_make_value(sizeof(height), &height); //TODO(fernando): podría estar afuera de la DBTx - auto value = kth_db_make_value(valuearr.size(), valuearr.data()); //TODO(fernando): podría estar afuera de la DBTx - - auto res = kth_db_put(db_txn, dbi_reorg_block_, &key, &value, KTH_DB_NOOVERWRITE); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key inserting in reorg block [push_block_reorg] ", res); - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting in reorg block [push_block_reorg] ", res); - return result_code::other; - } - - return result_code::success; -} - -template -result_code internal_database_basis::insert_output_from_reorg_and_remove(domain::chain::output_point const& point, KTH_DB_txn* db_txn) { - auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); - auto key = kth_db_make_value(keyarr.size(), keyarr.data()); - - KTH_DB_val value; - auto res = kth_db_get(db_txn, dbi_reorg_pool_, &key, &value); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found in reorg pool [insert_output_from_reorg_and_remove] ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error in reorg pool [insert_output_from_reorg_and_remove] ", res); - return result_code::other; - } - - res = kth_db_put(db_txn, dbi_utxo_, &key, &value, KTH_DB_NOOVERWRITE); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key inserting in UTXO [insert_output_from_reorg_and_remove] ", res); - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting in UTXO [insert_output_from_reorg_and_remove] ", res); - return result_code::other; - } - - res = kth_db_del(db_txn, dbi_reorg_pool_, &key, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting in reorg pool [insert_output_from_reorg_and_remove] ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting in reorg pool [insert_output_from_reorg_and_remove] ", res); - return result_code::other; - } - return result_code::success; -} - -template -result_code internal_database_basis::remove_block_reorg(uint32_t height, KTH_DB_txn* db_txn) { - - auto key = kth_db_make_value(sizeof(height), &height); - auto res = kth_db_del(db_txn, dbi_reorg_block_, &key, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting reorg block in LMDB [remove_block_reorg] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting reorg block in LMDB [remove_block_reorg] - kth_db_del: ", res); - return result_code::other; - } - return result_code::success; -} - -template -result_code internal_database_basis::remove_reorg_index(uint32_t height, KTH_DB_txn* db_txn) { - - auto key = kth_db_make_value(sizeof(height), &height); - auto res = kth_db_del(db_txn, dbi_reorg_index_, &key, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_DEBUG(LOG_DATABASE, "Key not found deleting reorg index in LMDB [remove_reorg_index] - height: ", height, " - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_DEBUG(LOG_DATABASE, "Error deleting reorg index in LMDB [remove_reorg_index] - height: ", height, " - kth_db_del: ", res); - return result_code::other; - } - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - -template -domain::chain::block internal_database_basis::get_block_reorg(uint32_t height, KTH_DB_txn* db_txn) const { - auto key = kth_db_make_value(sizeof(height), &height); - KTH_DB_val value; - - if (kth_db_get(db_txn, dbi_reorg_block_, &key, &value) != KTH_DB_SUCCESS) { - return {}; - } - - auto data = db_value_to_data_chunk(value); - auto res = domain::create(data); //TODO(fernando): mover fuera de la DbTx - return res; -} - -template -domain::chain::block internal_database_basis::get_block_reorg(uint32_t height) const { - KTH_DB_txn* db_txn; - auto zzz = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (zzz != KTH_DB_SUCCESS) { - return {}; - } - - auto res = get_block_reorg(height, db_txn); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return {}; - } - - return res; -} - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::prune_reorg_index(uint32_t remove_until, KTH_DB_txn* db_txn) { - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_reorg_index_, &cursor) != KTH_DB_SUCCESS) { - return result_code::other; - } - - KTH_DB_val key; - KTH_DB_val value; - int rc; - while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == KTH_DB_SUCCESS) { - auto current_height = *static_cast(kth_db_get_data(key)); - if (current_height < remove_until) { - - auto res = kth_db_del(db_txn, dbi_reorg_pool_, &value, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting reorg pool in LMDB [prune_reorg_index] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting reorg pool in LMDB [prune_reorg_index] - kth_db_del: ", res); - return result_code::other; - } - - if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { - kth_db_cursor_close(cursor); - return result_code::other; - } - } else { - break; - } - } - - kth_db_cursor_close(cursor); - return result_code::success; -} - -template -result_code internal_database_basis::prune_reorg_block(uint32_t amount_to_delete, KTH_DB_txn* db_txn) { - //precondition: amount_to_delete >= 1 - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_reorg_block_, &cursor) != KTH_DB_SUCCESS) { - return result_code::other; - } - - int rc; - while ((rc = kth_db_cursor_get(cursor, nullptr, nullptr, KTH_DB_NEXT)) == KTH_DB_SUCCESS) { - if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { - kth_db_cursor_close(cursor); - return result_code::other; - } - if (--amount_to_delete == 0) break; - } - - kth_db_cursor_close(cursor); - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::get_first_reorg_block_height(uint32_t& out_height) const { - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return result_code::other; - } - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_reorg_block_, &cursor) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - return result_code::other; - } - - KTH_DB_val key; - int rc; - if ((rc = kth_db_cursor_get(cursor, &key, nullptr, KTH_DB_FIRST)) != KTH_DB_SUCCESS) { - return result_code::db_empty; - } - - // assert kth_db_get_size(key) == 4; - out_height = *static_cast(kth_db_get_data(key)); - - kth_db_cursor_close(cursor); - - // kth_db_txn_abort(db_txn); - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return result_code::other; - } - - return result_code::success; -} - - -} // namespace kth::database - -#endif // KTH_DATABASE_REORG_DATABASE_HPP_ From 0d58dcebdda8b30f5b6a32a0469b300c73908c78 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:23 -0300 Subject: [PATCH 06/26] remove implementation files --- .../kth/database/databases/spend_database.ipp | 118 ------------------ 1 file changed, 118 deletions(-) delete mode 100644 include/kth/database/databases/spend_database.ipp diff --git a/include/kth/database/databases/spend_database.ipp b/include/kth/database/databases/spend_database.ipp deleted file mode 100644 index 1f81c4df..00000000 --- a/include/kth/database/databases/spend_database.ipp +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (c) 2016-2020 Knuth Project developers. -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef KTH_DATABASE_SPEND_DATABASE_IPP_ -#define KTH_DATABASE_SPEND_DATABASE_IPP_ - -namespace kth::database { - -#if defined(KTH_DB_NEW_FULL) - -//public -template -domain::chain::input_point internal_database_basis::get_spend(domain::chain::output_point const& point) const { - - auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); - auto key = kth_db_make_value(keyarr.size(), keyarr.data()); - KTH_DB_val value; - - KTH_DB_txn* db_txn; - auto res0 = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res0 != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error begining LMDB Transaction [get_spend] ", res0); - return domain::chain::input_point{}; - } - - res0 = kth_db_get(db_txn, dbi_spend_db_, &key, &value); - if (res0 != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - // kth_db_txn_abort(db_txn); - return domain::chain::input_point{}; - } - - auto data = db_value_to_data_chunk(value); - - res0 = kth_db_txn_commit(db_txn); - if (res0 != KTH_DB_SUCCESS) { - LOG_DEBUG(LOG_DATABASE, "Error commiting LMDB Transaction [get_spend] ", res0); - return domain::chain::input_point{}; - } - - auto res = domain::create(data); - return res; -} - -#if ! defined(KTH_DB_READONLY) - -//pivate -template -result_code internal_database_basis::insert_spend(domain::chain::output_point const& out_point, domain::chain::input_point const& in_point, KTH_DB_txn* db_txn) { - - auto keyarr = out_point.to_data(KTH_INTERNAL_DB_WIRE); - auto key = kth_db_make_value(keyarr.size(), keyarr.data()); - - auto value_arr = in_point.to_data(); - auto value = kth_db_make_value(value_arr.size(), value_arr.data()); - - auto res = kth_db_put(db_txn, dbi_spend_db_, &key, &value, KTH_DB_NOOVERWRITE); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key inserting spend [insert_spend] ", res); - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting spend [insert_spend] ", res); - return result_code::other; - } - - return result_code::success; -} - -template -result_code internal_database_basis::remove_transaction_spend_db(domain::chain::transaction const& tx, KTH_DB_txn* db_txn) { - - for (auto const& input: tx.inputs()) { - - auto const& prevout = input.previous_output(); - - auto res = remove_spend(prevout, db_txn); - if (res != result_code::success) { - return res; - } - - /*res = set_unspend(prevout, db_txn); - if (res != result_code::success) { - return res; - }*/ - } - - return result_code::success; -} - -template -result_code internal_database_basis::remove_spend(domain::chain::output_point const& out_point, KTH_DB_txn* db_txn) { - - auto keyarr = out_point.to_data(KTH_INTERNAL_DB_WIRE); //TODO(fernando): podría estar afuera de la DBTx - auto key = kth_db_make_value(keyarr.size(), keyarr.data()); //TODO(fernando): podría estar afuera de la DBTx - - auto res = kth_db_del(db_txn, dbi_spend_db_, &key, NULL); - - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting spend [remove_spend] ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting spend [remove_spend] ", res); - return result_code::other; - } - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - - -#endif // defined(KTH_DB_NEW_FULL) - -} // namespace kth::database - -#endif // KTH_DATABASE_SPEND_DATABASE_IPP_ From 9fbd71282c10f2ba97dd03b6b7371a2b3cf989f8 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:31 -0300 Subject: [PATCH 07/26] remove implementation files --- .../databases/transaction_database.ipp | 438 ------------------ 1 file changed, 438 deletions(-) delete mode 100644 include/kth/database/databases/transaction_database.ipp diff --git a/include/kth/database/databases/transaction_database.ipp b/include/kth/database/databases/transaction_database.ipp deleted file mode 100644 index 3705ee5b..00000000 --- a/include/kth/database/databases/transaction_database.ipp +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright (c) 2016-2020 Knuth Project developers. -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef KTH_DATABASE_TRANSACTION_DATABASE_HPP_ -#define KTH_DATABASE_TRANSACTION_DATABASE_HPP_ - -namespace kth::database { - -#if defined(KTH_DB_NEW_FULL) - -//public -template -transaction_entry internal_database_basis::get_transaction(hash_digest const& hash, size_t fork_height) const { - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return transaction_entry{}; - } - - auto entry = get_transaction(hash, fork_height, db_txn); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return transaction_entry{}; - } - - return entry; - -} - -#if ! defined(KTH_DB_READONLY) - -template -template -result_code internal_database_basis::insert_transactions(I f, I l, uint32_t height, uint32_t median_time_past, uint64_t tx_count, KTH_DB_txn* db_txn) { - - auto id = tx_count; - uint32_t pos = 0; - - while (f != l) { - auto const& tx = *f; - - //TODO: (Mario) : Implement tx.Confirm to update existing transactions - auto res = insert_transaction(id, tx, height, median_time_past, pos, db_txn); - if (res != result_code::success && res != result_code::duplicated_key) { - return res; - } - - //remove unconfirmed transaction if exists - //TODO (Mario): don't recalculate tx.hash - res = remove_transaction_unconfirmed(tx.hash(), db_txn); - if (res != result_code::success && res != result_code::key_not_found) { - return res; - } - - ++f; - ++pos; - ++id; - } - - return result_code::success; -} -#endif // ! defined(KTH_DB_READONLY) - -template -transaction_entry internal_database_basis::get_transaction(uint64_t id, KTH_DB_txn* db_txn) const { - - auto key = kth_db_make_value(sizeof(id), &id); - KTH_DB_val value; - - auto res = kth_db_get(db_txn, dbi_transaction_db_, &key, &value); - - if (res != KTH_DB_SUCCESS) { - return {}; - } - - auto data = db_value_to_data_chunk(value); - auto entry = domain::create(data); - - return entry; -} - -template -transaction_entry internal_database_basis::get_transaction(hash_digest const& hash, size_t fork_height, KTH_DB_txn* db_txn) const { - auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); - KTH_DB_val value; - - auto res = kth_db_get(db_txn, dbi_transaction_hash_db_, &key, &value); - if (res != KTH_DB_SUCCESS) { - return {}; - } - - auto tx_id = *static_cast(kth_db_get_data(value));; - - auto const entry = get_transaction(tx_id, db_txn); - - if (entry.height() > fork_height) { - return {}; - } - - //Note(Knuth): Transaction stored in dbi_transaction_db_ are always confirmed - //the parameter requiere_confirmed is never used. - /*if ( ! require_confirmed) { - return entry; - } - - auto const confirmed = entry.confirmed(); - - return (confirmed && entry.height() > fork_height) || (require_confirmed && ! confirmed) ? transaction_entry{} : entry; - */ - - return entry; -} - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::insert_transaction(uint64_t id, domain::chain::transaction const& tx, uint32_t height, uint32_t median_time_past, uint32_t position, KTH_DB_txn* db_txn) { - - auto key = kth_db_make_value(sizeof(id), &id); - - //auto key_arr = tx.hash(); //TODO(fernando): podría estar afuera de la DBTx - //auto key = kth_db_make_value(key_arr.size(), key_arr.data()); - - auto valuearr = transaction_entry::factory_to_data(tx, height, median_time_past, position); - auto value = kth_db_make_value(valuearr.size(), valuearr.data()); - - auto res = kth_db_put(db_txn, dbi_transaction_db_, &key, &value, KTH_DB_APPEND); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key in Transaction DB [insert_transaction] ", res); - return result_code::duplicated_key; - } - - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error saving in Transaction DB [insert_transaction] ", res); - return result_code::other; - } - - - auto key_arr = tx.hash(); //TODO(fernando): podría estar afuera de la DBTx - auto key_tx = kth_db_make_value(key_arr.size(), key_arr.data()); - - res = kth_db_put(db_txn, dbi_transaction_hash_db_, &key_tx, &key, KTH_DB_NOOVERWRITE); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key in Transaction DB [insert_transaction] ", res); - return result_code::duplicated_key; - } - - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error saving in Transaction DB [insert_transaction] ", res); - return result_code::other; - } - - return result_code::success; -} - -template -result_code internal_database_basis::remove_transactions(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { - - auto const& txs = block.transactions(); - uint32_t pos = 0; - for (auto const& tx : txs) { - - auto const& hash = tx.hash(); - - auto res0 = remove_transaction_history_db(tx, height, db_txn); - if (res0 != result_code::success) { - return res0; - } - - if (pos > 0) { - auto res0 = remove_transaction_spend_db(tx, db_txn); - if (res0 != result_code::success && res0 != result_code::key_not_found) { - return res0; - } - } - - auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); - KTH_DB_val value; - - auto res = kth_db_get(db_txn, dbi_transaction_hash_db_, &key, &value); - if (res != KTH_DB_SUCCESS) { - return result_code::other; - } - - auto tx_id = *static_cast(kth_db_get_data(value));; - auto key_tx = kth_db_make_value(sizeof(tx_id), &tx_id); - - res = kth_db_del(db_txn, dbi_transaction_db_, &key_tx, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::other; - } - - res = kth_db_del(db_txn, dbi_transaction_hash_db_, &key, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::other; - } - - ++pos; - } - - - /*auto key = kth_db_make_value(sizeof(height), &height); - KTH_DB_val value; - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_block_db_, &cursor) != KTH_DB_SUCCESS) { - return {}; - } - - uint32_t pos = 0; - int rc; - if ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_SET)) == 0) { - - auto tx_id = *static_cast(kth_db_get_data(value));; - auto const entry = get_transaction(tx_id, db_txn); - auto const& tx = entry.transaction(); - - auto res0 = remove_transaction_history_db(tx, height, db_txn); - if (res0 != result_code::success) { - return res0; - } - - if (pos > 0) { - res0 = remove_transaction_spend_db(tx, db_txn); - if (res0 != result_code::success && res0 != result_code::key_not_found) { - return res0; - } - } - - auto key_tx = kth_db_make_value(sizeof(tx_id), &tx_id); - auto res = kth_db_del(db_txn, dbi_transaction_db_, &key_tx, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::other; - } - - auto key_hash = kth_db_make_value(tx.hash().size(), tx.hash().data()); - res = kth_db_del(db_txn, dbi_transaction_hash_db_, &key_hash, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::other; - } - - while ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_NEXT_DUP)) == 0) { - ++pos; - auto tx_id = *static_cast(kth_db_get_data(value));; - auto const entry = get_transaction(tx_id, db_txn); - auto const& tx = entry.transaction(); - - auto res0 = remove_transaction_history_db(tx, height, db_txn); - if (res0 != result_code::success) { - return res0; - } - - if (pos > 0) { - res0 = remove_transaction_spend_db(tx, db_txn); - if (res0 != result_code::success && res0 != result_code::key_not_found) { - return res0; - } - } - - auto key_tx = kth_db_make_value(sizeof(tx_id), &tx_id); - auto res = kth_db_del(db_txn, dbi_transaction_db_, &key_tx, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::other; - } - - auto key_hash = kth_db_make_value(tx.hash().size(), tx.hash().data()); - res = kth_db_del(db_txn, dbi_transaction_hash_db_, &key_hash, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::other; - } - } - } - - kth_db_cursor_close(cursor);*/ - - - /*//To get the tx hashes to remove, we need to read the block db - if (kth_db_get(db_txn, dbi_block_db_, &key, &value) != KTH_DB_SUCCESS) { - return result_code::other; - } - - auto n = kth_db_get_size(value); - auto f = static_cast(kth_db_get_data(value)); - //precondition: mv_size es multiplo de 32 - - uint32_t pos = 0; - while (n != 0) { - hash_digest h; - std::copy(f, f + kth::hash_size, h.data()); - - auto key_tx = kth_db_make_value(h.size(), h.data()); - - auto const& entry = get_transaction(h, height, db_txn); - if ( ! entry.is_valid() ) { - return result_code::other; - } - - auto const& tx = entry.transaction(); - - auto res0 = remove_transaction_history_db(tx, height, db_txn); - if (res0 != result_code::success) { - return res0; - } - - if (pos > 0) { - res0 = remove_transaction_spend_db(tx, db_txn); - if (res0 != result_code::success && res0 != result_code::key_not_found) { - return res0; - } - } - - auto res = kth_db_del(db_txn, dbi_transaction_db_, &key_tx, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); - return result_code::other; - } - - n -= kth::hash_size; - f += kth::hash_size; - ++pos; - }*/ - - return result_code::success; -} - -template -result_code internal_database_basis::update_transaction(domain::chain::transaction const& tx, uint32_t height, uint32_t median_time_past, uint32_t position, KTH_DB_txn* db_txn) { - auto key_arr = tx.hash(); //TODO(fernando): podría estar afuera de la DBTx - auto key = kth_db_make_value(key_arr.size(), key_arr.data()); - - auto valuearr = transaction_entry::factory_to_data(tx, height, median_time_past, position); - auto value = kth_db_make_value(valuearr.size(), valuearr.data()); - - auto res = kth_db_put(db_txn, dbi_transaction_db_, &key, &value, 0); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key in Transaction DB [insert_transaction] ", res); - return result_code::duplicated_key; - } - - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error saving in Transaction DB [insert_transaction] ", res); - return result_code::other; - } - - return result_code::success; -} - -template -result_code internal_database_basis::set_spend(domain::chain::output_point const& point, uint32_t spender_height, KTH_DB_txn* db_txn) { - - // Limit search to confirmed transactions at or below the spender height, - // since a spender cannot spend above its own height. - // Transactions are not marked as spent unless the spender is confirmed. - // This is consistent with support for unconfirmed double spends. - - auto entry = get_transaction(point.hash(), spender_height, db_txn); - - // The transaction is not exist as confirmed at or below the height. - if ( ! entry.is_valid() ) { - return result_code::other; - } - - auto const& tx = entry.transaction(); - - if (point.index() >= tx.outputs().size()) { - return result_code::other; - } - - //update spender_height - auto& output = tx.outputs()[point.index()]; - output.validation.spender_height = spender_height; - - //overwrite transaction - auto ret = update_transaction(tx, entry.height(), entry.median_time_past(), entry.position(), db_txn); - if (ret != result_code::success & ret != result_code::duplicated_key) { - return ret; - } - - return result_code::success; -} - -template -result_code internal_database_basis::set_unspend(domain::chain::output_point const& point, KTH_DB_txn* db_txn) { - return set_spend(point, domain::chain::output::validation::not_spent, db_txn); -} -#endif // ! defined(KTH_DB_READONLY) - -template -uint64_t internal_database_basis::get_tx_count(KTH_DB_txn* db_txn) const { - MDB_stat db_stats; - auto ret = mdb_stat(db_txn, dbi_transaction_db_, &db_stats); - if (ret != KTH_DB_SUCCESS) { - return max_uint64; - } - return db_stats.ms_entries; -} - -#endif //KTH_NEW_DB_FULL - -} // namespace kth::database - -#endif // KTH_DATABASE_TRANSACTION_DATABASE_HPP_ From 6e5029f41685f2b0743bdb4208631ef4d5062327 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:35 -0300 Subject: [PATCH 08/26] remove implementation files --- .../transaction_unconfirmed_database.ipp | 154 ------------------ 1 file changed, 154 deletions(-) delete mode 100644 include/kth/database/databases/transaction_unconfirmed_database.ipp diff --git a/include/kth/database/databases/transaction_unconfirmed_database.ipp b/include/kth/database/databases/transaction_unconfirmed_database.ipp deleted file mode 100644 index 53801a1d..00000000 --- a/include/kth/database/databases/transaction_unconfirmed_database.ipp +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright (c) 2016-2020 Knuth Project developers. -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef KTH_DATABASE_TRANSACTION_UNCONFIRMED_DATABASE_HPP_ -#define KTH_DATABASE_TRANSACTION_UNCONFIRMED_DATABASE_HPP_ - -namespace kth::database { - -#if defined(KTH_DB_NEW_FULL) - - -template -transaction_unconfirmed_entry internal_database_basis::get_transaction_unconfirmed(hash_digest const& hash) const { - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return {}; - } - - auto const& ret = get_transaction_unconfirmed(hash, db_txn); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return {}; - } - - return ret; -} - -template -transaction_unconfirmed_entry internal_database_basis::get_transaction_unconfirmed(hash_digest const& hash, KTH_DB_txn* db_txn) const { - auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); - KTH_DB_val value; - - if (kth_db_get(db_txn, dbi_transaction_unconfirmed_db_, &key, &value) != KTH_DB_SUCCESS) { - return {}; - } - - auto data = db_value_to_data_chunk(value); - auto res = domain::create(data); - - return res; -} - -template -std::vector internal_database_basis::get_all_transaction_unconfirmed() const { - - std::vector result; - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return result; - } - - KTH_DB_cursor* cursor; - - if (kth_db_cursor_open(db_txn, dbi_transaction_unconfirmed_db_, &cursor) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - return result; - } - - KTH_DB_val key; - KTH_DB_val value; - - - int rc; - if ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == 0) { - - auto data = db_value_to_data_chunk(value); - auto res = domain::create(data); - result.push_back(res); - - while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == 0) { - auto data = db_value_to_data_chunk(value); - auto res = domain::create(data); - result.push_back(res); - } - } - - kth_db_cursor_close(cursor); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return result; - } - - return result; -} - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::remove_transaction_unconfirmed(hash_digest const& tx_id, KTH_DB_txn* db_txn) { - - auto key = kth_db_make_value(tx_id.size(), const_cast(tx_id).data()); - - auto res = kth_db_del(db_txn, dbi_transaction_unconfirmed_db_, &key, NULL); - if (res == KTH_DB_NOTFOUND) { - //LOG_DEBUG(LOG_DATABASE, "Key not found deleting transaction unconfirmed DB in LMDB [remove_transaction_unconfirmed] - kth_db_del: ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting transaction unconfirmed DB in LMDB [remove_transaction_unconfirmed] - kth_db_del: ", res); - return result_code::other; - } - - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - -template -inline -uint32_t internal_database_basis::get_clock_now() const { - auto const now = std::chrono::high_resolution_clock::now(); - return static_cast(std::chrono::duration_cast(now.time_since_epoch()).count()); -} - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::insert_transaction_unconfirmed(domain::chain::transaction const& tx, uint32_t height, KTH_DB_txn* db_txn) { - - uint32_t arrival_time = get_clock_now(); - - auto key_arr = tx.hash(); //TODO(fernando): podría estar afuera de la DBTx - auto key = kth_db_make_value(key_arr.size(), key_arr.data()); - - auto value_arr = transaction_unconfirmed_entry::factory_to_data(tx, arrival_time, height); - auto value = kth_db_make_value(value_arr.size(), value_arr.data()); - - auto res = kth_db_put(db_txn, dbi_transaction_unconfirmed_db_, &key, &value, KTH_DB_NOOVERWRITE); - if (res == KTH_DB_KEYEXIST) { - LOG_INFO(LOG_DATABASE, "Duplicate key in Transaction Unconfirmed DB [insert_transaction_unconfirmed] ", res); - return result_code::duplicated_key; - } - - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error saving in Transaction Unconfirmed DB [insert_transaction_unconfirmed] ", res); - return result_code::other; - } - - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - -#endif //KTH_NEW_DB_FULL - - -} // namespace kth::database - -#endif // KTH_DATABASE_TRANSACTION_UNCONFIRMED_DATABASE_HPP_ From 29939222154c4c215189d521b28fe24604466889 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:39 -0300 Subject: [PATCH 09/26] remove implementation files --- .../kth/database/databases/utxo_database.ipp | 58 ------------------- 1 file changed, 58 deletions(-) delete mode 100644 include/kth/database/databases/utxo_database.ipp diff --git a/include/kth/database/databases/utxo_database.ipp b/include/kth/database/databases/utxo_database.ipp deleted file mode 100644 index 0baea278..00000000 --- a/include/kth/database/databases/utxo_database.ipp +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2016-2020 Knuth Project developers. -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef KTH_DATABASE_UTXO_DATABASE_HPP_ -#define KTH_DATABASE_UTXO_DATABASE_HPP_ - -namespace kth::database { - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::remove_utxo(uint32_t height, domain::chain::output_point const& point, bool insert_reorg, KTH_DB_txn* db_txn) { - auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); //TODO(fernando): podría estar afuera de la DBTx - auto key = kth_db_make_value(keyarr.size(), keyarr.data()); //TODO(fernando): podría estar afuera de la DBTx - - if (insert_reorg) { - auto res0 = insert_reorg_pool(height, key, db_txn); - if (res0 != result_code::success) return res0; - } - - auto res = kth_db_del(db_txn, dbi_utxo_, &key, NULL); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found deleting UTXO [remove_utxo] ", res); - return result_code::key_not_found; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error deleting UTXO [remove_utxo] ", res); - return result_code::other; - } - return result_code::success; -} - -template -result_code internal_database_basis::insert_utxo(domain::chain::output_point const& point, domain::chain::output const& output, data_chunk const& fixed_data, KTH_DB_txn* db_txn) { - auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); //TODO(fernando): podría estar afuera de la DBTx - auto valuearr = utxo_entry::to_data_with_fixed(output, fixed_data); //TODO(fernando): podría estar afuera de la DBTx - - auto key = kth_db_make_value(keyarr.size(), keyarr.data()); //TODO(fernando): podría estar afuera de la DBTx - auto value = kth_db_make_value(valuearr.size(), valuearr.data()); //TODO(fernando): podría estar afuera de la DBTx - auto res = kth_db_put(db_txn, dbi_utxo_, &key, &value, KTH_DB_NOOVERWRITE); - - if (res == KTH_DB_KEYEXIST) { - LOG_DEBUG(LOG_DATABASE, "Duplicate Key inserting UTXO [insert_utxo] ", res); - return result_code::duplicated_key; - } - if (res != KTH_DB_SUCCESS) { - LOG_INFO(LOG_DATABASE, "Error inserting UTXO [insert_utxo] ", res); - return result_code::other; - } - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - -} // namespace kth::database - -#endif // KTH_DATABASE_UTXO_DATABASE_HPP_ From 09fc9f02eb010e26b1836e53c94db10d18047e40 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:06:55 -0300 Subject: [PATCH 10/26] Improve compilation times --- include/kth/database.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/kth/database.hpp b/include/kth/database.hpp index 317d0c63..d884734d 100644 --- a/include/kth/database.hpp +++ b/include/kth/database.hpp @@ -53,6 +53,7 @@ #include #endif // KTH_DB_TRANSACTION_UNCONFIRMED +#ifdef KTH_DB_LEGACY #include #include #include @@ -67,7 +68,6 @@ #include #include -#ifdef KTH_DB_LEGACY #include #include #endif // KTH_DB_LEGACY @@ -76,5 +76,4 @@ #include #endif // KTH_DB_TRANSACTION_UNCONFIRMED - #endif From 6f20b0d96571c9678a25135f74eacae010e6e7d3 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:07:14 -0300 Subject: [PATCH 11/26] Modularize DB parameters --- .../kth/database/databases/db_parameters.hpp | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 include/kth/database/databases/db_parameters.hpp diff --git a/include/kth/database/databases/db_parameters.hpp b/include/kth/database/databases/db_parameters.hpp new file mode 100644 index 00000000..ab20f8d1 --- /dev/null +++ b/include/kth/database/databases/db_parameters.hpp @@ -0,0 +1,42 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef KTH_DATABASE_DB_PARAMETERS_HPP_ +#define KTH_DATABASE_DB_PARAMETERS_HPP_ + +#ifdef KTH_INTERNAL_DB_4BYTES_INDEX +#define KTH_INTERNAL_DB_WIRE true +#else +#define KTH_INTERNAL_DB_WIRE false +#endif + +#if defined(KTH_DB_READONLY) +#define KTH_DB_CONDITIONAL_CREATE 0 +#else +#define KTH_DB_CONDITIONAL_CREATE KTH_DB_CREATE +#endif + +#if defined(KTH_DB_READONLY) +#define KTH_DB_CONDITIONAL_READONLY KTH_DB_RDONLY +#else +#define KTH_DB_CONDITIONAL_READONLY 0 +#endif + +namespace kth::database { + +#if defined(KTH_DB_NEW_FULL) +constexpr size_t max_dbs_ = 13; +#elif defined(KTH_DB_NEW_BLOCKS) +constexpr size_t max_dbs_ = 8; +#else +constexpr size_t max_dbs_ = 7; +#endif + +// constexpr size_t env_open_mode_ = 0664; +constexpr size_t env_open_mode_ = 0644; +constexpr int directory_exists = 0; + +} // namespace kth::database + +#endif // KTH_DATABASE_DB_PARAMETERS_HPP_ From 4b8be65d79136850c5ae82d1804843feb475b252 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:07:34 -0300 Subject: [PATCH 12/26] remove internal_database_basis template parameter --- .../database/databases/internal_database.hpp | 134 ++++-------------- 1 file changed, 28 insertions(+), 106 deletions(-) diff --git a/include/kth/database/databases/internal_database.hpp b/include/kth/database/databases/internal_database.hpp index 44f1381d..5a87b706 100644 --- a/include/kth/database/databases/internal_database.hpp +++ b/include/kth/database/databases/internal_database.hpp @@ -16,55 +16,25 @@ // #include // #endif -#include - #include #include -#include - -#include +#include +#include +#include #include +#include #include -#include -#include #include #include +#include +#include #include -#ifdef KTH_INTERNAL_DB_4BYTES_INDEX -#define KTH_INTERNAL_DB_WIRE true -#else -#define KTH_INTERNAL_DB_WIRE false -#endif - -#if defined(KTH_DB_READONLY) -#define KTH_DB_CONDITIONAL_CREATE 0 -#else -#define KTH_DB_CONDITIONAL_CREATE KTH_DB_CREATE -#endif - -#if defined(KTH_DB_READONLY) -#define KTH_DB_CONDITIONAL_READONLY KTH_DB_RDONLY -#else -#define KTH_DB_CONDITIONAL_READONLY 0 -#endif - namespace kth::database { -#if defined(KTH_DB_NEW_FULL) -constexpr size_t max_dbs_ = 13; -#elif defined(KTH_DB_NEW_BLOCKS) -constexpr size_t max_dbs_ = 8; -#else -constexpr size_t max_dbs_ = 7; -#endif - -constexpr size_t env_open_mode_ = 0664; -constexpr int directory_exists = 0; - -template +// template class KD_API internal_database_basis { public: using path = kth::path; @@ -107,10 +77,14 @@ class KD_API internal_database_basis { bool close(); #if ! defined(KTH_DB_READONLY) + + void print_stats(KTH_DB_txn* db_txn); + result_code push_genesis(domain::chain::block const& block); //TODO(fernando): optimization: consider passing a list of outputs to insert and a list of inputs to delete instead of an entire Block. // avoiding inserting and erasing internal spenders + template result_code push_block(domain::chain::block const& block, uint32_t height, uint32_t median_time_past); #endif @@ -165,6 +139,7 @@ class KD_API internal_database_basis { bool open_internal(); + template bool is_old_block(domain::chain::block const& block) const; size_t get_db_page_size() const; @@ -178,12 +153,6 @@ class KD_API internal_database_basis { utxo_entry get_utxo(domain::chain::output_point const& point, KTH_DB_txn* db_txn) const; #if ! defined(KTH_DB_READONLY) - result_code insert_reorg_pool(uint32_t height, KTH_DB_val& key, KTH_DB_txn* db_txn); - - result_code remove_utxo(uint32_t height, domain::chain::output_point const& point, bool insert_reorg, KTH_DB_txn* db_txn); - - result_code insert_utxo(domain::chain::output_point const& point, domain::chain::output const& output, data_chunk const& fixed_data, KTH_DB_txn* db_txn); - result_code remove_inputs(hash_digest const& tx_id, uint32_t height, domain::chain::input::list const& inputs, bool insert_reorg, KTH_DB_txn* db_txn); result_code insert_outputs(hash_digest const& tx_id, uint32_t height, domain::chain::output::list const& outputs, data_chunk const& fixed_data, KTH_DB_txn* db_txn); @@ -201,16 +170,12 @@ class KD_API internal_database_basis { result_code push_block_header(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn); - result_code push_block_reorg(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn); - result_code push_block(domain::chain::block const& block, uint32_t height, uint32_t median_time_past, bool insert_reorg, KTH_DB_txn* db_txn); result_code push_genesis(domain::chain::block const& block, KTH_DB_txn* db_txn); result_code remove_outputs(hash_digest const& txid, domain::chain::output::list const& outputs, KTH_DB_txn* db_txn); - result_code insert_output_from_reorg_and_remove(domain::chain::output_point const& point, KTH_DB_txn* db_txn); - result_code insert_inputs(domain::chain::input::list const& inputs, KTH_DB_txn* db_txn); template @@ -224,26 +189,15 @@ class KD_API internal_database_basis { result_code remove_block_header(hash_digest const& hash, uint32_t height, KTH_DB_txn* db_txn); - result_code remove_block_reorg(uint32_t height, KTH_DB_txn* db_txn); - - result_code remove_reorg_index(uint32_t height, KTH_DB_txn* db_txn); - result_code remove_block(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn); #endif domain::chain::header get_header(uint32_t height, KTH_DB_txn* db_txn) const; - domain::chain::block get_block_reorg(uint32_t height, KTH_DB_txn* db_txn) const; - domain::chain::block get_block_reorg(uint32_t height) const; - #if ! defined(KTH_DB_READONLY) result_code remove_block(domain::chain::block const& block, uint32_t height); - result_code prune_reorg_index(uint32_t remove_until, KTH_DB_txn* db_txn); - result_code prune_reorg_block(uint32_t amount_to_delete, KTH_DB_txn* db_txn); #endif - result_code get_first_reorg_block_height(uint32_t& out_height) const; - //TODO(fernando): is taking KTH_DB_val by value, is that Ok? result_code insert_reorg_into_pool(utxo_pool_t& pool, KTH_DB_val key_point, KTH_DB_txn* db_txn) const; @@ -359,65 +313,33 @@ class KD_API internal_database_basis { #endif }; -template -constexpr char internal_database_basis::block_header_db_name[]; //key: block height, value: block header - -template -constexpr char internal_database_basis::block_header_by_hash_db_name[]; //key: block hash, value: block height - -template -constexpr char internal_database_basis::utxo_db_name[]; //key: point, value: output - -template -constexpr char internal_database_basis::reorg_pool_name[]; //key: key: point, value: output - -template -constexpr char internal_database_basis::reorg_index_name[]; //key: block height, value: point list - -template -constexpr char internal_database_basis::reorg_block_name[]; //key: block height, value: block - -template -constexpr char internal_database_basis::db_properties_name[]; //key: propery, value: data +constexpr char internal_database_basis::block_header_db_name[]; //key: block height, value: block header +constexpr char internal_database_basis::block_header_by_hash_db_name[]; //key: block hash, value: block height +constexpr char internal_database_basis::utxo_db_name[]; //key: point, value: output +constexpr char internal_database_basis::reorg_pool_name[]; //key: key: point, value: output +constexpr char internal_database_basis::reorg_index_name[]; //key: block height, value: point list +constexpr char internal_database_basis::reorg_block_name[]; //key: block height, value: block +constexpr char internal_database_basis::db_properties_name[]; //key: propery, value: data #if defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) -template -constexpr char internal_database_basis::block_db_name[]; //key: block height, value: block - //key: block height, value: tx hashes +constexpr char internal_database_basis::block_db_name[]; //key: block height, value: block + //key: block height, value: tx hashes #endif #ifdef KTH_DB_NEW_FULL -template -constexpr char internal_database_basis::transaction_db_name[]; //key: tx hash, value: tx - -template -constexpr char internal_database_basis::transaction_hash_db_name[]; //key: tx hash, value: tx - -template -constexpr char internal_database_basis::history_db_name[]; //key: tx hash, value: tx - -template -constexpr char internal_database_basis::spend_db_name[]; //key: output_point, value: input_point - -template -constexpr char internal_database_basis::transaction_unconfirmed_db_name[]; //key: tx hash, value: tx - +constexpr char internal_database_basis::transaction_db_name[]; //key: tx hash, value: tx +constexpr char internal_database_basis::transaction_hash_db_name[]; //key: tx hash, value: tx +constexpr char internal_database_basis::history_db_name[]; //key: tx hash, value: tx +constexpr char internal_database_basis::spend_db_name[]; //key: output_point, value: input_point +constexpr char internal_database_basis::transaction_unconfirmed_db_name[];//key: tx hash, value: tx #endif -using internal_database = internal_database_basis; +// using internal_database = internal_database_basis; +using internal_database = internal_database_basis; } // namespace kth::database - -#include -#include -#include -#include -#include #include -#include -#include -#include #endif // KTH_DATABASE_INTERNAL_DATABASE_HPP_ From 60a303eccad62d2e539510e4a87f0ae1274f9fcd Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:08:00 -0300 Subject: [PATCH 13/26] move the code from implementation file to translation unit --- src/databases/utxo_database.cpp | 58 +++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/databases/utxo_database.cpp diff --git a/src/databases/utxo_database.cpp b/src/databases/utxo_database.cpp new file mode 100644 index 00000000..907d2f6e --- /dev/null +++ b/src/databases/utxo_database.cpp @@ -0,0 +1,58 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include + +namespace kth::database { + +#if ! defined(KTH_DB_READONLY) + +result_code remove_utxo(uint32_t height, domain::chain::output_point const& point, bool insert_reorg, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_utxo, KTH_DB_dbi dbi_reorg_pool, KTH_DB_dbi dbi_reorg_index) { + auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); //TODO(fernando): podría estar afuera de la DBTx + auto key = kth_db_make_value(keyarr.size(), keyarr.data()); //TODO(fernando): podría estar afuera de la DBTx + + if (insert_reorg) { + std::cout << "insert_reorg: " << insert_reorg << std::endl; + auto res0 = insert_reorg_pool(height, key, db_txn, dbi_utxo, dbi_reorg_pool, dbi_reorg_index); + if (res0 != result_code::success) return res0; + } + + auto res = kth_db_del(db_txn, dbi_utxo, &key, NULL); + // if (res == KTH_DB_NOTFOUND) { + // LOG_INFO(LOG_DATABASE, "Key not found deleting UTXO [remove_utxo] ", res); + // return result_code::key_not_found; + // } + // if (res != KTH_DB_SUCCESS) { + // LOG_INFO(LOG_DATABASE, "Error deleting UTXO [remove_utxo] ", res); + // return result_code::other; + // } + return result_code::success; +} + +result_code insert_utxo(domain::chain::output_point const& point, domain::chain::output const& output, data_chunk const& fixed_data, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_utxo) { + // auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); //TODO(fernando): podría estar afuera de la DBTx + // auto valuearr = utxo_entry::to_data_with_fixed(output, fixed_data); //TODO(fernando): podría estar afuera de la DBTx + + // auto key = kth_db_make_value(keyarr.size(), keyarr.data()); //TODO(fernando): podría estar afuera de la DBTx + // auto value = kth_db_make_value(valuearr.size(), valuearr.data()); //TODO(fernando): podría estar afuera de la DBTx + + // auto res = kth_db_put(db_txn, dbi_utxo, &key, &value, KTH_DB_NOOVERWRITE); + + // if (res == KTH_DB_KEYEXIST) { + // LOG_DEBUG(LOG_DATABASE, "Duplicate Key inserting UTXO [insert_utxo] ", res); + // return result_code::duplicated_key; + // } + // if (res != KTH_DB_SUCCESS) { + // LOG_INFO(LOG_DATABASE, "Error inserting UTXO [insert_utxo] ", res); + // return result_code::other; + // } + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +} // namespace kth::database From 2f3748cace34d969722c61664ae14d08b5bf91f6 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:08:05 -0300 Subject: [PATCH 14/26] move the code from implementation file to translation unit --- .../transaction_unconfirmed_database.cpp | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 src/databases/transaction_unconfirmed_database.cpp diff --git a/src/databases/transaction_unconfirmed_database.cpp b/src/databases/transaction_unconfirmed_database.cpp new file mode 100644 index 00000000..7eab7f89 --- /dev/null +++ b/src/databases/transaction_unconfirmed_database.cpp @@ -0,0 +1,144 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace kth::database { + +#if defined(KTH_DB_NEW_FULL) + +transaction_unconfirmed_entry internal_database_basis::get_transaction_unconfirmed(hash_digest const& hash) const { + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return {}; + } + + auto const& ret = get_transaction_unconfirmed(hash, db_txn); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return {}; + } + + return ret; +} + +transaction_unconfirmed_entry internal_database_basis::get_transaction_unconfirmed(hash_digest const& hash, KTH_DB_txn* db_txn) const { + auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); + KTH_DB_val value; + + if (kth_db_get(db_txn, dbi_transaction_unconfirmed_db_, &key, &value) != KTH_DB_SUCCESS) { + return {}; + } + + auto data = db_value_to_data_chunk(value); + auto res = domain::create(data); + + return res; +} + +std::vector internal_database_basis::get_all_transaction_unconfirmed() const { + + std::vector result; + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return result; + } + + KTH_DB_cursor* cursor; + + if (kth_db_cursor_open(db_txn, dbi_transaction_unconfirmed_db_, &cursor) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + return result; + } + + KTH_DB_val key; + KTH_DB_val value; + + + int rc; + if ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == 0) { + + auto data = db_value_to_data_chunk(value); + auto res = domain::create(data); + result.push_back(res); + + while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == 0) { + auto data = db_value_to_data_chunk(value); + auto res = domain::create(data); + result.push_back(res); + } + } + + kth_db_cursor_close(cursor); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return result; + } + + return result; +} + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::remove_transaction_unconfirmed(hash_digest const& tx_id, KTH_DB_txn* db_txn) { + + auto key = kth_db_make_value(tx_id.size(), const_cast(tx_id).data()); + + auto res = kth_db_del(db_txn, dbi_transaction_unconfirmed_db_, &key, NULL); + if (res == KTH_DB_NOTFOUND) { + //LOG_DEBUG(LOG_DATABASE, "Key not found deleting transaction unconfirmed DB in LMDB [remove_transaction_unconfirmed] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting transaction unconfirmed DB in LMDB [remove_transaction_unconfirmed] - kth_db_del: ", res); + return result_code::other; + } + + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +inline +uint32_t internal_database_basis::get_clock_now() const { + auto const now = std::chrono::high_resolution_clock::now(); + return static_cast(std::chrono::duration_cast(now.time_since_epoch()).count()); +} + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::insert_transaction_unconfirmed(domain::chain::transaction const& tx, uint32_t height, KTH_DB_txn* db_txn) { + + uint32_t arrival_time = get_clock_now(); + + auto key_arr = tx.hash(); //TODO(fernando): podría estar afuera de la DBTx + auto key = kth_db_make_value(key_arr.size(), key_arr.data()); + + auto value_arr = transaction_unconfirmed_entry::factory_to_data(tx, arrival_time, height); + auto value = kth_db_make_value(value_arr.size(), value_arr.data()); + + auto res = kth_db_put(db_txn, dbi_transaction_unconfirmed_db_, &key, &value, KTH_DB_NOOVERWRITE); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key in Transaction Unconfirmed DB [insert_transaction_unconfirmed] ", res); + return result_code::duplicated_key; + } + + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error saving in Transaction Unconfirmed DB [insert_transaction_unconfirmed] ", res); + return result_code::other; + } + + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +#endif //KTH_NEW_DB_FULL + + +} // namespace kth::database From 1f5d081bc8da33103a9af9d4fb0312fafa22fc0a Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:08:11 -0300 Subject: [PATCH 15/26] move the code from implementation file to translation unit --- src/databases/transaction_database.cpp | 426 +++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 src/databases/transaction_database.cpp diff --git a/src/databases/transaction_database.cpp b/src/databases/transaction_database.cpp new file mode 100644 index 00000000..4db6e492 --- /dev/null +++ b/src/databases/transaction_database.cpp @@ -0,0 +1,426 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace kth::database { + +#if defined(KTH_DB_NEW_FULL) + +//public +transaction_entry internal_database_basis::get_transaction(hash_digest const& hash, size_t fork_height) const { + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return transaction_entry{}; + } + + auto entry = get_transaction(hash, fork_height, db_txn); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return transaction_entry{}; + } + + return entry; + +} + +#if ! defined(KTH_DB_READONLY) + +template +result_code internal_database_basis::insert_transactions(I f, I l, uint32_t height, uint32_t median_time_past, uint64_t tx_count, KTH_DB_txn* db_txn) { + + auto id = tx_count; + uint32_t pos = 0; + + while (f != l) { + auto const& tx = *f; + + //TODO: (Mario) : Implement tx.Confirm to update existing transactions + auto res = insert_transaction(id, tx, height, median_time_past, pos, db_txn); + if (res != result_code::success && res != result_code::duplicated_key) { + return res; + } + + //remove unconfirmed transaction if exists + //TODO (Mario): don't recalculate tx.hash + res = remove_transaction_unconfirmed(tx.hash(), db_txn); + if (res != result_code::success && res != result_code::key_not_found) { + return res; + } + + ++f; + ++pos; + ++id; + } + + return result_code::success; +} +#endif // ! defined(KTH_DB_READONLY) + +transaction_entry internal_database_basis::get_transaction(uint64_t id, KTH_DB_txn* db_txn) const { + + auto key = kth_db_make_value(sizeof(id), &id); + KTH_DB_val value; + + auto res = kth_db_get(db_txn, dbi_transaction_db_, &key, &value); + + if (res != KTH_DB_SUCCESS) { + return {}; + } + + auto data = db_value_to_data_chunk(value); + auto entry = domain::create(data); + + return entry; +} + +transaction_entry internal_database_basis::get_transaction(hash_digest const& hash, size_t fork_height, KTH_DB_txn* db_txn) const { + auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); + KTH_DB_val value; + + auto res = kth_db_get(db_txn, dbi_transaction_hash_db_, &key, &value); + if (res != KTH_DB_SUCCESS) { + return {}; + } + + auto tx_id = *static_cast(kth_db_get_data(value));; + + auto const entry = get_transaction(tx_id, db_txn); + + if (entry.height() > fork_height) { + return {}; + } + + //Note(Knuth): Transaction stored in dbi_transaction_db_ are always confirmed + //the parameter requiere_confirmed is never used. + /*if ( ! require_confirmed) { + return entry; + } + + auto const confirmed = entry.confirmed(); + + return (confirmed && entry.height() > fork_height) || (require_confirmed && ! confirmed) ? transaction_entry{} : entry; + */ + + return entry; +} + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::insert_transaction(uint64_t id, domain::chain::transaction const& tx, uint32_t height, uint32_t median_time_past, uint32_t position, KTH_DB_txn* db_txn) { + + auto key = kth_db_make_value(sizeof(id), &id); + + //auto key_arr = tx.hash(); //TODO(fernando): podría estar afuera de la DBTx + //auto key = kth_db_make_value(key_arr.size(), key_arr.data()); + + auto valuearr = transaction_entry::factory_to_data(tx, height, median_time_past, position); + auto value = kth_db_make_value(valuearr.size(), valuearr.data()); + + auto res = kth_db_put(db_txn, dbi_transaction_db_, &key, &value, KTH_DB_APPEND); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key in Transaction DB [insert_transaction] ", res); + return result_code::duplicated_key; + } + + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error saving in Transaction DB [insert_transaction] ", res); + return result_code::other; + } + + + auto key_arr = tx.hash(); //TODO(fernando): podría estar afuera de la DBTx + auto key_tx = kth_db_make_value(key_arr.size(), key_arr.data()); + + res = kth_db_put(db_txn, dbi_transaction_hash_db_, &key_tx, &key, KTH_DB_NOOVERWRITE); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key in Transaction DB [insert_transaction] ", res); + return result_code::duplicated_key; + } + + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error saving in Transaction DB [insert_transaction] ", res); + return result_code::other; + } + + return result_code::success; +} + +result_code internal_database_basis::remove_transactions(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { + + auto const& txs = block.transactions(); + uint32_t pos = 0; + for (auto const& tx : txs) { + + auto const& hash = tx.hash(); + + auto res0 = remove_transaction_history_db(tx, height, db_txn); + if (res0 != result_code::success) { + return res0; + } + + if (pos > 0) { + auto res0 = remove_transaction_spend_db(tx, db_txn); + if (res0 != result_code::success && res0 != result_code::key_not_found) { + return res0; + } + } + + auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); + KTH_DB_val value; + + auto res = kth_db_get(db_txn, dbi_transaction_hash_db_, &key, &value); + if (res != KTH_DB_SUCCESS) { + return result_code::other; + } + + auto tx_id = *static_cast(kth_db_get_data(value));; + auto key_tx = kth_db_make_value(sizeof(tx_id), &tx_id); + + res = kth_db_del(db_txn, dbi_transaction_db_, &key_tx, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::other; + } + + res = kth_db_del(db_txn, dbi_transaction_hash_db_, &key, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::other; + } + + ++pos; + } + + + /*auto key = kth_db_make_value(sizeof(height), &height); + KTH_DB_val value; + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_block_db_, &cursor) != KTH_DB_SUCCESS) { + return {}; + } + + uint32_t pos = 0; + int rc; + if ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_SET)) == 0) { + + auto tx_id = *static_cast(kth_db_get_data(value));; + auto const entry = get_transaction(tx_id, db_txn); + auto const& tx = entry.transaction(); + + auto res0 = remove_transaction_history_db(tx, height, db_txn); + if (res0 != result_code::success) { + return res0; + } + + if (pos > 0) { + res0 = remove_transaction_spend_db(tx, db_txn); + if (res0 != result_code::success && res0 != result_code::key_not_found) { + return res0; + } + } + + auto key_tx = kth_db_make_value(sizeof(tx_id), &tx_id); + auto res = kth_db_del(db_txn, dbi_transaction_db_, &key_tx, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::other; + } + + auto key_hash = kth_db_make_value(tx.hash().size(), tx.hash().data()); + res = kth_db_del(db_txn, dbi_transaction_hash_db_, &key_hash, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::other; + } + + while ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_NEXT_DUP)) == 0) { + ++pos; + auto tx_id = *static_cast(kth_db_get_data(value));; + auto const entry = get_transaction(tx_id, db_txn); + auto const& tx = entry.transaction(); + + auto res0 = remove_transaction_history_db(tx, height, db_txn); + if (res0 != result_code::success) { + return res0; + } + + if (pos > 0) { + res0 = remove_transaction_spend_db(tx, db_txn); + if (res0 != result_code::success && res0 != result_code::key_not_found) { + return res0; + } + } + + auto key_tx = kth_db_make_value(sizeof(tx_id), &tx_id); + auto res = kth_db_del(db_txn, dbi_transaction_db_, &key_tx, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::other; + } + + auto key_hash = kth_db_make_value(tx.hash().size(), tx.hash().data()); + res = kth_db_del(db_txn, dbi_transaction_hash_db_, &key_hash, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::other; + } + } + } + + kth_db_cursor_close(cursor);*/ + + + /*//To get the tx hashes to remove, we need to read the block db + if (kth_db_get(db_txn, dbi_block_db_, &key, &value) != KTH_DB_SUCCESS) { + return result_code::other; + } + + auto n = kth_db_get_size(value); + auto f = static_cast(kth_db_get_data(value)); + //precondition: mv_size es multiplo de 32 + + uint32_t pos = 0; + while (n != 0) { + hash_digest h; + std::copy(f, f + kth::hash_size, h.data()); + + auto key_tx = kth_db_make_value(h.size(), h.data()); + + auto const& entry = get_transaction(h, height, db_txn); + if ( ! entry.is_valid() ) { + return result_code::other; + } + + auto const& tx = entry.transaction(); + + auto res0 = remove_transaction_history_db(tx, height, db_txn); + if (res0 != result_code::success) { + return res0; + } + + if (pos > 0) { + res0 = remove_transaction_spend_db(tx, db_txn); + if (res0 != result_code::success && res0 != result_code::key_not_found) { + return res0; + } + } + + auto res = kth_db_del(db_txn, dbi_transaction_db_, &key_tx, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting transaction DB in LMDB [remove_transactions] - kth_db_del: ", res); + return result_code::other; + } + + n -= kth::hash_size; + f += kth::hash_size; + ++pos; + }*/ + + return result_code::success; +} + +result_code internal_database_basis::update_transaction(domain::chain::transaction const& tx, uint32_t height, uint32_t median_time_past, uint32_t position, KTH_DB_txn* db_txn) { + auto key_arr = tx.hash(); //TODO(fernando): podría estar afuera de la DBTx + auto key = kth_db_make_value(key_arr.size(), key_arr.data()); + + auto valuearr = transaction_entry::factory_to_data(tx, height, median_time_past, position); + auto value = kth_db_make_value(valuearr.size(), valuearr.data()); + + auto res = kth_db_put(db_txn, dbi_transaction_db_, &key, &value, 0); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key in Transaction DB [insert_transaction] ", res); + return result_code::duplicated_key; + } + + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error saving in Transaction DB [insert_transaction] ", res); + return result_code::other; + } + + return result_code::success; +} + +result_code internal_database_basis::set_spend(domain::chain::output_point const& point, uint32_t spender_height, KTH_DB_txn* db_txn) { + + // Limit search to confirmed transactions at or below the spender height, + // since a spender cannot spend above its own height. + // Transactions are not marked as spent unless the spender is confirmed. + // This is consistent with support for unconfirmed double spends. + + auto entry = get_transaction(point.hash(), spender_height, db_txn); + + // The transaction is not exist as confirmed at or below the height. + if ( ! entry.is_valid() ) { + return result_code::other; + } + + auto const& tx = entry.transaction(); + + if (point.index() >= tx.outputs().size()) { + return result_code::other; + } + + //update spender_height + auto& output = tx.outputs()[point.index()]; + output.validation.spender_height = spender_height; + + //overwrite transaction + auto ret = update_transaction(tx, entry.height(), entry.median_time_past(), entry.position(), db_txn); + if (ret != result_code::success & ret != result_code::duplicated_key) { + return ret; + } + + return result_code::success; +} + +result_code internal_database_basis::set_unspend(domain::chain::output_point const& point, KTH_DB_txn* db_txn) { + return set_spend(point, domain::chain::output::validation::not_spent, db_txn); +} +#endif // ! defined(KTH_DB_READONLY) + +//TODO(fernando): this must be a generic function +uint64_t internal_database_basis::get_tx_count(KTH_DB_txn* db_txn) const { + MDB_stat db_stats; + auto ret = mdb_stat(db_txn, dbi_transaction_db_, &db_stats); + if (ret != KTH_DB_SUCCESS) { + return max_uint64; + } + return db_stats.ms_entries; +} + +#endif //KTH_NEW_DB_FULL + +} // namespace kth::database From f4551d26c5dad8030d54b5dea963a58f38dcb412 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:08:16 -0300 Subject: [PATCH 16/26] move the code from implementation file to translation unit --- src/databases/spend_database.cpp | 111 +++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 src/databases/spend_database.cpp diff --git a/src/databases/spend_database.cpp b/src/databases/spend_database.cpp new file mode 100644 index 00000000..0361dc44 --- /dev/null +++ b/src/databases/spend_database.cpp @@ -0,0 +1,111 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace kth::database { + +#if defined(KTH_DB_NEW_FULL) + +//public +domain::chain::input_point internal_database_basis::get_spend(domain::chain::output_point const& point) const { + + auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); + auto key = kth_db_make_value(keyarr.size(), keyarr.data()); + KTH_DB_val value; + + KTH_DB_txn* db_txn; + auto res0 = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res0 != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error begining LMDB Transaction [get_spend] ", res0); + return domain::chain::input_point{}; + } + + res0 = kth_db_get(db_txn, dbi_spend_db_, &key, &value); + if (res0 != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + // kth_db_txn_abort(db_txn); + return domain::chain::input_point{}; + } + + auto data = db_value_to_data_chunk(value); + + res0 = kth_db_txn_commit(db_txn); + if (res0 != KTH_DB_SUCCESS) { + LOG_DEBUG(LOG_DATABASE, "Error commiting LMDB Transaction [get_spend] ", res0); + return domain::chain::input_point{}; + } + + auto res = domain::create(data); + return res; +} + +#if ! defined(KTH_DB_READONLY) + +//pivate +result_code internal_database_basis::insert_spend(domain::chain::output_point const& out_point, domain::chain::input_point const& in_point, KTH_DB_txn* db_txn) { + + auto keyarr = out_point.to_data(KTH_INTERNAL_DB_WIRE); + auto key = kth_db_make_value(keyarr.size(), keyarr.data()); + + auto value_arr = in_point.to_data(); + auto value = kth_db_make_value(value_arr.size(), value_arr.data()); + + auto res = kth_db_put(db_txn, dbi_spend_db_, &key, &value, KTH_DB_NOOVERWRITE); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key inserting spend [insert_spend] ", res); + return result_code::duplicated_key; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error inserting spend [insert_spend] ", res); + return result_code::other; + } + + return result_code::success; +} + +result_code internal_database_basis::remove_transaction_spend_db(domain::chain::transaction const& tx, KTH_DB_txn* db_txn) { + + for (auto const& input: tx.inputs()) { + + auto const& prevout = input.previous_output(); + + auto res = remove_spend(prevout, db_txn); + if (res != result_code::success) { + return res; + } + + /*res = set_unspend(prevout, db_txn); + if (res != result_code::success) { + return res; + }*/ + } + + return result_code::success; +} + +result_code internal_database_basis::remove_spend(domain::chain::output_point const& out_point, KTH_DB_txn* db_txn) { + + auto keyarr = out_point.to_data(KTH_INTERNAL_DB_WIRE); //TODO(fernando): podría estar afuera de la DBTx + auto key = kth_db_make_value(keyarr.size(), keyarr.data()); //TODO(fernando): podría estar afuera de la DBTx + + auto res = kth_db_del(db_txn, dbi_spend_db_, &key, NULL); + + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting spend [remove_spend] ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting spend [remove_spend] ", res); + return result_code::other; + } + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + + +#endif // defined(KTH_DB_NEW_FULL) + +} // namespace kth::database From 9b58d10a33fb8cea1a8ed17a67f4d2168eb69c6c Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:08:23 -0300 Subject: [PATCH 17/26] move the code from implementation file to translation unit --- src/databases/block_database.cpp | 250 +++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 src/databases/block_database.cpp diff --git a/src/databases/block_database.cpp b/src/databases/block_database.cpp new file mode 100644 index 00000000..f574c5db --- /dev/null +++ b/src/databases/block_database.cpp @@ -0,0 +1,250 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace kth::database { + +//public +std::pair internal_database_basis::get_block(hash_digest const& hash) const { + auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return {}; + } + + KTH_DB_val value; + if (kth_db_get(db_txn, dbi_block_header_by_hash_, &key, &value) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + // kth_db_txn_abort(db_txn); + return {}; + } + + // assert kth_db_get_size(value) == 4; + auto height = *static_cast(kth_db_get_data(value)); + + auto block = get_block(height, db_txn); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return {}; + } + + return {block, height}; +} + +//public +domain::chain::block internal_database_basis::get_block(uint32_t height) const { + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return domain::chain::block{}; + } + + auto block = get_block(height, db_txn); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return domain::chain::block{}; + } + + return block; +} + +domain::chain::block internal_database_basis::get_block(uint32_t height, KTH_DB_txn* db_txn) const { + + auto key = kth_db_make_value(sizeof(height), &height); + +#if defined(KTH_DB_NEW_BLOCKS) + + KTH_DB_val value; + + if (kth_db_get(db_txn, dbi_block_db_, &key, &value) != KTH_DB_SUCCESS) { + return domain::chain::block{}; + } + + auto data = db_value_to_data_chunk(value); + auto res = domain::create(data); + return res; + +#elif defined(KTH_DB_NEW_FULL) + + auto header = get_header(height, db_txn); + if ( ! header.is_valid()) { + return {}; + } + + domain::chain::transaction::list tx_list; + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_block_db_, &cursor) != KTH_DB_SUCCESS) { + return {}; + } + + KTH_DB_val value; + int rc; + if ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_SET)) == 0) { + + auto tx_id = *static_cast(kth_db_get_data(value));; + auto const entry = get_transaction(tx_id, db_txn); + + if ( ! entry.is_valid()) { + return {}; + } + + tx_list.push_back(std::move(entry.transaction())); + + while ((rc = kth_db_cursor_get(cursor, &key, &value, MDB_NEXT_DUP)) == 0) { + auto tx_id = *static_cast(kth_db_get_data(value));; + auto const entry = get_transaction(tx_id, db_txn); + tx_list.push_back(std::move(entry.transaction())); + } + } + + kth_db_cursor_close(cursor); + + /*auto n = kth_db_get_size(value); + auto f = static_cast(kth_db_get_data(value)); + //precondition: mv_size es multiplo de 32 + + domain::chain::transaction::list tx_list; + tx_list.reserve(n / kth::hash_size); + + while (n != 0) { + hash_digest h; + std::copy(f, f + kth::hash_size, h.data()); + + auto tx_entry = get_transaction(h,max_uint32, db_txn); + + if ( ! tx_entry.is_valid() ) { + return domain::chain::block{}; + } + + auto const& tx = tx_entry.transaction(); + + tx_list.push_back(std::move(tx)); + + n -= kth::hash_size; + f += kth::hash_size; + }*/ + + return domain::chain::block{header, std::move(tx_list)}; + +#else + auto block = get_block_reorg(height, db_txn, dbi_reorg_block_); + return block; +#endif //defined(KTH_DB_NEW_FULL) +} + + +#if defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) + +#if ! defined(KTH_DB_READONLY) + +#if defined(KTH_DB_NEW_BLOCKS) +result_code internal_database_basis::insert_block(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { +#elif defined(KTH_DB_NEW_FULL) +result_code internal_database_basis::insert_block(domain::chain::block const& block, uint32_t height, uint64_t tx_count, KTH_DB_txn* db_txn) { +#endif + +/*#if defined(KTH_DB_NEW_BLOCKS) + auto data = block.to_data(false); +#elif defined(KTH_DB_NEW_FULL) + auto data = serialize_txs(block); +#endif +*/ + +auto key = kth_db_make_value(sizeof(height), &height); + +#if defined(KTH_DB_NEW_BLOCKS) + + //TODO: store tx hash + auto data = block.to_data(false); + auto value = kth_db_make_value(data.size(), data.data()); + + auto res = kth_db_put(db_txn, dbi_block_db_, &key, &value, KTH_DB_APPEND); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key in Block DB [insert_block] ", res); + return result_code::duplicated_key; + } + + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error saving in Block DB [insert_block] ", res); + return result_code::other; + } + +#elif defined(KTH_DB_NEW_FULL) + + auto const& txs = block.transactions(); + + for (uint64_t i = tx_count; i Date: Mon, 21 Dec 2020 12:08:31 -0300 Subject: [PATCH 18/26] move the code from implementation file to translation unit --- src/databases/header_database.cpp | 89 +++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 src/databases/header_database.cpp diff --git a/src/databases/header_database.cpp b/src/databases/header_database.cpp new file mode 100644 index 00000000..25b00bbf --- /dev/null +++ b/src/databases/header_database.cpp @@ -0,0 +1,89 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace kth::database { + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::push_block_header(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { + // auto valuearr = block.header().to_data(true); //TODO(fernando): podría estar afuera de la DBTx + // auto key = kth_db_make_value(sizeof(height), &height); + // auto value = kth_db_make_value(valuearr.size(), valuearr.data()); + + // auto res = kth_db_put(db_txn, dbi_block_header_, &key, &value, KTH_DB_APPEND); + // if (res == KTH_DB_KEYEXIST) { + // //TODO(fernando): El logging en general no está bueno que esté en la DbTx. + // LOG_INFO(LOG_DATABASE, "Duplicate key inserting block header [push_block_header] ", res); //TODO(fernando): podría estar afuera de la DBTx. + // return result_code::duplicated_key; + // } + // if (res != KTH_DB_SUCCESS) { + // LOG_INFO(LOG_DATABASE, "Error inserting block header [push_block_header] ", res); + // return result_code::other; + // } + + // auto key_by_hash_arr = block.hash(); //TODO(fernando): podría estar afuera de la DBTx + // auto key_by_hash = kth_db_make_value(key_by_hash_arr.size(), key_by_hash_arr.data()); + + // res = kth_db_put(db_txn, dbi_block_header_by_hash_, &key_by_hash, &key, KTH_DB_NOOVERWRITE); + // if (res == KTH_DB_KEYEXIST) { + // LOG_INFO(LOG_DATABASE, "Duplicate key inserting block header by hash [push_block_header] ", res); + // return result_code::duplicated_key; + // } + // if (res != KTH_DB_SUCCESS) { + // LOG_INFO(LOG_DATABASE, "Error inserting block header by hash [push_block_header] ", res); + // return result_code::other; + // } + + return result_code::success; +} +#endif // ! defined(KTH_DB_READONLY) + +domain::chain::header internal_database_basis::get_header(uint32_t height, KTH_DB_txn* db_txn) const { + auto key = kth_db_make_value(sizeof(height), &height); + KTH_DB_val value; + + if (kth_db_get(db_txn, dbi_block_header_, &key, &value) != KTH_DB_SUCCESS) { + return domain::chain::header{}; + } + + auto data = db_value_to_data_chunk(value); + auto res = domain::create(data); + return res; +} + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::remove_block_header(hash_digest const& hash, uint32_t height, KTH_DB_txn* db_txn) { + auto key = kth_db_make_value(sizeof(height), &height); + auto res = kth_db_del(db_txn, dbi_block_header_, &key, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting block header in LMDB [remove_block_header] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Erro deleting block header in LMDB [remove_block_header] - kth_db_del: ", res); + return result_code::other; + } + + auto key_hash = kth_db_make_value(hash.size(), const_cast(hash).data()); + + res = kth_db_del(db_txn, dbi_block_header_by_hash_, &key_hash, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting block header by hash in LMDB [remove_block_header] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Erro deleting block header by hash in LMDB [remove_block_header] - kth_db_del: ", res); + return result_code::other; + } + + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + + +} // namespace kth::database From fe3a3c1d4e9992d1b8cfbfc8007974e9626a8354 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:09:00 -0300 Subject: [PATCH 19/26] move the code from implementation file to translation unit --- src/databases/history_database.cpp | 370 +++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 src/databases/history_database.cpp diff --git a/src/databases/history_database.cpp b/src/databases/history_database.cpp new file mode 100644 index 00000000..a9be71e0 --- /dev/null +++ b/src/databases/history_database.cpp @@ -0,0 +1,370 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace kth::database { + +#if defined(KTH_DB_NEW_FULL) + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::insert_history_db(domain::wallet::payment_address const& address, data_chunk const& entry, KTH_DB_txn* db_txn) { + + auto key_arr = address.hash(); //TODO(fernando): should I take a reference? + auto key = kth_db_make_value(key_arr.size(), key_arr.data()); + auto value = kth_db_make_value(entry.size(), const_cast(entry).data()); + + auto res = kth_db_put(db_txn, dbi_history_db_, &key, &value, MDB_APPENDDUP); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key inserting history [insert_history_db] ", res); + return result_code::duplicated_key; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error inserting history [insert_history_db] ", res); + return result_code::other; + } + + return result_code::success; +} + +result_code internal_database_basis::insert_input_history(domain::chain::input_point const& inpoint, uint32_t height, domain::chain::input const& input, KTH_DB_txn* db_txn) { + + auto const& prevout = input.previous_output(); + + if (prevout.validation.cache.is_valid()) { + + uint64_t history_count = get_history_count(db_txn); + + if (history_count == max_uint64) { + LOG_INFO(LOG_DATABASE, "Error getting history items count"); + return result_code::other; + } + + uint64_t id = history_count; + + // This results in a complete and unambiguous history for the + // address since standard outputs contain unambiguous address data. + for (auto const& address : prevout.validation.cache.addresses()) { + auto valuearr = history_entry::factory_to_data(id, inpoint, domain::chain::point_kind::spend, height, inpoint.index(), prevout.checksum()); + auto res = insert_history_db(address, valuearr, db_txn); + if (res != result_code::success) { + return res; + } + ++id; + } + } else { + //During an IBD with checkpoints some previous output info is missing. + //We can recover it by accessing the database + + //TODO (Mario) check if we can query UTXO + //TODO (Mario) requiere_confirmed = true ?? + + auto entry = get_utxo(prevout, db_txn); + + //auto entry = get_transaction(prevout.hash(), max_uint32, true, db_txn); + + if (entry.is_valid()) { + + //auto const& tx = entry.transaction(); + + //auto const& out_output = tx.outputs()[prevout.index()]; + + uint64_t history_count = get_history_count(db_txn); + if (history_count == max_uint64) { + LOG_INFO(LOG_DATABASE, "Error getting history items count"); + return result_code::other; + } + + uint64_t id = history_count; + + auto const& out_output = entry.output(); + for (auto const& address : out_output.addresses()) { + auto valuearr = history_entry::factory_to_data(id, inpoint, domain::chain::point_kind::spend, height, inpoint.index(), prevout.checksum()); + auto res = insert_history_db(address, valuearr, db_txn); + if (res != result_code::success) { + return res; + } + ++id; + } + } + else { + LOG_INFO(LOG_DATABASE, "Error finding UTXO for input history [insert_input_history]"); + } + } + + return result_code::success; +} + +result_code internal_database_basis::insert_output_history(hash_digest const& tx_hash,uint32_t height, uint32_t index, domain::chain::output const& output, KTH_DB_txn* db_txn ) { + + uint64_t history_count = get_history_count(db_txn); + if (history_count == max_uint64) { + LOG_INFO(LOG_DATABASE, "Error getting history items count"); + return result_code::other; + } + + uint64_t id = history_count; + + auto const outpoint = domain::chain::output_point {tx_hash, index}; + auto const value = output.value(); + + // Standard outputs contain unambiguous address data. + for (auto const& address : output.addresses()) { + auto valuearr = history_entry::factory_to_data(id, outpoint, domain::chain::point_kind::output, height, index, value); + auto res = insert_history_db(address, valuearr, db_txn); + if (res != result_code::success) { + return res; + } + ++id; + } + + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +// static +domain::chain::history_compact internal_database_basis::history_entry_to_history_compact(history_entry const& entry) { + return domain::chain::history_compact{entry.point_kind(), entry.point(), entry.height(), entry.value_or_checksum()}; +} + +domain::chain::history_compact::list internal_database_basis::get_history(short_hash const& key, size_t limit, size_t from_height) const { + + domain::chain::history_compact::list result; + + if (limit == 0) { + return result; + } + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return result; + } + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_history_db_, &cursor) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + return result; + } + + auto key_hash = kth_db_make_value(key.size(), const_cast(key).data()); + KTH_DB_val value; + int rc; + if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) { + + auto data = db_value_to_data_chunk(value); + auto entry = domain::create(data); + + if (from_height == 0 || entry.height() >= from_height) { + result.push_back(history_entry_to_history_compact(entry)); + } + + while ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_NEXT_DUP)) == 0) { + + if (limit > 0 && result.size() >= limit) { + break; + } + + auto data = db_value_to_data_chunk(value); + auto entry = domain::create(data); + + if (from_height == 0 || entry.height() >= from_height) { + result.push_back(history_entry_to_history_compact(entry)); + } + + } + } + + kth_db_cursor_close(cursor); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return result; + } + + return result; +} + +std::vector internal_database_basis::get_history_txns(short_hash const& key, size_t limit, size_t from_height) const { + + std::set temp; + std::vector result; + + // Stop once we reach the limit (if specified). + if (limit == 0) + return result; + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return result; + } + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_history_db_, &cursor) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + return result; + } + + auto key_hash = kth_db_make_value(key.size(), const_cast(key).data());; + KTH_DB_val value; + int rc; + if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) { + + auto data = db_value_to_data_chunk(value); + auto entry = domain::create(data); + + if (from_height == 0 || entry.height() >= from_height) { + // Avoid inserting the same tx + auto const & pair = temp.insert(entry.point().hash()); + if (pair.second){ + // Add valid txns to the result vector + result.push_back(*pair.first); + } + } + + while ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_NEXT_DUP)) == 0) { + + if (limit > 0 && result.size() >= limit) { + break; + } + + auto data = db_value_to_data_chunk(value); + auto entry = domain::create(data); + + if (from_height == 0 || entry.height() >= from_height) { + // Avoid inserting the same tx + auto const & pair = temp.insert(entry.point().hash()); + if (pair.second){ + // Add valid txns to the result vector + result.push_back(*pair.first); + } + } + + } + } + + kth_db_cursor_close(cursor); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return result; + } + + return result; +} + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::remove_transaction_history_db(domain::chain::transaction const& tx, size_t height, KTH_DB_txn* db_txn) { + + for (auto const& output: tx.outputs()) { + for (auto const& address : output.addresses()) { + auto res = remove_history_db(address, height, db_txn); + if (res != result_code::success) { + return res; + } + } + } + + for (auto const& input: tx.inputs()) { + + auto const& prevout = input.previous_output(); + + if (prevout.validation.cache.is_valid()) { + for (auto const& address : prevout.validation.cache.addresses()) { + auto res = remove_history_db(address, height, db_txn); + if (res != result_code::success) { + return res; + } + } + } + else { + + auto const& entry = get_transaction(prevout.hash(), max_uint32, db_txn); + + if (entry.is_valid()) { + + auto const& tx = entry.transaction(); + auto const& out_output = tx.outputs()[prevout.index()]; + + for (auto const& address : out_output.addresses()) { + + auto res = remove_history_db(address, height, db_txn); + if (res != result_code::success) { + return res; + } + } + } + + /* + for (auto const& address : input.addresses()) { + auto res = remove_history_db(address, height, db_txn); + if (res != result_code::success) { + return res; + } + }*/ + } + } + + return result_code::success; +} + +result_code internal_database_basis::remove_history_db(const short_hash& key, size_t height, KTH_DB_txn* db_txn) { + + KTH_DB_cursor* cursor; + + if (kth_db_cursor_open(db_txn, dbi_history_db_, &cursor) != KTH_DB_SUCCESS) { + return result_code::other; + } + + auto key_hash = kth_db_make_value(key.size(), const_cast(key).data());; + KTH_DB_val value; + int rc; + if ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_SET)) == 0) { + + auto data = db_value_to_data_chunk(value); + auto entry = domain::create(data); + + if (entry.height() == height) { + + if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { + kth_db_cursor_close(cursor); + return result_code::other; + } + } + + while ((rc = kth_db_cursor_get(cursor, &key_hash, &value, MDB_NEXT_DUP)) == 0) { + + auto data = db_value_to_data_chunk(value); + auto entry = domain::create(data); + + if (entry.height() == height) { + if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { + kth_db_cursor_close(cursor); + return result_code::other; + } + } + } + } + + kth_db_cursor_close(cursor); + + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +uint64_t internal_database_basis::get_history_count(KTH_DB_txn* db_txn) const { + MDB_stat db_stats; + auto ret = mdb_stat(db_txn, dbi_history_db_, &db_stats); + if (ret != KTH_DB_SUCCESS) { + return max_uint64; + } + return db_stats.ms_entries; +} + +#endif //KTH_NEW_DB_FULL + +} // namespace kth::database From 983b6db2b2ed43034f83857b792f551330419c96 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:09:08 -0300 Subject: [PATCH 20/26] move the code from implementation file to translation unit --- src/databases/reorg_database.cpp | 266 +++++++++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 src/databases/reorg_database.cpp diff --git a/src/databases/reorg_database.cpp b/src/databases/reorg_database.cpp new file mode 100644 index 00000000..74319299 --- /dev/null +++ b/src/databases/reorg_database.cpp @@ -0,0 +1,266 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include + +#include +#include +#include + +namespace kth::database { + +#if ! defined(KTH_DB_READONLY) + +result_code insert_reorg_pool(uint32_t height, KTH_DB_val& key, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_utxo, KTH_DB_dbi dbi_reorg_pool, KTH_DB_dbi dbi_reorg_index) { + KTH_DB_val value; + //TODO: use cursors + auto res = kth_db_get(db_txn, dbi_utxo, &key, &value); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found getting UTXO [insert_reorg_pool] ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error getting UTXO [insert_reorg_pool] ", res); + return result_code::other; + } + + res = kth_db_put(db_txn, dbi_reorg_pool, &key, &value, KTH_DB_NOOVERWRITE); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key inserting in reorg pool [insert_reorg_pool] ", res); + return result_code::duplicated_key; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error inserting in reorg pool [insert_reorg_pool] ", res); + return result_code::other; + } + + auto key_index = kth_db_make_value(sizeof(height), &height); //TODO(fernando): podría estar afuera de la DBTx + auto value_index = kth_db_make_value(kth_db_get_size(key), kth_db_get_data(key)); //TODO(fernando): podría estar afuera de la DBTx + res = kth_db_put(db_txn, dbi_reorg_index, &key_index, &value_index, 0); + + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key inserting in reorg index [insert_reorg_pool] ", res); + return result_code::duplicated_key; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error inserting in reorg index [insert_reorg_pool] ", res); + return result_code::other; + } + + return result_code::success; +} + +//TODO : remove this database in db_new_with_blocks and db_new_full +result_code push_block_reorg(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_block) { + + auto valuearr = block.to_data(false); //TODO(fernando): podría estar afuera de la DBTx + auto key = kth_db_make_value(sizeof(height), &height); //TODO(fernando): podría estar afuera de la DBTx + auto value = kth_db_make_value(valuearr.size(), valuearr.data()); //TODO(fernando): podría estar afuera de la DBTx + + auto res = kth_db_put(db_txn, dbi_reorg_block, &key, &value, KTH_DB_NOOVERWRITE); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key inserting in reorg block [push_block_reorg] ", res); + return result_code::duplicated_key; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error inserting in reorg block [push_block_reorg] ", res); + return result_code::other; + } + + return result_code::success; +} + +result_code insert_output_from_reorg_and_remove(domain::chain::output_point const& point, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_utxo, KTH_DB_dbi dbi_reorg_pool) { + auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); + auto key = kth_db_make_value(keyarr.size(), keyarr.data()); + + KTH_DB_val value; + auto res = kth_db_get(db_txn, dbi_reorg_pool, &key, &value); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found in reorg pool [insert_output_from_reorg_and_remove] ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error in reorg pool [insert_output_from_reorg_and_remove] ", res); + return result_code::other; + } + + res = kth_db_put(db_txn, dbi_utxo, &key, &value, KTH_DB_NOOVERWRITE); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key inserting in UTXO [insert_output_from_reorg_and_remove] ", res); + return result_code::duplicated_key; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error inserting in UTXO [insert_output_from_reorg_and_remove] ", res); + return result_code::other; + } + + res = kth_db_del(db_txn, dbi_reorg_pool, &key, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting in reorg pool [insert_output_from_reorg_and_remove] ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting in reorg pool [insert_output_from_reorg_and_remove] ", res); + return result_code::other; + } + return result_code::success; +} + +result_code remove_block_reorg(uint32_t height, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_block) { + + auto key = kth_db_make_value(sizeof(height), &height); + auto res = kth_db_del(db_txn, dbi_reorg_block, &key, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting reorg block in LMDB [remove_block_reorg] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting reorg block in LMDB [remove_block_reorg] - kth_db_del: ", res); + return result_code::other; + } + return result_code::success; +} + +result_code remove_reorg_index(uint32_t height, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_index) { + + auto key = kth_db_make_value(sizeof(height), &height); + auto res = kth_db_del(db_txn, dbi_reorg_index, &key, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_DEBUG(LOG_DATABASE, "Key not found deleting reorg index in LMDB [remove_reorg_index] - height: ", height, " - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_DEBUG(LOG_DATABASE, "Error deleting reorg index in LMDB [remove_reorg_index] - height: ", height, " - kth_db_del: ", res); + return result_code::other; + } + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +domain::chain::block get_block_reorg(uint32_t height, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_block) { + auto key = kth_db_make_value(sizeof(height), &height); + KTH_DB_val value; + + if (kth_db_get(db_txn, dbi_reorg_block, &key, &value) != KTH_DB_SUCCESS) { + return {}; + } + + auto data = db_value_to_data_chunk(value); + auto res = domain::create(data); //TODO(fernando): mover fuera de la DbTx + return res; +} + +domain::chain::block get_block_reorg(uint32_t height, KTH_DB_dbi dbi_reorg_block, KTH_DB_env* env) { + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return {}; + } + + auto block = get_block_reorg(height, db_txn, dbi_reorg_block); + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return {}; + } + + return block; +} + +#if ! defined(KTH_DB_READONLY) + +result_code prune_reorg_index(uint32_t remove_until, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_pool, KTH_DB_dbi dbi_reorg_index) { + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_reorg_index, &cursor) != KTH_DB_SUCCESS) { + return result_code::other; + } + + KTH_DB_val key; + KTH_DB_val value; + int rc; + while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == KTH_DB_SUCCESS) { + auto current_height = *static_cast(kth_db_get_data(key)); + if (current_height < remove_until) { + + auto res = kth_db_del(db_txn, dbi_reorg_pool, &value, NULL); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found deleting reorg pool in LMDB [prune_reorg_index] - kth_db_del: ", res); + return result_code::key_not_found; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error deleting reorg pool in LMDB [prune_reorg_index] - kth_db_del: ", res); + return result_code::other; + } + + if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { + kth_db_cursor_close(cursor); + return result_code::other; + } + } else { + break; + } + } + + kth_db_cursor_close(cursor); + return result_code::success; +} + +result_code prune_reorg_block(uint32_t amount_to_delete, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_block) { + //precondition: amount_to_delete >= 1 + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_reorg_block, &cursor) != KTH_DB_SUCCESS) { + return result_code::other; + } + + int rc; + while ((rc = kth_db_cursor_get(cursor, nullptr, nullptr, KTH_DB_NEXT)) == KTH_DB_SUCCESS) { + if (kth_db_cursor_del(cursor, 0) != KTH_DB_SUCCESS) { + kth_db_cursor_close(cursor); + return result_code::other; + } + if (--amount_to_delete == 0) break; + } + + kth_db_cursor_close(cursor); + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +result_code get_first_reorg_block_height(uint32_t& out_height, KTH_DB_dbi dbi_reorg_block, KTH_DB_env* env) { + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return result_code::other; + } + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_reorg_block, &cursor) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + return result_code::other; + } + + KTH_DB_val key; + int rc; + if ((rc = kth_db_cursor_get(cursor, &key, nullptr, KTH_DB_FIRST)) != KTH_DB_SUCCESS) { + return result_code::db_empty; + } + + // assert kth_db_get_size(key) == 4; + out_height = *static_cast(kth_db_get_data(key)); + + kth_db_cursor_close(cursor); + + // kth_db_txn_abort(db_txn); + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return result_code::other; + } + + return result_code::success; +} + +} // namespace kth::database From 8223b109d2de805e96a7ba1cc755f3292068f33f Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:09:16 -0300 Subject: [PATCH 21/26] move the code from implementation file to translation unit --- src/databases/internal_database.cpp | 1085 +++++++++++++++++++++++++++ 1 file changed, 1085 insertions(+) create mode 100644 src/databases/internal_database.cpp diff --git a/src/databases/internal_database.cpp b/src/databases/internal_database.cpp new file mode 100644 index 00000000..2b5ef55e --- /dev/null +++ b/src/databases/internal_database.cpp @@ -0,0 +1,1085 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +namespace kth::database { + +using utxo_pool_t = std::unordered_map; + +internal_database_basis::internal_database_basis(path const& db_dir, uint32_t reorg_pool_limit, uint64_t db_max_size, bool safe_mode) + : db_dir_(db_dir) + , reorg_pool_limit_(reorg_pool_limit) + , limit_(blocks_to_seconds(reorg_pool_limit)) + , db_max_size_(db_max_size) + , safe_mode_(safe_mode) +{} + +internal_database_basis::~internal_database_basis() { + close(); +} + +#if ! defined(KTH_DB_READONLY) + +bool internal_database_basis::create() { + std::error_code ec; + + if ( ! std::filesystem::create_directories(db_dir_, ec)) { + if (ec.value() == directory_exists) { + LOG_ERROR(LOG_DATABASE, "Failed because the directory ", db_dir_.string(), " already exists."); + return false; + } + + LOG_ERROR(LOG_DATABASE, "Failed to create directory ", db_dir_.string(), " with error, '", ec.message(), "'."); + return false; + } + + auto ret = open_internal(); + if ( ! ret ) { + return false; + } + + ret = create_db_mode_property(); + if ( ! ret ) { + return false; + } + + return true; +} + +bool internal_database_basis::create_db_mode_property() { + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, 0, &db_txn); + if (res != KTH_DB_SUCCESS) { + return false; + } + + db_mode_code db_mode_; + +#if defined(KTH_DB_NEW_FULL) + db_mode_ = db_mode_code::db_new_full; +#elif defined(KTH_DB_NEW_BLOCKS) + db_mode_ = db_mode_code::db_new_with_blocks; +#else + db_mode_ = db_mode_code::db_new; +#endif + + property_code property_code_ = property_code::db_mode; + + auto key = kth_db_make_value(sizeof(property_code_), &property_code_); + auto value = kth_db_make_value(sizeof(db_mode_), &db_mode_); + + res = kth_db_put(db_txn, dbi_properties_, &key, &value, KTH_DB_NOOVERWRITE); + if (res != KTH_DB_SUCCESS) { + LOG_ERROR(LOG_DATABASE, "Failed saving in DB Properties [create_db_mode_property] ", static_cast(res)); + kth_db_txn_abort(db_txn); + return false; + } + + res = kth_db_txn_commit(db_txn); + if (res != KTH_DB_SUCCESS) { + return false; + } + + return true; +} + +#endif // ! defined(KTH_DB_READONLY) + +bool internal_database_basis::open() { + + auto ret = open_internal(); + if ( ! ret ) { + return false; + } + + ret = verify_db_mode_property(); + if ( ! ret ) { + return false; + } + + return true; +} + +bool internal_database_basis::open_internal() { + + if ( ! create_and_open_environment()) { + LOG_ERROR(LOG_DATABASE, "Error configuring LMDB environment."); + return false; + } + + return open_databases(); +} + +bool internal_database_basis::verify_db_mode_property() const { + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return false; + } + + property_code property_code_ = property_code::db_mode; + + auto key = kth_db_make_value(sizeof(property_code_), &property_code_); + KTH_DB_val value; + + res = kth_db_get(db_txn, dbi_properties_, &key, &value); + if (res != KTH_DB_SUCCESS) { + LOG_ERROR(LOG_DATABASE, "Failed getting DB Properties [verify_db_mode_property] ", static_cast(res)); + kth_db_txn_abort(db_txn); + return false; + } + + auto db_mode_ = *static_cast(kth_db_get_data(value)); + + res = kth_db_txn_commit(db_txn); + if (res != KTH_DB_SUCCESS) { + return false; + } + +#if defined(KTH_DB_NEW_FULL) + auto db_mode_node_ = db_mode_code::db_new_full; +#elif defined(KTH_DB_NEW_BLOCKS) + auto db_mode_node_ = db_mode_code::db_new_with_blocks; +#else + auto db_mode_node_ = db_mode_code::db_new; +#endif + + if (db_mode_ != db_mode_node_) { + LOG_ERROR(LOG_DATABASE, "Error validating DB Mode, the node is compiled for another DB mode. Node DB Mode: " + , static_cast(db_mode_node_) + , ", Actual DB Mode: " + , static_cast(db_mode_)); + return false; + } + + return true; +} + +bool internal_database_basis::close() { + if (db_opened_) { + + //TODO(fernando): check sync + //Force synchronous flush (use with KTH_DB_NOSYNC or MDB_NOMETASYNC, with other flags do nothing) +#if defined(KTH_USE_LIBMDBX) + kth_db_env_sync(env_); +#else + kth_db_env_sync(env_, true); +#endif + kth_db_dbi_close(env_, dbi_block_header_); + kth_db_dbi_close(env_, dbi_block_header_by_hash_); + kth_db_dbi_close(env_, dbi_utxo_); + kth_db_dbi_close(env_, dbi_reorg_pool_); + kth_db_dbi_close(env_, dbi_reorg_index_); + kth_db_dbi_close(env_, dbi_reorg_block_); + kth_db_dbi_close(env_, dbi_properties_); + + #if defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) + kth_db_dbi_close(env_, dbi_block_db_); + #endif + + #if defined(KTH_DB_NEW_FULL) + kth_db_dbi_close(env_, dbi_transaction_db_); + kth_db_dbi_close(env_, dbi_transaction_hash_db_); + kth_db_dbi_close(env_, dbi_history_db_); + kth_db_dbi_close(env_, dbi_spend_db_); + kth_db_dbi_close(env_, dbi_transaction_unconfirmed_db_); + #endif + + db_opened_ = false; + } + + if (env_created_) { + kth_db_env_close(env_); + + env_created_ = false; + } + + return true; +} + +#if ! defined(KTH_DB_READONLY) + + +result_code internal_database_basis::push_genesis(domain::chain::block const& block) { + + KTH_DB_txn* db_txn; + auto res0 = kth_db_txn_begin(env_, NULL, 0, &db_txn); + if (res0 != KTH_DB_SUCCESS) { + return result_code::other; + } + + auto res = push_genesis(block, db_txn); + if ( ! succeed(res)) { + kth_db_txn_abort(db_txn); + return res; + } + + auto res2 = kth_db_txn_commit(db_txn); + if (res2 != KTH_DB_SUCCESS) { + return result_code::other; + } + return res; +} +#endif // ! defined(KTH_DB_READONLY) + +utxo_entry internal_database_basis::get_utxo(domain::chain::output_point const& point, KTH_DB_txn* db_txn) const { + + auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); + auto key = kth_db_make_value(keyarr.size(), keyarr.data()); + KTH_DB_val value; + + auto res0 = kth_db_get(db_txn, dbi_utxo_, &key, &value); + if (res0 != KTH_DB_SUCCESS) { + return utxo_entry{}; + } + + return domain::create(db_value_to_data_chunk(value)); +} + +utxo_entry internal_database_basis::get_utxo(domain::chain::output_point const& point) const { + + KTH_DB_txn* db_txn; + auto res0 = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res0 != KTH_DB_SUCCESS) { + LOG_ERROR(LOG_DATABASE, "Error begining LMDB Transaction [get_utxo] ", res0); + return {}; + } + + auto ret = get_utxo(point, db_txn); + + res0 = kth_db_txn_commit(db_txn); + if (res0 != KTH_DB_SUCCESS) { + LOG_ERROR(LOG_DATABASE, "Error commiting LMDB Transaction [get_utxo] ", res0); + return {}; + } + + return ret; +} + +result_code internal_database_basis::get_last_height(uint32_t& out_height) const { + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return result_code::other; + } + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_block_header_, &cursor) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + return result_code::other; + } + + KTH_DB_val key; + int rc; + if ((rc = kth_db_cursor_get(cursor, &key, nullptr, KTH_DB_LAST)) != KTH_DB_SUCCESS) { + return result_code::db_empty; + } + + // assert kth_db_get_size(key) == 4; + out_height = *static_cast(kth_db_get_data(key)); + + kth_db_cursor_close(cursor); + + // kth_db_txn_abort(db_txn); + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return result_code::other; + } + + return result_code::success; +} + +std::pair internal_database_basis::get_header(hash_digest const& hash) const { + auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); + + KTH_DB_txn* db_txn; + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return {}; + } + + KTH_DB_val value; + if (kth_db_get(db_txn, dbi_block_header_by_hash_, &key, &value) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + // kth_db_txn_abort(db_txn); + return {}; + } + + // assert kth_db_get_size(value) == 4; + auto height = *static_cast(kth_db_get_data(value)); + + auto header = get_header(height, db_txn); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return {}; + } + + return {header, height}; +} + +domain::chain::header internal_database_basis::get_header(uint32_t height) const { + KTH_DB_txn* db_txn; + auto zzz = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (zzz != KTH_DB_SUCCESS) { + return domain::chain::header{}; + } + + auto res = get_header(height, db_txn); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return domain::chain::header{}; + } + + return res; +} + +domain::chain::header::list internal_database_basis::get_headers(uint32_t from, uint32_t to) const { + // precondition: from <= to + domain::chain::header::list list; + + KTH_DB_txn* db_txn; + auto zzz = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (zzz != KTH_DB_SUCCESS) { + return list; + } + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_block_header_, &cursor) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + return list; + } + + auto key = kth_db_make_value(sizeof(from), &from); + + KTH_DB_val value; + int rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_SET); + if (rc != KTH_DB_SUCCESS) { + kth_db_cursor_close(cursor); + kth_db_txn_commit(db_txn); + return list; + } + + auto data = db_value_to_data_chunk(value); + list.push_back(domain::create(data)); + + while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == KTH_DB_SUCCESS) { + auto height = *static_cast(kth_db_get_data(key)); + if (height > to) break; + auto data = db_value_to_data_chunk(value); + list.push_back(domain::create(data)); + } + + kth_db_cursor_close(cursor); + kth_db_txn_commit(db_txn); + return list; +} + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::pop_block(domain::chain::block& out_block) { + uint32_t height; + + //TODO: (Mario) use only one transaction ? + + //TODO: (Mario) add overload with tx + // The blockchain is empty (nothing to pop, not even genesis). + auto res = get_last_height(height); + if (res != result_code::success ) { + return res; + } + + //TODO: (Mario) add overload with tx + // This should never become invalid if this call is protected. + out_block = get_block_reorg(height, dbi_reorg_block_, env_); + if ( ! out_block.is_valid()) { + return result_code::key_not_found; + } + + res = remove_block(out_block, height); + if (res != result_code::success) { + return res; + } + + return result_code::success; +} + +result_code internal_database_basis::prune() { + //TODO: (Mario) add overload with tx + uint32_t last_height; + auto res = get_last_height(last_height); + + if (res == result_code::db_empty) return result_code::no_data_to_prune; + if (res != result_code::success) return res; + if (last_height < reorg_pool_limit_) return result_code::no_data_to_prune; + + uint32_t first_height; + res = get_first_reorg_block_height(first_height, dbi_reorg_block_, env_); + if (res == result_code::db_empty) return result_code::no_data_to_prune; + if (res != result_code::success) return res; + if (first_height > last_height) return result_code::db_corrupt; + + auto reorg_count = last_height - first_height + 1; + if (reorg_count <= reorg_pool_limit_) return result_code::no_data_to_prune; + + auto amount_to_delete = reorg_count - reorg_pool_limit_; + auto remove_until = first_height + amount_to_delete; + + KTH_DB_txn* db_txn; + auto zzz = kth_db_txn_begin(env_, NULL, 0, &db_txn); + if (zzz != KTH_DB_SUCCESS) { + return result_code::other; + } + + res = prune_reorg_block(amount_to_delete, db_txn, dbi_reorg_block_); + + if (res != result_code::success) { + kth_db_txn_abort(db_txn); + return res; + } + + res = prune_reorg_index(remove_until, db_txn, dbi_reorg_pool_, dbi_reorg_index_); + if (res != result_code::success) { + kth_db_txn_abort(db_txn); + return res; + } + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return result_code::other; + } + + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +//TODO(fernando): move to private +//TODO(fernando): rename it +//TODO(fernando): taking KTH_DB_val by value, warning! +result_code internal_database_basis::insert_reorg_into_pool(utxo_pool_t& pool, KTH_DB_val key_point, KTH_DB_txn* db_txn) const { + + KTH_DB_val value; + auto res = kth_db_get(db_txn, dbi_reorg_pool_, &key_point, &value); + if (res == KTH_DB_NOTFOUND) { + LOG_INFO(LOG_DATABASE, "Key not found in reorg pool [insert_reorg_into_pool] ", res); + return result_code::key_not_found; + } + + if (res != KTH_DB_SUCCESS) { + LOG_ERROR(LOG_DATABASE, "Error in reorg pool [insert_reorg_into_pool] ", res); + return result_code::other; + } + + auto entry_data = db_value_to_data_chunk(value); + auto entry = domain::create(entry_data); + + auto point_data = db_value_to_data_chunk(key_point); + auto point = domain::create(point_data, KTH_INTERNAL_DB_WIRE); + pool.insert({point, std::move(entry)}); //TODO(fernando): use emplace? + + return result_code::success; +} + +std::pair internal_database_basis::get_utxo_pool_from(uint32_t from, uint32_t to) const { + // precondition: from <= to + utxo_pool_t pool; + + KTH_DB_txn* db_txn; + auto zzz = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); + if (zzz != KTH_DB_SUCCESS) { + return {result_code::other, pool}; + } + + KTH_DB_cursor* cursor; + if (kth_db_cursor_open(db_txn, dbi_reorg_index_, &cursor) != KTH_DB_SUCCESS) { + kth_db_txn_commit(db_txn); + return {result_code::other, pool}; + } + + auto key = kth_db_make_value(sizeof(from), &from); + KTH_DB_val value; + + // int rc = kth_db_cursor_get(cursor, &key, &value, MDB_SET); + int rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_SET_RANGE); + if (rc != KTH_DB_SUCCESS) { + kth_db_cursor_close(cursor); + kth_db_txn_commit(db_txn); + return {result_code::key_not_found, pool}; + } + + auto current_height = *static_cast(kth_db_get_data(key)); + if (current_height < from) { + kth_db_cursor_close(cursor); + kth_db_txn_commit(db_txn); + return {result_code::other, pool}; + } + // if (current_height > from) { + // kth_db_cursor_close(cursor); + // kth_db_txn_commit(db_txn); + // return {result_code::other, pool}; + // } + if (current_height > to) { + kth_db_cursor_close(cursor); + kth_db_txn_commit(db_txn); + return {result_code::other, pool}; + } + + auto res = insert_reorg_into_pool(pool, value, db_txn); + if (res != result_code::success) { + kth_db_cursor_close(cursor); + kth_db_txn_commit(db_txn); + return {res, pool}; + } + + while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == KTH_DB_SUCCESS) { + current_height = *static_cast(kth_db_get_data(key)); + if (current_height > to) { + kth_db_cursor_close(cursor); + kth_db_txn_commit(db_txn); + return {result_code::other, pool}; + } + + res = insert_reorg_into_pool(pool, value, db_txn); + if (res != result_code::success) { + kth_db_cursor_close(cursor); + kth_db_txn_commit(db_txn); + return {res, pool}; + } + } + + kth_db_cursor_close(cursor); + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return {result_code::other, pool}; + } + + return {result_code::success, pool}; +} + +#if defined(KTH_DB_NEW_FULL) +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::push_transaction_unconfirmed(domain::chain::transaction const& tx, uint32_t height) { + + KTH_DB_txn* db_txn; + if (kth_db_txn_begin(env_, NULL, 0, &db_txn) != KTH_DB_SUCCESS) { + return result_code::other; + } + + auto res = insert_transaction_unconfirmed(tx, height, db_txn); + if (res != result_code::success) { + kth_db_txn_abort(db_txn); + return res; + } + + if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { + return result_code::other; + } + + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) +#endif // defined(KTH_DB_NEW_FULL) + +// Private functions +// ------------------------------------------------------------------------------------------------------ +size_t internal_database_basis::get_db_page_size() const { + return boost::interprocess::mapped_region::get_page_size(); +} + +size_t internal_database_basis::adjust_db_size(size_t size) const { + // precondition: env_ have to be created (kth_db_env_create) + // The kth_db_env_set_mapsize should be a multiple of the OS page size. + size_t const page_size = get_db_page_size(); + auto res = size_t(std::ceil(double(size) / page_size)) * page_size; + return res; + + // size_t const mod = size % page_size; + // return size + (mod != 0) ? (page_size - mod) : 0; +} + +bool internal_database_basis::create_and_open_environment() { + + if (kth_db_env_create(&env_) != KTH_DB_SUCCESS) { + return false; + } + env_created_ = true; + + // TODO(fernando): see what to do with mdb_env_set_maxreaders ---------------------------------------------- + // int threads = tools::get_max_concurrency(); + // if (threads > 110 && /* maxreaders default is 126, leave some slots for other read processes */ + // (result = mdb_env_set_maxreaders(m_env, threads+16))) + // throw0(DB_ERROR(lmdb_error("Failed to set max number of readers: ", result).c_str())); + // ---------------------------------------------------------------------------------------------------------------- + + auto res = kth_db_env_set_mapsize(env_, adjust_db_size(db_max_size_)); + if (res != KTH_DB_SUCCESS) { + LOG_ERROR(LOG_DATABASE, "Error setting max memory map size. Verify do you have enough free space. [create_and_open_environment] ", static_cast(res)); + return false; + } + + res = kth_db_env_set_maxdbs(env_, max_dbs_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + //Note(knuth): fastest flags + //KTH_DB_NORDAHEAD | KTH_DB_NOSYNC | KTH_DB_NOTLS | KTH_DB_WRITEMAP | KTH_DB_MAPASYNC + //for more secure flags use: KTH_DB_NORDAHEAD | KTH_DB_NOSYNC | KTH_DB_NOTLS + + // int mdb_flags = KTH_DB_NORDAHEAD | KTH_DB_NOSYNC | KTH_DB_NOTLS; + // int mdb_flags = KTH_DB_NORDAHEAD; + int mdb_flags = MDB_NORDAHEAD; + +#if defined(KTH_DB_READONLY) + mdb_flags |= KTH_DB_RDONLY; +#else + std::cout << "safe_mode_: " << safe_mode_ << "\n"; + if ( ! safe_mode_) { + // Monero: mdb_flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + // Old Knuth: mdb_flags |= KTH_DB_WRITEMAP | KTH_DB_MAPASYNC; + mdb_flags |= KTH_DB_NOSYNC | KTH_DB_WRITEMAP | KTH_DB_MAPASYNC; + // mdb_flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + } +#endif + std::cout << "mdb_flags: " << mdb_flags << "\n"; + + res = kth_db_env_open(env_, db_dir_.string().c_str(), mdb_flags, env_open_mode_); + return res == KTH_DB_SUCCESS; +} + +/* +bool internal_database_basis::set_fast_flags_environment(bool enabled) { + + if (fast_mode && enabled) { + return true; + } + + if ( ! fast_mode && !enabled) { + return true; + } + + LOG_INFO(LOG_DATABASE, "Setting LMDB Environment Flags. Fast mode: ", (enabled ? "yes" : "no" )); + + //KTH_DB_WRITEMAP | + auto res = mdb_env_set_flags(env_, KTH_DB_MAPASYNC, enabled ? 1 : 0); + if ( res != KTH_DB_SUCCESS ) { + LOG_ERROR(LOG_DATABASE, "Error setting LMDB Environmet flags. [set_fast_flags_environment] ", static_cast(res)); + return false; + } + + fast_mode = enabled; + return true; +} +*/ + +inline +int compare_uint64(KTH_DB_val const* a, KTH_DB_val const* b) { + //TODO(fernando): check this casts... something smells bad + const uint64_t va = *(const uint64_t *)kth_db_get_data(*a); + const uint64_t vb = *(const uint64_t *)kth_db_get_data(*b); + return (va < vb) ? -1 : va > vb; +} + +bool internal_database_basis::open_databases() { + KTH_DB_txn* db_txn; + + auto res = kth_db_txn_begin(env_, NULL, KTH_DB_CONDITIONAL_READONLY, &db_txn); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, block_header_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_block_header_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, block_header_by_hash_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_block_header_by_hash_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, utxo_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_utxo_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, reorg_pool_name, KTH_DB_CONDITIONAL_CREATE, &dbi_reorg_pool_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, reorg_index_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_DUPSORT | KTH_DB_INTEGERKEY | KTH_DB_DUPFIXED, &dbi_reorg_index_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, reorg_block_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_reorg_block_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, db_properties_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_properties_); + if (res != KTH_DB_SUCCESS) { + return false; + } + +#if defined(KTH_DB_NEW_BLOCKS) + res = kth_db_dbi_open(db_txn, block_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_block_db_); + if (res != KTH_DB_SUCCESS) { + return false; + } +#endif + +#if defined(KTH_DB_NEW_FULL) + + res = kth_db_dbi_open(db_txn, block_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_DUPSORT | KTH_DB_INTEGERKEY | KTH_DB_DUPFIXED | MDB_INTEGERDUP, &dbi_block_db_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, transaction_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_transaction_db_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, transaction_hash_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_transaction_hash_db_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, history_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_DUPSORT | KTH_DB_DUPFIXED, &dbi_history_db_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, spend_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_spend_db_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + res = kth_db_dbi_open(db_txn, transaction_unconfirmed_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_transaction_unconfirmed_db_); + if (res != KTH_DB_SUCCESS) { + return false; + } + + mdb_set_dupsort(db_txn, dbi_history_db_, compare_uint64); + +#endif //KTH_DB_NEW_FULL + + db_opened_ = kth_db_txn_commit(db_txn) == KTH_DB_SUCCESS; + return db_opened_; +} + +#if ! defined(KTH_DB_READONLY) + +result_code internal_database_basis::remove_inputs(hash_digest const& tx_id, uint32_t height, domain::chain::input::list const& inputs, bool insert_reorg, KTH_DB_txn* db_txn) { + uint32_t pos = 0; + for (auto const& input: inputs) { + + domain::chain::input_point const inpoint {tx_id, pos}; + auto const& prevout = input.previous_output(); + +#if defined(KTH_DB_NEW_FULL) + auto res = insert_input_history(inpoint, height, input, db_txn); + if (res != result_code::success) { + return res; + } + + //set spender height in tx database + //Note(Knuth): Commented because we don't validate transaction duplicates (BIP-30) + /*res = set_spend(prevout, height, db_txn); + if (res != result_code::success) { + return res; + }*/ + +#else + result_code res; +#endif + + res = remove_utxo(height, prevout, insert_reorg, db_txn, dbi_utxo_, dbi_reorg_pool_, dbi_reorg_index_); + if (res != result_code::success) { + return res; + } + +#if defined(KTH_DB_NEW_FULL) + + //insert in spend database + res = insert_spend(prevout, inpoint, db_txn); + if (res != result_code::success) { + return res; + } + +#endif + + ++pos; + } + return result_code::success; +} + +result_code internal_database_basis::insert_outputs(hash_digest const& tx_id, uint32_t height, domain::chain::output::list const& outputs, data_chunk const& fixed_data, KTH_DB_txn* db_txn) { + uint32_t pos = 0; + for (auto const& output: outputs) { + + // auto res = insert_utxo(domain::chain::point{tx_id, pos}, output, fixed_data, db_txn); + // if (res != result_code::success) { + // return res; + // } + +#if defined(KTH_DB_NEW_FULL) + res = insert_output_history(tx_id, height, pos, output, db_txn); + if (res != result_code::success) { + return res; + } +#endif + + ++pos; + } + return result_code::success; +} + +result_code internal_database_basis::insert_outputs_error_treatment(uint32_t height, data_chunk const& fixed_data, hash_digest const& txid, domain::chain::output::list const& outputs, KTH_DB_txn* db_txn) { + auto res = insert_outputs(txid, height, outputs, fixed_data, db_txn); + + if (res == result_code::duplicated_key) { + //TODO(fernando): log and continue + return result_code::success_duplicate_coinbase; + } + return res; +} + + +void internal_database_basis::print_stats(KTH_DB_txn* db_txn) { + MDB_stat db_stats; + auto ret = mdb_stat(db_txn, dbi_block_header_, &db_stats); + if (ret == KTH_DB_SUCCESS) { + std::cout << "headers count: " << db_stats.ms_entries << std::endl; + } + ret = mdb_stat(db_txn, dbi_block_header_by_hash_, &db_stats); + if (ret == KTH_DB_SUCCESS) { + std::cout << "headers by hash count: " << db_stats.ms_entries << std::endl; + } + ret = mdb_stat(db_txn, dbi_utxo_, &db_stats); + if (ret == KTH_DB_SUCCESS) { + std::cout << "UTXO Set count: " << db_stats.ms_entries << std::endl; + } + ret = mdb_stat(db_txn, dbi_reorg_pool_, &db_stats); + if (ret == KTH_DB_SUCCESS) { + std::cout << "Reorg Pool count: " << db_stats.ms_entries << std::endl; + } + ret = mdb_stat(db_txn, dbi_reorg_index_, &db_stats); + if (ret == KTH_DB_SUCCESS) { + std::cout << "Reorg Pool Index count: " << db_stats.ms_entries << std::endl; + } + ret = mdb_stat(db_txn, dbi_reorg_index_, &db_stats); + if (ret == KTH_DB_SUCCESS) { + std::cout << "Reorg Pool Blocks count: " << db_stats.ms_entries << std::endl; + } +} + + +result_code internal_database_basis::push_block(domain::chain::block const& block, uint32_t height, uint32_t median_time_past, bool insert_reorg, KTH_DB_txn* db_txn) { + //precondition: block.transactions().size() >= 1 + + auto res = push_block_header(block, height, db_txn); + if (res != result_code::success) { + return res; + } + + auto const& txs = block.transactions(); + +#if defined(KTH_DB_NEW_BLOCKS) + res = insert_block(block, height, db_txn); + if (res != result_code::success) { + return res; + } + +#elif defined(KTH_DB_NEW_FULL) + auto tx_count = get_tx_count(db_txn); + + res = insert_block(block, height, tx_count, db_txn); + if (res != result_code::success) { + return res; + } + + res = insert_transactions(txs.begin(), txs.end(), height, median_time_past, tx_count, db_txn); + if (res == result_code::duplicated_key) { + res = result_code::success_duplicate_coinbase; + } else if (res != result_code::success) { + return res; + } +#endif + + if (insert_reorg) { + // std::cout << "insert_reorg: " << insert_reorg << std::endl; + res = push_block_reorg(block, height, db_txn, dbi_reorg_block_); + if (res != result_code::success) { + return res; + } + } + // std::cout << "insert_utxo() -- commented code that kth_db_put" << std::endl; + // std::cout << "UTXO FULL" << std::endl; + // std::cout << "UTXO NO remove inputs" << std::endl; + // return result_code::success; + + auto const& coinbase = txs.front(); + + // auto fixed = utxo_entry::to_data_fixed(height, median_time_past, true); //TODO(fernando): podría estar afuera de la DBTx + // auto res0 = insert_outputs_error_treatment(height, fixed, coinbase.hash(), coinbase.outputs(), db_txn); //TODO(fernando): tx.hash() debe ser llamado fuera de la DBTx + // if ( ! succeed(res0)) { + // return res0; + // } + + // fixed.back() = 0; //The last byte equal to 0 means NonCoinbaseTx + // res = push_transactions_non_coinbase(height, fixed, txs.begin() + 1, txs.end(), insert_reorg, db_txn); + // if (res != result_code::success) { + // return res; + // } + // if (res == result_code::success_duplicate_coinbase) { + // return res; + // } + // return res0; + + return result_code::success; +} + +result_code internal_database_basis::push_genesis(domain::chain::block const& block, KTH_DB_txn* db_txn) { + auto res = push_block_header(block, 0, db_txn); + if (res != result_code::success) { + return res; + } + +#if defined(KTH_DB_NEW_BLOCKS) + res = insert_block(block, 0, db_txn); +#elif defined(KTH_DB_NEW_FULL) + auto tx_count = get_tx_count(db_txn); + res = insert_block(block, 0, tx_count, db_txn); + + if (res != result_code::success) { + return res; + } + + auto const& txs = block.transactions(); + auto const& coinbase = txs.front(); + auto const& hash = coinbase.hash(); + auto const median_time_past = block.header().validation.median_time_past; + + res = insert_transaction(tx_count, coinbase, 0, median_time_past, 0, db_txn); + if (res != result_code::success && res != result_code::duplicated_key) { + return res; + } + + res = insert_output_history(hash, 0, 0, coinbase.outputs()[0], db_txn); + if (res != result_code::success) { + return res; + } + +#endif + + return res; +} + +result_code internal_database_basis::remove_outputs(hash_digest const& txid, domain::chain::output::list const& outputs, KTH_DB_txn* db_txn) { + uint32_t pos = outputs.size() - 1; + for (auto const& output: outputs) { + domain::chain::output_point const point {txid, pos}; + auto res = remove_utxo(0, point, false, db_txn, dbi_utxo_, dbi_reorg_pool_, dbi_reorg_index_); + if (res != result_code::success) { + return res; + } + --pos; + } + return result_code::success; +} + +result_code internal_database_basis::insert_inputs(domain::chain::input::list const& inputs, KTH_DB_txn* db_txn) { + for (auto const& input: inputs) { + auto const& point = input.previous_output(); + + auto res = insert_output_from_reorg_and_remove(point, db_txn, dbi_utxo_, dbi_reorg_pool_); + if (res != result_code::success) { + return res; + } + } + return result_code::success; +} + +result_code internal_database_basis::remove_block(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { + //precondition: block.transactions().size() >= 1 + + auto const& txs = block.transactions(); + auto const& coinbase = txs.front(); + + //UTXO + auto res = remove_transactions_non_coinbase(txs.begin() + 1, txs.end(), db_txn); + if (res != result_code::success) { + return res; + } + + //UTXO Coinbase + //TODO(fernando): tx.hash() debe ser llamado fuera de la DBTx + res = remove_outputs(coinbase.hash(), coinbase.outputs(), db_txn); + if (res != result_code::success) { + return res; + } + + //TODO(fernando): tx.hash() debe ser llamado fuera de la DBTx + res = remove_block_header(block.hash(), height, db_txn); + if (res != result_code::success) { + return res; + } + + res = remove_block_reorg(height, db_txn, dbi_reorg_block_); + if (res != result_code::success) { + return res; + } + + res = remove_reorg_index(height, db_txn, dbi_reorg_index_); + if (res != result_code::success && res != result_code::key_not_found) { + return res; + } + +#if defined(KTH_DB_NEW_FULL) + //Transaction Database + res = remove_transactions(block, height, db_txn); + if (res != result_code::success) { + return res; + } +#endif //defined(KTH_DB_NEW_FULL) + +#if defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) + res = remove_blocks_db(height, db_txn); + if (res != result_code::success) { + return res; + } +#endif //defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) + + return result_code::success; +} + +result_code internal_database_basis::remove_block(domain::chain::block const& block, uint32_t height) { + KTH_DB_txn* db_txn; + auto res0 = kth_db_txn_begin(env_, NULL, 0, &db_txn); + if (res0 != KTH_DB_SUCCESS) { + return result_code::other; + } + + auto res = remove_block(block, height, db_txn); + if (res != result_code::success) { + kth_db_txn_abort(db_txn); + return res; + } + + auto res2 = kth_db_txn_commit(db_txn); + if (res2 != KTH_DB_SUCCESS) { + return result_code::other; + } + return result_code::success; +} + +#endif // ! defined(KTH_DB_READONLY) + +} // namespace kth::database From 39208416d8353194492150fa84c1f4df084af207 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:09:31 -0300 Subject: [PATCH 22/26] move the code from implementation file to translation unit --- .../database/databases/internal_database.ipp | 1136 +---------------- 1 file changed, 37 insertions(+), 1099 deletions(-) diff --git a/include/kth/database/databases/internal_database.ipp b/include/kth/database/databases/internal_database.ipp index ca423eda..72e14d15 100644 --- a/include/kth/database/databases/internal_database.ipp +++ b/include/kth/database/databases/internal_database.ipp @@ -7,923 +7,71 @@ #include +#include +#include + namespace kth::database { using utxo_pool_t = std::unordered_map; -template -internal_database_basis::internal_database_basis(path const& db_dir, uint32_t reorg_pool_limit, uint64_t db_max_size, bool safe_mode) - : db_dir_(db_dir) - , reorg_pool_limit_(reorg_pool_limit) - , limit_(blocks_to_seconds(reorg_pool_limit)) - , db_max_size_(db_max_size) - , safe_mode_(safe_mode) -{} - -template -internal_database_basis::~internal_database_basis() { - close(); -} - -#if ! defined(KTH_DB_READONLY) - -template -bool internal_database_basis::create() { - std::error_code ec; - - if ( ! std::filesystem::create_directories(db_dir_, ec)) { - if (ec.value() == directory_exists) { - LOG_ERROR(LOG_DATABASE, "Failed because the directory ", db_dir_.string(), " already exists."); - return false; - } - - LOG_ERROR(LOG_DATABASE, "Failed to create directory ", db_dir_.string(), " with error, '", ec.message(), "'."); - return false; - } - - auto ret = open_internal(); - if ( ! ret ) { - return false; - } - - ret = create_db_mode_property(); - if ( ! ret ) { - return false; - } - - return true; -} - -template -bool internal_database_basis::create_db_mode_property() { - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, 0, &db_txn); - if (res != KTH_DB_SUCCESS) { - return false; - } - - db_mode_code db_mode_; - -#if defined(KTH_DB_NEW_FULL) - db_mode_ = db_mode_code::db_new_full; -#elif defined(KTH_DB_NEW_BLOCKS) - db_mode_ = db_mode_code::db_new_with_blocks; -#else - db_mode_ = db_mode_code::db_new; -#endif - - property_code property_code_ = property_code::db_mode; - - auto key = kth_db_make_value(sizeof(property_code_), &property_code_); - auto value = kth_db_make_value(sizeof(db_mode_), &db_mode_); - - res = kth_db_put(db_txn, dbi_properties_, &key, &value, KTH_DB_NOOVERWRITE); - if (res != KTH_DB_SUCCESS) { - LOG_ERROR(LOG_DATABASE, "Failed saving in DB Properties [create_db_mode_property] ", static_cast(res)); - kth_db_txn_abort(db_txn); - return false; - } - - res = kth_db_txn_commit(db_txn); - if (res != KTH_DB_SUCCESS) { - return false; - } - - return true; -} - -#endif // ! defined(KTH_DB_READONLY) - - -template -bool internal_database_basis::open() { - - auto ret = open_internal(); - if ( ! ret ) { - return false; - } - - ret = verify_db_mode_property(); - if ( ! ret ) { - return false; - } - - return true; -} - - -template -bool internal_database_basis::open_internal() { - - if ( ! create_and_open_environment()) { - LOG_ERROR(LOG_DATABASE, "Error configuring LMDB environment."); - return false; - } - - return open_databases(); -} - -template -bool internal_database_basis::verify_db_mode_property() const { - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return false; - } - - property_code property_code_ = property_code::db_mode; - - auto key = kth_db_make_value(sizeof(property_code_), &property_code_); - KTH_DB_val value; - - res = kth_db_get(db_txn, dbi_properties_, &key, &value); - if (res != KTH_DB_SUCCESS) { - LOG_ERROR(LOG_DATABASE, "Failed getting DB Properties [verify_db_mode_property] ", static_cast(res)); - kth_db_txn_abort(db_txn); - return false; - } - - auto db_mode_ = *static_cast(kth_db_get_data(value)); - - res = kth_db_txn_commit(db_txn); - if (res != KTH_DB_SUCCESS) { - return false; - } - -#if defined(KTH_DB_NEW_FULL) - auto db_mode_node_ = db_mode_code::db_new_full; -#elif defined(KTH_DB_NEW_BLOCKS) - auto db_mode_node_ = db_mode_code::db_new_with_blocks; -#else - auto db_mode_node_ = db_mode_code::db_new; -#endif - - if (db_mode_ != db_mode_node_) { - LOG_ERROR(LOG_DATABASE, "Error validating DB Mode, the node is compiled for another DB mode. Node DB Mode: " - , static_cast(db_mode_node_) - , ", Actual DB Mode: " - , static_cast(db_mode_)); - return false; - } - - return true; -} - -template -bool internal_database_basis::close() { - if (db_opened_) { - - //TODO(fernando): check sync - //Force synchronous flush (use with KTH_DB_NOSYNC or MDB_NOMETASYNC, with other flags do nothing) -#if defined(KTH_USE_LIBMDBX) - kth_db_env_sync(env_); -#else - kth_db_env_sync(env_, true); -#endif - kth_db_dbi_close(env_, dbi_block_header_); - kth_db_dbi_close(env_, dbi_block_header_by_hash_); - kth_db_dbi_close(env_, dbi_utxo_); - kth_db_dbi_close(env_, dbi_reorg_pool_); - kth_db_dbi_close(env_, dbi_reorg_index_); - kth_db_dbi_close(env_, dbi_reorg_block_); - kth_db_dbi_close(env_, dbi_properties_); - - #if defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) - kth_db_dbi_close(env_, dbi_block_db_); - #endif - - #if defined(KTH_DB_NEW_FULL) - kth_db_dbi_close(env_, dbi_transaction_db_); - kth_db_dbi_close(env_, dbi_transaction_hash_db_); - kth_db_dbi_close(env_, dbi_history_db_); - kth_db_dbi_close(env_, dbi_spend_db_); - kth_db_dbi_close(env_, dbi_transaction_unconfirmed_db_); - #endif - - db_opened_ = false; - } - - if (env_created_) { - kth_db_env_close(env_); - - env_created_ = false; - } - - return true; -} #if ! defined(KTH_DB_READONLY) -template -result_code internal_database_basis::push_genesis(domain::chain::block const& block) { - - KTH_DB_txn* db_txn; - auto res0 = kth_db_txn_begin(env_, NULL, 0, &db_txn); - if (res0 != KTH_DB_SUCCESS) { - return result_code::other; - } - - auto res = push_genesis(block, db_txn); - if ( ! succeed(res)) { - kth_db_txn_abort(db_txn); - return res; - } - - auto res2 = kth_db_txn_commit(db_txn); - if (res2 != KTH_DB_SUCCESS) { - return result_code::other; - } - return res; -} - //TODO(fernando): optimization: consider passing a list of outputs to insert and a list of inputs to delete instead of an entire Block. // avoiding inserting and erasing internal spenders - -template -result_code internal_database_basis::push_block(domain::chain::block const& block, uint32_t height, uint32_t median_time_past) { - - KTH_DB_txn* db_txn; - auto res0 = kth_db_txn_begin(env_, NULL, 0, &db_txn); - if (res0 != KTH_DB_SUCCESS) { - LOG_ERROR(LOG_DATABASE, "Error begining LMDB Transaction [push_block] ", res0); - return result_code::other; - } - - //TODO: save reorg blocks after the last checkpoint - auto res = push_block(block, height, median_time_past, ! is_old_block(block), db_txn); - if ( ! succeed(res)) { - kth_db_txn_abort(db_txn); - return res; - } - - auto res2 = kth_db_txn_commit(db_txn); - if (res2 != KTH_DB_SUCCESS) { - LOG_ERROR(LOG_DATABASE, "Error commiting LMDB Transaction [push_block] ", res2); - return result_code::other; - } - - return res; -} - -#endif // ! defined(KTH_DB_READONLY) - - template -utxo_entry internal_database_basis::get_utxo(domain::chain::output_point const& point, KTH_DB_txn* db_txn) const { - - auto keyarr = point.to_data(KTH_INTERNAL_DB_WIRE); - auto key = kth_db_make_value(keyarr.size(), keyarr.data()); - KTH_DB_val value; - - auto res0 = kth_db_get(db_txn, dbi_utxo_, &key, &value); - if (res0 != KTH_DB_SUCCESS) { - return utxo_entry{}; - } - - return domain::create(db_value_to_data_chunk(value)); -} - -template -utxo_entry internal_database_basis::get_utxo(domain::chain::output_point const& point) const { - - KTH_DB_txn* db_txn; - auto res0 = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res0 != KTH_DB_SUCCESS) { - LOG_ERROR(LOG_DATABASE, "Error begining LMDB Transaction [get_utxo] ", res0); - return {}; - } - - auto ret = get_utxo(point, db_txn); - - res0 = kth_db_txn_commit(db_txn); - if (res0 != KTH_DB_SUCCESS) { - LOG_ERROR(LOG_DATABASE, "Error commiting LMDB Transaction [get_utxo] ", res0); - return {}; - } - - return ret; -} - -template -result_code internal_database_basis::get_last_height(uint32_t& out_height) const { - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return result_code::other; - } - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_block_header_, &cursor) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - return result_code::other; - } - - KTH_DB_val key; - int rc; - if ((rc = kth_db_cursor_get(cursor, &key, nullptr, KTH_DB_LAST)) != KTH_DB_SUCCESS) { - return result_code::db_empty; - } - - // assert kth_db_get_size(key) == 4; - out_height = *static_cast(kth_db_get_data(key)); - - kth_db_cursor_close(cursor); +result_code internal_database_basis::push_block(domain::chain::block const& block, uint32_t height, uint32_t median_time_past) { - // kth_db_txn_abort(db_txn); - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return result_code::other; - } - - return result_code::success; -} - -template -std::pair internal_database_basis::get_header(hash_digest const& hash) const { - auto key = kth_db_make_value(hash.size(), const_cast(hash).data()); - - KTH_DB_txn* db_txn; - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return {}; - } - - KTH_DB_val value; - if (kth_db_get(db_txn, dbi_block_header_by_hash_, &key, &value) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - // kth_db_txn_abort(db_txn); - return {}; - } - - // assert kth_db_get_size(value) == 4; - auto height = *static_cast(kth_db_get_data(value)); - - auto header = get_header(height, db_txn); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return {}; - } - - return {header, height}; -} - -template -domain::chain::header internal_database_basis::get_header(uint32_t height) const { - KTH_DB_txn* db_txn; - auto zzz = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (zzz != KTH_DB_SUCCESS) { - return domain::chain::header{}; - } - - auto res = get_header(height, db_txn); - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return domain::chain::header{}; - } - - return res; -} - -template -domain::chain::header::list internal_database_basis::get_headers(uint32_t from, uint32_t to) const { - // precondition: from <= to - domain::chain::header::list list; - - KTH_DB_txn* db_txn; - auto zzz = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (zzz != KTH_DB_SUCCESS) { - return list; - } - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_block_header_, &cursor) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - return list; - } - - auto key = kth_db_make_value(sizeof(from), &from); - - KTH_DB_val value; - int rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_SET); - if (rc != KTH_DB_SUCCESS) { - kth_db_cursor_close(cursor); - kth_db_txn_commit(db_txn); - return list; - } - - auto data = db_value_to_data_chunk(value); - list.push_back(domain::create(data)); - - while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == KTH_DB_SUCCESS) { - auto height = *static_cast(kth_db_get_data(key)); - if (height > to) break; - auto data = db_value_to_data_chunk(value); - list.push_back(domain::create(data)); - } - - kth_db_cursor_close(cursor); - kth_db_txn_commit(db_txn); - return list; -} - -#if ! defined(KTH_DB_READONLY) - -template -result_code internal_database_basis::pop_block(domain::chain::block& out_block) { - uint32_t height; - - //TODO: (Mario) use only one transaction ? - - //TODO: (Mario) add overload with tx - // The blockchain is empty (nothing to pop, not even genesis). - auto res = get_last_height(height); - if (res != result_code::success ) { - return res; - } - - //TODO: (Mario) add overload with tx - // This should never become invalid if this call is protected. - out_block = get_block_reorg(height); - if ( ! out_block.is_valid()) { - return result_code::key_not_found; - } - - res = remove_block(out_block, height); - if (res != result_code::success) { - return res; - } - - return result_code::success; -} - -template -result_code internal_database_basis::prune() { - //TODO: (Mario) add overload with tx - uint32_t last_height; - auto res = get_last_height(last_height); - - if (res == result_code::db_empty) return result_code::no_data_to_prune; - if (res != result_code::success) return res; - if (last_height < reorg_pool_limit_) return result_code::no_data_to_prune; - - uint32_t first_height; - res = get_first_reorg_block_height(first_height); - if (res == result_code::db_empty) return result_code::no_data_to_prune; - if (res != result_code::success) return res; - if (first_height > last_height) return result_code::db_corrupt; - - auto reorg_count = last_height - first_height + 1; - if (reorg_count <= reorg_pool_limit_) return result_code::no_data_to_prune; - - auto amount_to_delete = reorg_count - reorg_pool_limit_; - auto remove_until = first_height + amount_to_delete; - - KTH_DB_txn* db_txn; - auto zzz = kth_db_txn_begin(env_, NULL, 0, &db_txn); - if (zzz != KTH_DB_SUCCESS) { - return result_code::other; - } - - res = prune_reorg_block(amount_to_delete, db_txn); - if (res != result_code::success) { - kth_db_txn_abort(db_txn); - return res; - } - - res = prune_reorg_index(remove_until, db_txn); - if (res != result_code::success) { - kth_db_txn_abort(db_txn); - return res; - } - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return result_code::other; - } - - return result_code::success; -} - -#endif // ! defined(KTH_DB_READONLY) - -//TODO(fernando): move to private -//TODO(fernando): rename it -//TODO(fernando): taking KTH_DB_val by value, warning! -template -result_code internal_database_basis::insert_reorg_into_pool(utxo_pool_t& pool, KTH_DB_val key_point, KTH_DB_txn* db_txn) const { - - KTH_DB_val value; - auto res = kth_db_get(db_txn, dbi_reorg_pool_, &key_point, &value); - if (res == KTH_DB_NOTFOUND) { - LOG_INFO(LOG_DATABASE, "Key not found in reorg pool [insert_reorg_into_pool] ", res); - return result_code::key_not_found; - } - - if (res != KTH_DB_SUCCESS) { - LOG_ERROR(LOG_DATABASE, "Error in reorg pool [insert_reorg_into_pool] ", res); - return result_code::other; - } - - auto entry_data = db_value_to_data_chunk(value); - auto entry = domain::create(entry_data); - - auto point_data = db_value_to_data_chunk(key_point); - auto point = domain::create(point_data, KTH_INTERNAL_DB_WIRE); - pool.insert({point, std::move(entry)}); //TODO(fernando): use emplace? - - return result_code::success; -} - -template -std::pair internal_database_basis::get_utxo_pool_from(uint32_t from, uint32_t to) const { - // precondition: from <= to - utxo_pool_t pool; - - KTH_DB_txn* db_txn; - auto zzz = kth_db_txn_begin(env_, NULL, KTH_DB_RDONLY, &db_txn); - if (zzz != KTH_DB_SUCCESS) { - return {result_code::other, pool}; - } - - KTH_DB_cursor* cursor; - if (kth_db_cursor_open(db_txn, dbi_reorg_index_, &cursor) != KTH_DB_SUCCESS) { - kth_db_txn_commit(db_txn); - return {result_code::other, pool}; - } - - auto key = kth_db_make_value(sizeof(from), &from); - KTH_DB_val value; - - // int rc = kth_db_cursor_get(cursor, &key, &value, MDB_SET); - int rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_SET_RANGE); - if (rc != KTH_DB_SUCCESS) { - kth_db_cursor_close(cursor); - kth_db_txn_commit(db_txn); - return {result_code::key_not_found, pool}; - } - - auto current_height = *static_cast(kth_db_get_data(key)); - if (current_height < from) { - kth_db_cursor_close(cursor); - kth_db_txn_commit(db_txn); - return {result_code::other, pool}; - } - // if (current_height > from) { - // kth_db_cursor_close(cursor); - // kth_db_txn_commit(db_txn); - // return {result_code::other, pool}; + // KTH_DB_txn* db_txn; + // auto res0 = kth_db_txn_begin(env_, NULL, 0, &db_txn); + // if (res0 != KTH_DB_SUCCESS) { + // LOG_ERROR(LOG_DATABASE, "Error begining LMDB Transaction [push_block] ", res0); + // return result_code::other; // } - if (current_height > to) { - kth_db_cursor_close(cursor); - kth_db_txn_commit(db_txn); - return {result_code::other, pool}; - } - - auto res = insert_reorg_into_pool(pool, value, db_txn); - if (res != result_code::success) { - kth_db_cursor_close(cursor); - kth_db_txn_commit(db_txn); - return {res, pool}; - } - - while ((rc = kth_db_cursor_get(cursor, &key, &value, KTH_DB_NEXT)) == KTH_DB_SUCCESS) { - current_height = *static_cast(kth_db_get_data(key)); - if (current_height > to) { - kth_db_cursor_close(cursor); - kth_db_txn_commit(db_txn); - return {result_code::other, pool}; - } - - res = insert_reorg_into_pool(pool, value, db_txn); - if (res != result_code::success) { - kth_db_cursor_close(cursor); - kth_db_txn_commit(db_txn); - return {res, pool}; - } - } - - kth_db_cursor_close(cursor); - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return {result_code::other, pool}; - } + // // print_stats(db_txn); - return {result_code::success, pool}; -} + // //TODO: save reorg blocks after the last checkpoint + // auto res = push_block(block, height, median_time_past, ! is_old_block(block), db_txn); + // if ( ! succeed(res)) { + // kth_db_txn_abort(db_txn); + // return res; + // } -#if defined(KTH_DB_NEW_FULL) -#if ! defined(KTH_DB_READONLY) + // // print_stats(db_txn); -template -result_code internal_database_basis::push_transaction_unconfirmed(domain::chain::transaction const& tx, uint32_t height) { + // auto res2 = kth_db_txn_commit(db_txn); + // if (res2 != KTH_DB_SUCCESS) { + // LOG_ERROR(LOG_DATABASE, "Error commiting LMDB Transaction [push_block] ", res2); + // return result_code::other; + // } - KTH_DB_txn* db_txn; - if (kth_db_txn_begin(env_, NULL, 0, &db_txn) != KTH_DB_SUCCESS) { - return result_code::other; - } - - auto res = insert_transaction_unconfirmed(tx, height, db_txn); - if (res != result_code::success) { - kth_db_txn_abort(db_txn); - return res; - } - - if (kth_db_txn_commit(db_txn) != KTH_DB_SUCCESS) { - return result_code::other; - } + // return res; return result_code::success; } #endif // ! defined(KTH_DB_READONLY) -#endif // defined(KTH_DB_NEW_FULL) // Private functions // ------------------------------------------------------------------------------------------------------ - template -bool internal_database_basis::is_old_block(domain::chain::block const& block) const { +bool internal_database_basis::is_old_block(domain::chain::block const& block) const { return is_old_block_(block, limit_); } -template -size_t internal_database_basis::get_db_page_size() const { - return boost::interprocess::mapped_region::get_page_size(); -} - -template -size_t internal_database_basis::adjust_db_size(size_t size) const { - // precondition: env_ have to be created (kth_db_env_create) - // The kth_db_env_set_mapsize should be a multiple of the OS page size. - size_t const page_size = get_db_page_size(); - auto res = size_t(std::ceil(double(size) / page_size)) * page_size; - return res; - - // size_t const mod = size % page_size; - // return size + (mod != 0) ? (page_size - mod) : 0; -} - - -template -bool internal_database_basis::create_and_open_environment() { - - if (kth_db_env_create(&env_) != KTH_DB_SUCCESS) { - return false; - } - env_created_ = true; - - // TODO(fernando): see what to do with mdb_env_set_maxreaders ---------------------------------------------- - // int threads = tools::get_max_concurrency(); - // if (threads > 110 && /* maxreaders default is 126, leave some slots for other read processes */ - // (result = mdb_env_set_maxreaders(m_env, threads+16))) - // throw0(DB_ERROR(lmdb_error("Failed to set max number of readers: ", result).c_str())); - // ---------------------------------------------------------------------------------------------------------------- - - auto res = kth_db_env_set_mapsize(env_, adjust_db_size(db_max_size_)); - if (res != KTH_DB_SUCCESS) { - LOG_ERROR(LOG_DATABASE, "Error setting max memory map size. Verify do you have enough free space. [create_and_open_environment] ", static_cast(res)); - return false; - } - - res = kth_db_env_set_maxdbs(env_, max_dbs_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - //Note(knuth): fastest flags - //KTH_DB_NORDAHEAD | KTH_DB_NOSYNC | KTH_DB_NOTLS | KTH_DB_WRITEMAP | KTH_DB_MAPASYNC - //for more secure flags use: KTH_DB_NORDAHEAD | KTH_DB_NOSYNC | KTH_DB_NOTLS - - int mdb_flags = KTH_DB_NORDAHEAD | KTH_DB_NOSYNC | KTH_DB_NOTLS; - - -#if defined(KTH_DB_READONLY) - mdb_flags |= KTH_DB_RDONLY; -#endif - - if ( ! safe_mode_) { - mdb_flags |= KTH_DB_WRITEMAP | KTH_DB_MAPASYNC; - } - - res = kth_db_env_open(env_, db_dir_.string().c_str(), mdb_flags, env_open_mode_); - return res == KTH_DB_SUCCESS; -} - -/* -template -bool internal_database_basis::set_fast_flags_environment(bool enabled) { - - if (fast_mode && enabled) { - return true; - } - - if ( ! fast_mode && !enabled) { - return true; - } - - LOG_INFO(LOG_DATABASE, "Setting LMDB Environment Flags. Fast mode: ", (enabled ? "yes" : "no" )); - - //KTH_DB_WRITEMAP | - auto res = mdb_env_set_flags(env_, KTH_DB_MAPASYNC, enabled ? 1 : 0); - if ( res != KTH_DB_SUCCESS ) { - LOG_ERROR(LOG_DATABASE, "Error setting LMDB Environmet flags. [set_fast_flags_environment] ", static_cast(res)); - return false; - } - - fast_mode = enabled; - return true; -} -*/ - -inline -int compare_uint64(KTH_DB_val const* a, KTH_DB_val const* b) { - - //TODO(fernando): check this casts... something smells bad - const uint64_t va = *(const uint64_t *)kth_db_get_data(*a); - const uint64_t vb = *(const uint64_t *)kth_db_get_data(*b); - - //std::cout << "va: " << va << std::endl; - //std::cout << "vb: " << va << std::endl; - - return (va < vb) ? -1 : va > vb; -} - -template -bool internal_database_basis::open_databases() { - KTH_DB_txn* db_txn; - - auto res = kth_db_txn_begin(env_, NULL, KTH_DB_CONDITIONAL_READONLY, &db_txn); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, block_header_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_block_header_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, block_header_by_hash_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_block_header_by_hash_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, utxo_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_utxo_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, reorg_pool_name, KTH_DB_CONDITIONAL_CREATE, &dbi_reorg_pool_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, reorg_index_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_DUPSORT | KTH_DB_INTEGERKEY | KTH_DB_DUPFIXED, &dbi_reorg_index_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, reorg_block_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_reorg_block_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, db_properties_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_properties_); - if (res != KTH_DB_SUCCESS) { - return false; - } - -#if defined(KTH_DB_NEW_BLOCKS) - res = kth_db_dbi_open(db_txn, block_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_block_db_); - if (res != KTH_DB_SUCCESS) { - return false; - } -#endif - -#if defined(KTH_DB_NEW_FULL) - - res = kth_db_dbi_open(db_txn, block_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_DUPSORT | KTH_DB_INTEGERKEY | KTH_DB_DUPFIXED | MDB_INTEGERDUP, &dbi_block_db_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, transaction_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_INTEGERKEY, &dbi_transaction_db_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, transaction_hash_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_transaction_hash_db_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, history_db_name, KTH_DB_CONDITIONAL_CREATE | KTH_DB_DUPSORT | KTH_DB_DUPFIXED, &dbi_history_db_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, spend_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_spend_db_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - res = kth_db_dbi_open(db_txn, transaction_unconfirmed_db_name, KTH_DB_CONDITIONAL_CREATE, &dbi_transaction_unconfirmed_db_); - if (res != KTH_DB_SUCCESS) { - return false; - } - - mdb_set_dupsort(db_txn, dbi_history_db_, compare_uint64); - -#endif //KTH_DB_NEW_FULL - - db_opened_ = kth_db_txn_commit(db_txn) == KTH_DB_SUCCESS; - return db_opened_; -} +// inline +// int compare_uint64(KTH_DB_val const* a, KTH_DB_val const* b) { +// //TODO(fernando): check this casts... something smells bad +// const uint64_t va = *(const uint64_t *)kth_db_get_data(*a); +// const uint64_t vb = *(const uint64_t *)kth_db_get_data(*b); +// return (va < vb) ? -1 : va > vb; +// } #if ! defined(KTH_DB_READONLY) -template -result_code internal_database_basis::remove_inputs(hash_digest const& tx_id, uint32_t height, domain::chain::input::list const& inputs, bool insert_reorg, KTH_DB_txn* db_txn) { - uint32_t pos = 0; - for (auto const& input: inputs) { - - domain::chain::input_point const inpoint {tx_id, pos}; - auto const& prevout = input.previous_output(); - -#if defined(KTH_DB_NEW_FULL) - auto res = insert_input_history(inpoint, height, input, db_txn); - if (res != result_code::success) { - return res; - } - - //set spender height in tx database - //Note(Knuth): Commented because we don't validate transaction duplicates (BIP-30) - /*res = set_spend(prevout, height, db_txn); - if (res != result_code::success) { - return res; - }*/ - -#else - result_code res; -#endif - - res = remove_utxo(height, prevout, insert_reorg, db_txn); - if (res != result_code::success) { - return res; - } - -#if defined(KTH_DB_NEW_FULL) - - //insert in spend database - res = insert_spend(prevout, inpoint, db_txn); - if (res != result_code::success) { - return res; - } - -#endif - - ++pos; - } - return result_code::success; -} - -template -result_code internal_database_basis::insert_outputs(hash_digest const& tx_id, uint32_t height, domain::chain::output::list const& outputs, data_chunk const& fixed_data, KTH_DB_txn* db_txn) { - uint32_t pos = 0; - for (auto const& output: outputs) { - - auto res = insert_utxo(domain::chain::point{tx_id, pos}, output, fixed_data, db_txn); - if (res != result_code::success) { - return res; - } - - #if defined(KTH_DB_NEW_FULL) - - res = insert_output_history(tx_id, height, pos, output, db_txn); - if (res != result_code::success) { - return res; - } - - #endif - - ++pos; - } - return result_code::success; -} - -template -result_code internal_database_basis::insert_outputs_error_treatment(uint32_t height, data_chunk const& fixed_data, hash_digest const& txid, domain::chain::output::list const& outputs, KTH_DB_txn* db_txn) { - auto res = insert_outputs(txid,height, outputs, fixed_data, db_txn); - - if (res == result_code::duplicated_key) { - //TODO(fernando): log and continue - return result_code::success_duplicate_coinbase; - } - return res; -} - -template template -result_code internal_database_basis::push_transactions_outputs_non_coinbase(uint32_t height, data_chunk const& fixed_data, I f, I l, KTH_DB_txn* db_txn) { +result_code internal_database_basis::push_transactions_outputs_non_coinbase(uint32_t height, data_chunk const& fixed_data, I f, I l, KTH_DB_txn* db_txn) { // precondition: [f, l) is a valid range and there are no coinbase transactions in it. while (f != l) { @@ -937,9 +85,8 @@ result_code internal_database_basis::push_transactions_outputs_non_coinba return result_code::success; } -template template -result_code internal_database_basis::remove_transactions_inputs_non_coinbase(uint32_t height, I f, I l, bool insert_reorg, KTH_DB_txn* db_txn) { +result_code internal_database_basis::remove_transactions_inputs_non_coinbase(uint32_t height, I f, I l, bool insert_reorg, KTH_DB_txn* db_txn) { while (f != l) { auto const& tx = *f; auto res = remove_inputs(tx.hash(), height, tx.inputs(), insert_reorg, db_txn); @@ -951,150 +98,19 @@ result_code internal_database_basis::remove_transactions_inputs_non_coinb return result_code::success; } -template template -result_code internal_database_basis::push_transactions_non_coinbase(uint32_t height, data_chunk const& fixed_data, I f, I l, bool insert_reorg, KTH_DB_txn* db_txn) { +result_code internal_database_basis::push_transactions_non_coinbase(uint32_t height, data_chunk const& fixed_data, I f, I l, bool insert_reorg, KTH_DB_txn* db_txn) { // precondition: [f, l) is a valid range and there are no coinbase transactions in it. auto res = push_transactions_outputs_non_coinbase(height, fixed_data, f, l, db_txn); if (res != result_code::success) { return res; } - return remove_transactions_inputs_non_coinbase(height, f, l, insert_reorg, db_txn); } -template -result_code internal_database_basis::push_block(domain::chain::block const& block, uint32_t height, uint32_t median_time_past, bool insert_reorg, KTH_DB_txn* db_txn) { - //precondition: block.transactions().size() >= 1 - - auto res = push_block_header(block, height, db_txn); - if (res != result_code::success) { - return res; - } - - auto const& txs = block.transactions(); - -#if defined(KTH_DB_NEW_BLOCKS) - res = insert_block(block, height, db_txn); - if (res != result_code::success) { - // std::cout << "22222222222222" << static_cast(res) << "\n"; - return res; - } - -#elif defined(KTH_DB_NEW_FULL) - - auto tx_count = get_tx_count(db_txn); - - res = insert_block(block, height, tx_count, db_txn); - if (res != result_code::success) { - // std::cout << "22222222222222" << static_cast(res) << "\n"; - return res; - } - - res = insert_transactions(txs.begin(), txs.end(), height, median_time_past, tx_count, db_txn); - if (res == result_code::duplicated_key) { - res = result_code::success_duplicate_coinbase; - } else if (res != result_code::success) { - return res; - } - -#endif - - if ( insert_reorg ) { - res = push_block_reorg(block, height, db_txn); - if (res != result_code::success) { - return res; - } - } - - auto const& coinbase = txs.front(); - - auto fixed = utxo_entry::to_data_fixed(height, median_time_past, true); //TODO(fernando): podría estar afuera de la DBTx - auto res0 = insert_outputs_error_treatment(height, fixed, coinbase.hash(), coinbase.outputs(), db_txn); //TODO(fernando): tx.hash() debe ser llamado fuera de la DBTx - if ( ! succeed(res0)) { - return res0; - } - - fixed.back() = 0; //The last byte equal to 0 means NonCoinbaseTx - res = push_transactions_non_coinbase(height, fixed, txs.begin() + 1, txs.end(), insert_reorg, db_txn); - if (res != result_code::success) { - return res; - } - - if (res == result_code::success_duplicate_coinbase) - return res; - - return res0; -} - -template -result_code internal_database_basis::push_genesis(domain::chain::block const& block, KTH_DB_txn* db_txn) { - auto res = push_block_header(block, 0, db_txn); - if (res != result_code::success) { - return res; - } - -#if defined(KTH_DB_NEW_BLOCKS) - res = insert_block(block, 0, db_txn); -#elif defined(KTH_DB_NEW_FULL) - auto tx_count = get_tx_count(db_txn); - res = insert_block(block, 0, tx_count, db_txn); - - if (res != result_code::success) { - return res; - } - - auto const& txs = block.transactions(); - auto const& coinbase = txs.front(); - auto const& hash = coinbase.hash(); - auto const median_time_past = block.header().validation.median_time_past; - - res = insert_transaction(tx_count, coinbase, 0, median_time_past, 0, db_txn); - if (res != result_code::success && res != result_code::duplicated_key) { - return res; - } - - res = insert_output_history(hash, 0, 0, coinbase.outputs()[0], db_txn); - if (res != result_code::success) { - return res; - } - -#endif - - return res; -} - -template -result_code internal_database_basis::remove_outputs(hash_digest const& txid, domain::chain::output::list const& outputs, KTH_DB_txn* db_txn) { - uint32_t pos = outputs.size() - 1; - for (auto const& output: outputs) { - domain::chain::output_point const point {txid, pos}; - auto res = remove_utxo(0, point, false, db_txn); - if (res != result_code::success) { - return res; - } - --pos; - } - return result_code::success; -} - -template -result_code internal_database_basis::insert_inputs(domain::chain::input::list const& inputs, KTH_DB_txn* db_txn) { - for (auto const& input: inputs) { - auto const& point = input.previous_output(); - - auto res = insert_output_from_reorg_and_remove(point, db_txn); - if (res != result_code::success) { - return res; - } - } - return result_code::success; -} - -template template -result_code internal_database_basis::insert_transactions_inputs_non_coinbase(I f, I l, KTH_DB_txn* db_txn) { +result_code internal_database_basis::insert_transactions_inputs_non_coinbase(I f, I l, KTH_DB_txn* db_txn) { // precondition: [f, l) is a valid range and there are no coinbase transactions in it. while (f != l) { @@ -1109,9 +125,8 @@ result_code internal_database_basis::insert_transactions_inputs_non_coinb return result_code::success; } -template template -result_code internal_database_basis::remove_transactions_outputs_non_coinbase(I f, I l, KTH_DB_txn* db_txn) { +result_code internal_database_basis::remove_transactions_outputs_non_coinbase(I f, I l, KTH_DB_txn* db_txn) { // precondition: [f, l) is a valid range and there are no coinbase transactions in it. while (f != l) { @@ -1126,9 +141,8 @@ result_code internal_database_basis::remove_transactions_outputs_non_coin return result_code::success; } -template template -result_code internal_database_basis::remove_transactions_non_coinbase(I f, I l, KTH_DB_txn* db_txn) { +result_code internal_database_basis::remove_transactions_non_coinbase(I f, I l, KTH_DB_txn* db_txn) { // precondition: [f, l) is a valid range and there are no coinbase transactions in it. auto res = insert_transactions_inputs_non_coinbase(f, l, db_txn); @@ -1138,82 +152,6 @@ result_code internal_database_basis::remove_transactions_non_coinbase(I f return remove_transactions_outputs_non_coinbase(f, l, db_txn); } - -template -result_code internal_database_basis::remove_block(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { - //precondition: block.transactions().size() >= 1 - - auto const& txs = block.transactions(); - auto const& coinbase = txs.front(); - - //UTXO - auto res = remove_transactions_non_coinbase(txs.begin() + 1, txs.end(), db_txn); - if (res != result_code::success) { - return res; - } - - //UTXO Coinbase - //TODO(fernando): tx.hash() debe ser llamado fuera de la DBTx - res = remove_outputs(coinbase.hash(), coinbase.outputs(), db_txn); - if (res != result_code::success) { - return res; - } - - //TODO(fernando): tx.hash() debe ser llamado fuera de la DBTx - res = remove_block_header(block.hash(), height, db_txn); - if (res != result_code::success) { - return res; - } - - res = remove_block_reorg(height, db_txn); - if (res != result_code::success) { - return res; - } - - res = remove_reorg_index(height, db_txn); - if (res != result_code::success && res != result_code::key_not_found) { - return res; - } - -#if defined(KTH_DB_NEW_FULL) - //Transaction Database - res = remove_transactions(block, height, db_txn); - if (res != result_code::success) { - return res; - } -#endif //defined(KTH_DB_NEW_FULL) - -#if defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) - res = remove_blocks_db(height, db_txn); - if (res != result_code::success) { - return res; - } -#endif //defined(KTH_DB_NEW_BLOCKS) || defined(KTH_DB_NEW_FULL) - - return result_code::success; -} - -template -result_code internal_database_basis::remove_block(domain::chain::block const& block, uint32_t height) { - KTH_DB_txn* db_txn; - auto res0 = kth_db_txn_begin(env_, NULL, 0, &db_txn); - if (res0 != KTH_DB_SUCCESS) { - return result_code::other; - } - - auto res = remove_block(block, height, db_txn); - if (res != result_code::success) { - kth_db_txn_abort(db_txn); - return res; - } - - auto res2 = kth_db_txn_commit(db_txn); - if (res2 != KTH_DB_SUCCESS) { - return result_code::other; - } - return result_code::success; -} - #endif // ! defined(KTH_DB_READONLY) } // namespace kth::database From 7dfc3bc50ec2cb31aa163b905576b89f93ece42c Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:09:56 -0300 Subject: [PATCH 23/26] refactor database class --- src/data_base.cpp | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/src/data_base.cpp b/src/data_base.cpp index b67b83af..a5429ec9 100644 --- a/src/data_base.cpp +++ b/src/data_base.cpp @@ -552,7 +552,7 @@ code data_base::insert(domain::chain::block const& block, size_t height) { if (ec) return ec; #ifdef KTH_DB_NEW - auto res = internal_db_->push_block(block, height, median_time_past); + auto res = internal_db_->push_block(block, height, median_time_past); if ( ! succeed(res)) { return error::operation_failed_1; //TODO(fernando): create a new operation_failed } @@ -673,7 +673,7 @@ code data_base::push(block const& block, size_t height) { auto const median_time_past = block.header().validation.median_time_past; #ifdef KTH_DB_NEW - auto res = internal_db_->push_block(block, height, median_time_past); + auto res = internal_db_->push_block(block, height, median_time_past); if ( ! succeed(res)) { return error::operation_failed_6; //TODO(fernando): create a new operation_failed } @@ -1228,13 +1228,28 @@ void data_base::push_next(code const& ec, block_const_ptr_list_const_ptr blocks, void data_base::do_push(block_const_ptr block, size_t height, uint32_t median_time_past, dispatcher& dispatch, result_handler handler) { + // if (block->transactions().size() >= 10000) { + // std::cout << block->transactions().size() << std::endl; + // } + + // auto do_push_start = asio::steady_clock::now(); + #ifdef KTH_DB_NEW // LOG_DEBUG(LOG_DATABASE, "Write flushed to disk: ", ec.message()); - auto res = internal_db_->push_block(*block, height, median_time_past); + auto res = internal_db_->push_block(*block, height, median_time_past); if ( ! succeed(res)) { handler(error::operation_failed_7); //TODO(fernando): create a new operation_failed return; } + + // auto do_push_end = asio::steady_clock::now(); + // auto ns = std::chrono::duration_cast(do_push_end - do_push_start).count(); + + // if (block->transactions().size() >= 10000) { + // std::cout << block->transactions().size() << std::endl; + // std::cout << "ns: " << ns << " - seconds: " << ((double)ns / 1000000000) << std::endl; + // } + block->validation.end_push = asio::steady_clock::now(); // This is the end of the block sub-sequence. handler(error::success); @@ -1302,14 +1317,21 @@ void data_base::pop_above(block_const_ptr_list_ptr out_blocks, hash_digest const auto const header_result = internal_db_->get_header(fork_hash); uint32_t top; + // // The fork point does not exist or failed to get it or the top, fail. + // if ( ! header_result.first.is_valid() || internal_db_->get_last_height(top) != result_code::success) { + // //**--** + // handler(error::operation_failed_9); + // return; + // } + // auto const fork = header_result.second; + // The fork point does not exist or failed to get it or the top, fail. - if ( ! header_result.first.is_valid() || internal_db_->get_last_height(top) != result_code::success) { + if ( internal_db_->get_last_height(top) != result_code::success) { //**--** handler(error::operation_failed_9); return; } - - auto const fork = header_result.second; + auto const fork = top; #endif From 892cae48c5db713c4216504583cb8b7420f8fd5f Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:10:10 -0300 Subject: [PATCH 24/26] add header file --- .../kth/database/databases/reorg_database.hpp | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 include/kth/database/databases/reorg_database.hpp diff --git a/include/kth/database/databases/reorg_database.hpp b/include/kth/database/databases/reorg_database.hpp new file mode 100644 index 00000000..3b514280 --- /dev/null +++ b/include/kth/database/databases/reorg_database.hpp @@ -0,0 +1,45 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef KTH_DATABASE_REORG_DATABASE_HPP_ +#define KTH_DATABASE_REORG_DATABASE_HPP_ + +#include +// #include +#include + +#include +#include + + +namespace kth::database { + +#if ! defined(KTH_DB_READONLY) + +result_code insert_reorg_pool(uint32_t height, KTH_DB_val& key, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_utxo, KTH_DB_dbi dbi_reorg_pool, KTH_DB_dbi dbi_reorg_index); + +//TODO : remove this database in db_new_with_blocks and db_new_full +result_code push_block_reorg(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_block); + +result_code insert_output_from_reorg_and_remove(domain::chain::output_point const& point, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_utxo, KTH_DB_dbi dbi_reorg_pool); + +result_code remove_block_reorg(uint32_t height, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_block); + +result_code remove_reorg_index(uint32_t height, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_index); + +result_code prune_reorg_index(uint32_t remove_until, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_pool, KTH_DB_dbi dbi_reorg_index); + +result_code prune_reorg_block(uint32_t amount_to_delete, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_block); + +#endif // ! defined(KTH_DB_READONLY) + +domain::chain::block get_block_reorg(uint32_t height, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_reorg_block) /*const*/; + +domain::chain::block get_block_reorg(uint32_t height, KTH_DB_dbi dbi_reorg_block, KTH_DB_env* env) /*const*/; + +result_code get_first_reorg_block_height(uint32_t& out_height, KTH_DB_dbi dbi_reorg_block, KTH_DB_env* env); /*const*/; + +} // namespace kth::database + +#endif // KTH_DATABASE_REORG_DATABASE_HPP_ From 3dcbf66f0178dc2667db962859e9d8eb72176a55 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Mon, 21 Dec 2020 12:10:20 -0300 Subject: [PATCH 25/26] add UTXO set header file --- .../kth/database/databases/utxo_database.hpp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 include/kth/database/databases/utxo_database.hpp diff --git a/include/kth/database/databases/utxo_database.hpp b/include/kth/database/databases/utxo_database.hpp new file mode 100644 index 00000000..3f9dff03 --- /dev/null +++ b/include/kth/database/databases/utxo_database.hpp @@ -0,0 +1,27 @@ +// Copyright (c) 2016-2020 Knuth Project developers. +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef KTH_DATABASE_UTXO_DATABASE_HPP_ +#define KTH_DATABASE_UTXO_DATABASE_HPP_ + +#include +// #include +#include + +#include + + +namespace kth::database { + +#if ! defined(KTH_DB_READONLY) + +result_code remove_utxo(uint32_t height, domain::chain::output_point const& point, bool insert_reorg, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_utxo, KTH_DB_dbi dbi_reorg_pool, KTH_DB_dbi dbi_reorg_index); + +result_code insert_utxo(domain::chain::output_point const& point, domain::chain::output const& output, data_chunk const& fixed_data, KTH_DB_txn* db_txn, KTH_DB_dbi dbi_utxo); + +#endif // ! defined(KTH_DB_READONLY) + +} // namespace kth::database + +#endif // KTH_DATABASE_UTXO_DATABASE_HPP_ From c99de22854fedbc5febe4712b5a8f1430b9f13c6 Mon Sep 17 00:00:00 2001 From: Fernando Pelliccioni Date: Sun, 14 Feb 2021 19:43:11 -0300 Subject: [PATCH 26/26] db tests --- .../database/databases/internal_database.ipp | 43 ++++++++-------- src/databases/header_database.cpp | 50 +++++++++---------- src/databases/internal_database.cpp | 4 +- test/data_base.cpp | 2 +- 4 files changed, 48 insertions(+), 51 deletions(-) diff --git a/include/kth/database/databases/internal_database.ipp b/include/kth/database/databases/internal_database.ipp index 72e14d15..72ae08b3 100644 --- a/include/kth/database/databases/internal_database.ipp +++ b/include/kth/database/databases/internal_database.ipp @@ -22,33 +22,32 @@ using utxo_pool_t = std::unordered_map; template result_code internal_database_basis::push_block(domain::chain::block const& block, uint32_t height, uint32_t median_time_past) { - // KTH_DB_txn* db_txn; - // auto res0 = kth_db_txn_begin(env_, NULL, 0, &db_txn); - // if (res0 != KTH_DB_SUCCESS) { - // LOG_ERROR(LOG_DATABASE, "Error begining LMDB Transaction [push_block] ", res0); - // return result_code::other; - // } - - // // print_stats(db_txn); + KTH_DB_txn* db_txn; + auto res0 = kth_db_txn_begin(env_, NULL, 0, &db_txn); + if (res0 != KTH_DB_SUCCESS) { + LOG_ERROR(LOG_DATABASE, "Error begining LMDB Transaction [push_block] ", res0); + return result_code::other; + } - // //TODO: save reorg blocks after the last checkpoint - // auto res = push_block(block, height, median_time_past, ! is_old_block(block), db_txn); - // if ( ! succeed(res)) { - // kth_db_txn_abort(db_txn); - // return res; - // } + // print_stats(db_txn); - // // print_stats(db_txn); + //TODO: save reorg blocks after the last checkpoint + auto res = push_block(block, height, median_time_past, ! is_old_block(block), db_txn); + if ( ! succeed(res)) { + kth_db_txn_abort(db_txn); + return res; + } - // auto res2 = kth_db_txn_commit(db_txn); - // if (res2 != KTH_DB_SUCCESS) { - // LOG_ERROR(LOG_DATABASE, "Error commiting LMDB Transaction [push_block] ", res2); - // return result_code::other; - // } + // print_stats(db_txn); - // return res; + auto res2 = kth_db_txn_commit(db_txn); + if (res2 != KTH_DB_SUCCESS) { + LOG_ERROR(LOG_DATABASE, "Error commiting LMDB Transaction [push_block] ", res2); + return result_code::other; + } - return result_code::success; + return res; + // return result_code::success; } #endif // ! defined(KTH_DB_READONLY) diff --git a/src/databases/header_database.cpp b/src/databases/header_database.cpp index 25b00bbf..8519a8eb 100644 --- a/src/databases/header_database.cpp +++ b/src/databases/header_database.cpp @@ -9,33 +9,33 @@ namespace kth::database { #if ! defined(KTH_DB_READONLY) result_code internal_database_basis::push_block_header(domain::chain::block const& block, uint32_t height, KTH_DB_txn* db_txn) { - // auto valuearr = block.header().to_data(true); //TODO(fernando): podría estar afuera de la DBTx - // auto key = kth_db_make_value(sizeof(height), &height); - // auto value = kth_db_make_value(valuearr.size(), valuearr.data()); + auto valuearr = block.header().to_data(true); //TODO(fernando): podría estar afuera de la DBTx + auto key = kth_db_make_value(sizeof(height), &height); + auto value = kth_db_make_value(valuearr.size(), valuearr.data()); - // auto res = kth_db_put(db_txn, dbi_block_header_, &key, &value, KTH_DB_APPEND); - // if (res == KTH_DB_KEYEXIST) { - // //TODO(fernando): El logging en general no está bueno que esté en la DbTx. - // LOG_INFO(LOG_DATABASE, "Duplicate key inserting block header [push_block_header] ", res); //TODO(fernando): podría estar afuera de la DBTx. - // return result_code::duplicated_key; - // } - // if (res != KTH_DB_SUCCESS) { - // LOG_INFO(LOG_DATABASE, "Error inserting block header [push_block_header] ", res); - // return result_code::other; - // } + auto res = kth_db_put(db_txn, dbi_block_header_, &key, &value, KTH_DB_APPEND); + if (res == KTH_DB_KEYEXIST) { + //TODO(fernando): El logging en general no está bueno que esté en la DbTx. + LOG_INFO(LOG_DATABASE, "Duplicate key inserting block header [push_block_header] ", res); //TODO(fernando): podría estar afuera de la DBTx. + return result_code::duplicated_key; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error inserting block header [push_block_header] ", res); + return result_code::other; + } - // auto key_by_hash_arr = block.hash(); //TODO(fernando): podría estar afuera de la DBTx - // auto key_by_hash = kth_db_make_value(key_by_hash_arr.size(), key_by_hash_arr.data()); - - // res = kth_db_put(db_txn, dbi_block_header_by_hash_, &key_by_hash, &key, KTH_DB_NOOVERWRITE); - // if (res == KTH_DB_KEYEXIST) { - // LOG_INFO(LOG_DATABASE, "Duplicate key inserting block header by hash [push_block_header] ", res); - // return result_code::duplicated_key; - // } - // if (res != KTH_DB_SUCCESS) { - // LOG_INFO(LOG_DATABASE, "Error inserting block header by hash [push_block_header] ", res); - // return result_code::other; - // } + auto key_by_hash_arr = block.hash(); //TODO(fernando): podría estar afuera de la DBTx + auto key_by_hash = kth_db_make_value(key_by_hash_arr.size(), key_by_hash_arr.data()); + + res = kth_db_put(db_txn, dbi_block_header_by_hash_, &key_by_hash, &key, KTH_DB_NOOVERWRITE); + if (res == KTH_DB_KEYEXIST) { + LOG_INFO(LOG_DATABASE, "Duplicate key inserting block header by hash [push_block_header] ", res); + return result_code::duplicated_key; + } + if (res != KTH_DB_SUCCESS) { + LOG_INFO(LOG_DATABASE, "Error inserting block header by hash [push_block_header] ", res); + return result_code::other; + } return result_code::success; } diff --git a/src/databases/internal_database.cpp b/src/databases/internal_database.cpp index 2b5ef55e..c2ab2871 100644 --- a/src/databases/internal_database.cpp +++ b/src/databases/internal_database.cpp @@ -851,7 +851,6 @@ result_code internal_database_basis::insert_outputs_error_treatment(uint32_t hei return res; } - void internal_database_basis::print_stats(KTH_DB_txn* db_txn) { MDB_stat db_stats; auto ret = mdb_stat(db_txn, dbi_block_header_, &db_stats); @@ -880,7 +879,6 @@ void internal_database_basis::print_stats(KTH_DB_txn* db_txn) { } } - result_code internal_database_basis::push_block(domain::chain::block const& block, uint32_t height, uint32_t median_time_past, bool insert_reorg, KTH_DB_txn* db_txn) { //precondition: block.transactions().size() >= 1 @@ -914,7 +912,7 @@ result_code internal_database_basis::push_block(domain::chain::block const& bloc #endif if (insert_reorg) { - // std::cout << "insert_reorg: " << insert_reorg << std::endl; + std::cout << "insert_reorg: " << insert_reorg << std::endl; res = push_block_reorg(block, height, db_txn, dbi_reorg_block_); if (res != result_code::success) { return res; diff --git a/test/data_base.cpp b/test/data_base.cpp index 10d5d0a3..1a7d0359 100644 --- a/test/data_base.cpp +++ b/test/data_base.cpp @@ -340,7 +340,7 @@ TEST_CASE("data base pushpop test", "[None]") { std::cout << "push_all blocks #2 & #3" << std::endl; auto const block2_ptr = std::make_shared(read_block(MAINNET_BLOCK2)); auto const block3_ptr = std::make_shared(read_block(MAINNET_BLOCK3)); - auto const blocks_push_ptr = std::make_shared(block_const_ptr_list{ block2_ptr, block3_ptr }); + auto const blocks_push_ptr = std::make_shared(block_const_ptr_list{ block2_ptr, block3_ptr }); test_block_not_exists(instance, *block2_ptr, indexed); test_block_not_exists(instance, *block3_ptr, indexed); REQUIRE(push_all_result(instance == blocks_push_ptr, 2, dispatch), error::success);