From 28288df1527161bcbc2e2b69a9b577dc8084a566 Mon Sep 17 00:00:00 2001 From: Aliaksandr Kantsevoi Date: Mon, 6 Jan 2025 07:37:45 +0100 Subject: [PATCH 1/5] a distilled example of a problem to solve --- Example1.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 Example1.md diff --git a/Example1.md b/Example1.md new file mode 100644 index 0000000..850806b --- /dev/null +++ b/Example1.md @@ -0,0 +1,54 @@ +Example of usage where we combing marroon-daemon operations and oltp operations + +```python + +# migrates users and accounts from one db to another while maintaining OLTP queries + +maroon-daemoon { + for u in storeMDB.users { + storePSQL.users.create(u, phantom: true) + while storeMDB.accounts.countForUser(u.uid) > 0 { + let accounts = storeMDB.accounts.get(u.uid, 10); + # + # some possible transformations here... + # + storePSQL.accounts.write(accounts); + } + storePSQL.users.update(u.uid, phantom: false); + } +} + +# oltp operations have higher priority than maroon-daemoon operations +oltp { + # combinedStore - universal address space on top of 2 storages + # keeps track of the object versions, knows where we have the updated version and queries the correct DB + + combinedStore = { + storePSQL + storeMDB + } + + # "O(1) operation" - meaning we have limited amount of operations that will be finished "quickly" + # transactional guarantees + # + def payment(srcAcc: string, dstAcc: string, amount: int) { + let tx = combinedStore.txStart(); + + if combinedStore.accounts[srcAcc].amount <= amount { + return InsufficientFunds{...} + } + + combinedStore.accounts[srcAcc].amount -= amount; + combinedStore.accounts[dstAcc].amount += amount; + combinedStore.transfers.create(Transfer{ + From: srcAcc, + To: dstAcc, + Amount: amount, + Time: now(), + }); + + tx.commit(); + } +} + +``` From 1687e723b9c9e4f2cf64a1b96a233a4f96dad329 Mon Sep 17 00:00:00 2001 From: Aliaksandr Kantsevoi Date: Mon, 6 Jan 2025 07:41:46 +0100 Subject: [PATCH 2/5] Few explanations --- Example1.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Example1.md b/Example1.md index 850806b..6654791 100644 --- a/Example1.md +++ b/Example1.md @@ -1,4 +1,6 @@ -Example of usage where we combing marroon-daemon operations and oltp operations +Example of usage where we are combining marroon-daemon operations and oltp operations. +Since we're changing the data that is being migrated our engine keeps track of object's versions, keeps track of what has been migrated and what's not and where exactly the update should be written and committed. + ```python From b469c9d3c5e5578213c5de3d5d60d2b88459ce1f Mon Sep 17 00:00:00 2001 From: Aliaksandr Kantsevoi Date: Tue, 7 Jan 2025 13:19:32 +0100 Subject: [PATCH 3/5] a little bit more explain --- Example1.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Example1.md b/Example1.md index 6654791..cc92b2b 100644 --- a/Example1.md +++ b/Example1.md @@ -54,3 +54,15 @@ oltp { } ``` + + +So, what happens in essense: +- we have long term process and short term process +- we can support consistent state only inside "short O(1)" operations +- but we need to support long-term scenarious as well +- in order to support these long-term scenarious we should be able to automatically slice our long-term scenarious into a sequence of correct/pausable "short O(1)" operations (are we talking about CRDT? again) +- then we just execute these operations with different priorities + - ex: maroon operations have lower priority, and when we finish yet another cycle iteration we check if we have something with the higher priority -> for example "oltp request" and if we have - we execute higher priority thing, if not - continue maroon execution + +Challenge: +- how to make sure that our slicing of operations will be ended up in the correct sequence of CRDT operations? From 902d947fd067391dae4fa71c0b80709708cefb57 Mon Sep 17 00:00:00 2001 From: Aliaksandr Kantsevoi Date: Tue, 7 Jan 2025 16:33:52 +0100 Subject: [PATCH 4/5] fuzzy view --- Example1.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Example1.md b/Example1.md index cc92b2b..296df3b 100644 --- a/Example1.md +++ b/Example1.md @@ -53,6 +53,42 @@ oltp { } } + +# +# my fuzzy view on how it might look like +# + +type Meta { + version int64 + hash string + + # the information about which storage hold the up to date version of this object. The source of truth for that specific object + storage string +} + +type Object { + Body # actual content + Meta +} + +impl combinedStore { + storages: [](storage, options) + objects: Map + + fn get(table: string, id: string) -> Object { + # table == "Accounts" for example + + let m = objects[id]; + return storages.get(m.storage, table, "id == {id}") + } + + fn writeNewVersion(obj: Object) { + obj.meta.version++ + obj.meta.hash = hash(obj.body) + storages.get(obj.storage).Update(table, obj) # unconditional write? since we control the sequence + } +} + ``` From fc4ec6c842760a08bf49c16e859fd01ee4553c60 Mon Sep 17 00:00:00 2001 From: Aliaksandr Kantsevoi Date: Tue, 7 Jan 2025 16:52:31 +0100 Subject: [PATCH 5/5] more thoughts --- Example1.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/Example1.md b/Example1.md index 296df3b..97e9551 100644 --- a/Example1.md +++ b/Example1.md @@ -4,6 +4,7 @@ Since we're changing the data that is being migrated our engine keeps track of o ```python + # migrates users and accounts from one db to another while maintaining OLTP queries maroon-daemoon { @@ -58,6 +59,11 @@ oltp { # my fuzzy view on how it might look like # +GLOBAL_SCHEMA_DEFINITION { + TARGET_STORAGE_FOR_TRANSFERS = storePSQL + TARGET_STORAGE_FOR_ACCOUNTS = storePSQL +} + type Meta { version int64 hash string @@ -89,6 +95,75 @@ impl combinedStore { } } + +# +# view on how payment function might be transpiled/unwrapped/translated +# + +def payment(srcAcc: string, dstAcc: string, amount: int) { + # let tx = combinedStore.txStart(); + # + ## !translated into:! ? TODO: think about what tx start/end here might be? + + # if combinedStore.accounts[srcAcc].amount <= amount { + # return InsufficientFunds{...} + # } + # + ## !translated into:! + + # keep in mind that in that case it doesn't matter in which storage `src` and `dst` objects are located. Can be the same can be different + let src = combinedStore.get("Accounts", srcAcc); + let dst = combinedStore.get("Accounts", dstAcc); + + if src.Body.amount <= amount { return InsufficientFunds{...} } + + + # combinedStore.accounts[srcAcc].amount -= amount; + # combinedStore.accounts[dstAcc].amount += amount; + # + ## !translated into:! + + src.Body.amount -= amount; + dst.Body.amount += amount; + + # combinedStore.transfers.create(Transfer{ + # From: srcAcc, + # To: dstAcc, + # Amount: amount, + # Time: now(), + # }); + # + ## !translated into:! + + combinedStore.writeNewVersion(Object{ + # ignore meta here as not full as other fields will be set by the function + Meta{ + storage: TARGET_STORAGE_FOR_TRANSFERS.DEFAULT_STORAGE_FOR_TRANSFERS, + } + Body{ + From: srcAcc, + To: dstAcc, + Amount: amount, + Time: now(), + } + }) + + # tx.commit(); + # + ## !translated into:! + if src.Meta.storage != GLOBAL_SCHEMA_DEFINITION.TARGET_STORAGE_FOR_ACCOUNTS { + src.Meta.storage = GLOBAL_SCHEMA_DEFINITION.TARGET_STORAGE_FOR_ACCOUNTS + } + if dst.Meta.storage != GLOBAL_SCHEMA_DEFINITION.TARGET_STORAGE_FOR_ACCOUNTS { + dst.Meta.storage = GLOBAL_SCHEMA_DEFINITION.TARGET_STORAGE_FOR_ACCOUNTS + } + combinedStore.writeNewVersion(src); + combinedStore.writeNewVersion(dst); + + + # TODO: movement should happen here somehow, How can I move the object from one to another store? + } + ```