From 0fca897622d83cc312b04b07b7ec9559888609be Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Thu, 26 Nov 2020 02:23:37 +0300 Subject: [PATCH 01/40] add shumaich proto --- rebar.config | 1 + rebar.lock | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/rebar.config b/rebar.config index 3b2d246aa..eadc1e3a8 100644 --- a/rebar.config +++ b/rebar.config @@ -43,6 +43,7 @@ {payproc_errors, {git, "git@github.com:rbkmoney/payproc-errors-erlang.git", {branch, "master"}}}, {mg_proto , {git, "git@github.com:rbkmoney/machinegun_proto.git" , {branch, "master"}}}, {shumpune_proto, {git, "git@github.com:rbkmoney/shumpune-proto.git" , {branch, "master"}}}, + {shumaich_proto, {git, "git@github.com:rbkmoney/shumaich-proto.git" , {branch, "master"}}}, {dmt_client , {git, "git@github.com:rbkmoney/dmt_client.git" , {branch, "master"}}}, {scoper , {git, "git@github.com:rbkmoney/scoper.git" , {branch, "master"}}}, {party_client , {git, "git@github.com:rbkmoney/party_client_erlang.git" , {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index ca71ccc33..583f3643b 100644 --- a/rebar.lock +++ b/rebar.lock @@ -72,6 +72,10 @@ {git,"git@github.com:rbkmoney/scoper.git", {ref,"23b1625bf2c6940a56cfc30389472a5e384a229f"}}, 0}, + {<<"shumaich_proto">>, + {git,"git@github.com:rbkmoney/shumaich-proto.git", + {ref,"be2c835aff55c90927df45d6fb85597b16cff747"}}, + 0}, {<<"shumpune_proto">>, {git,"git@github.com:rbkmoney/shumpune-proto.git", {ref,"a0aed3bdce6aafdb832bbcde45e6278222b08c0b"}}, From b65b2f1e910f2637d7a2c1fdad913220cfd613bd Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Thu, 26 Nov 2020 02:24:25 +0300 Subject: [PATCH 02/40] add shumaich proto to app.src --- apps/hellgate/src/hellgate.app.src | 1 + apps/party_management/src/party_management.app.src | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/hellgate/src/hellgate.app.src b/apps/hellgate/src/hellgate.app.src index aef909409..592e2f831 100644 --- a/apps/hellgate/src/hellgate.app.src +++ b/apps/hellgate/src/hellgate.app.src @@ -12,6 +12,7 @@ fault_detector_proto, hg_proto, shumpune_proto, + shumaich_proto, cowboy, how_are_you, % must be after ranch and before any woody usage woody, diff --git a/apps/party_management/src/party_management.app.src b/apps/party_management/src/party_management.app.src index 39679d000..d6704d146 100644 --- a/apps/party_management/src/party_management.app.src +++ b/apps/party_management/src/party_management.app.src @@ -11,6 +11,7 @@ genlib, pm_proto, shumpune_proto, + shumaich_proto, cowboy, how_are_you, % must be after ranch and before any woody usage woody, From 49da102f81005def4624ca36deadabbf6dde9211 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Thu, 26 Nov 2020 02:24:41 +0300 Subject: [PATCH 03/40] update compose file --- docker-compose.sh | 153 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 130 insertions(+), 23 deletions(-) diff --git a/docker-compose.sh b/docker-compose.sh index 7edb4c832..c05ad2ad3 100755 --- a/docker-compose.sh +++ b/docker-compose.sh @@ -1,6 +1,6 @@ #!/bin/bash cat < Date: Thu, 26 Nov 2020 02:29:30 +0300 Subject: [PATCH 04/40] add bender client --- rebar.config | 1 + rebar.lock | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/rebar.config b/rebar.config index eadc1e3a8..c6f6c6e63 100644 --- a/rebar.config +++ b/rebar.config @@ -45,6 +45,7 @@ {shumpune_proto, {git, "git@github.com:rbkmoney/shumpune-proto.git" , {branch, "master"}}}, {shumaich_proto, {git, "git@github.com:rbkmoney/shumaich-proto.git" , {branch, "master"}}}, {dmt_client , {git, "git@github.com:rbkmoney/dmt_client.git" , {branch, "master"}}}, + {bender_client , {git, "git@github.com:rbkmoney/bender_client_erlang.git" , {branch, "master"}}}, {scoper , {git, "git@github.com:rbkmoney/scoper.git" , {branch, "master"}}}, {party_client , {git, "git@github.com:rbkmoney/party_client_erlang.git" , {branch, "master"}}}, {how_are_you , {git, "https://github.com/rbkmoney/how_are_you.git" , {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index 583f3643b..b5790fed6 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,6 +1,14 @@ {"1.1.0", [{<<"accept">>,{pkg,<<"accept">>,<<"0.3.5">>},2}, {<<"bear">>,{pkg,<<"bear">>,<<"0.8.7">>},2}, + {<<"bender_client">>, + {git,"git@github.com:rbkmoney/bender_client_erlang.git", + {ref,"3c1489a397dacd1e613b777834ab511023afad36"}}, + 0}, + {<<"bender_proto">>, + {git,"git@github.com:rbkmoney/bender-proto.git", + {ref,"0d5813b8a25c8d03e4e59e42aa5f4e9b785a3849"}}, + 1}, {<<"cache">>,{pkg,<<"cache">>,<<"2.3.2">>},0}, {<<"certifi">>,{pkg,<<"certifi">>,<<"2.5.1">>},2}, {<<"cg_mon">>, @@ -55,6 +63,10 @@ {ref,"ebae56fe2b3e79e4eb34afc8cb55c9012ae989f8"}}, 0}, {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},2}, + {<<"msgpack_proto">>, + {git,"git@github.com:rbkmoney/msgpack-proto.git", + {ref,"ec15d5e854ea60c58467373077d90c2faf6273d8"}}, + 2}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.0">>},3}, {<<"party_client">>, {git,"git@github.com:rbkmoney/party_client_erlang.git", From bd7f022359c617e1a32353648d00eb8b03104e27 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Thu, 26 Nov 2020 02:30:38 +0300 Subject: [PATCH 05/40] add bender client to app.src --- apps/hellgate/src/hellgate.app.src | 1 + apps/party_management/src/party_management.app.src | 1 + 2 files changed, 2 insertions(+) diff --git a/apps/hellgate/src/hellgate.app.src b/apps/hellgate/src/hellgate.app.src index 592e2f831..bbbef4380 100644 --- a/apps/hellgate/src/hellgate.app.src +++ b/apps/hellgate/src/hellgate.app.src @@ -20,6 +20,7 @@ gproc, dmt_client, party_client, + bender_client, woody_user_identity, payproc_errors, erl_health, diff --git a/apps/party_management/src/party_management.app.src b/apps/party_management/src/party_management.app.src index d6704d146..a61eac9de 100644 --- a/apps/party_management/src/party_management.app.src +++ b/apps/party_management/src/party_management.app.src @@ -21,6 +21,7 @@ woody_user_identity, payproc_errors, erl_health, + bender_client, cache ]}, {env, []}, From bca20aaa4683ef023dafcc284e6bf8294f3c9f17 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Thu, 26 Nov 2020 02:31:00 +0300 Subject: [PATCH 06/40] add gen_sequence to utils --- apps/hellgate/src/hg_utils.erl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/apps/hellgate/src/hg_utils.erl b/apps/hellgate/src/hg_utils.erl index 435c055e2..011eac1fe 100644 --- a/apps/hellgate/src/hg_utils.erl +++ b/apps/hellgate/src/hg_utils.erl @@ -13,6 +13,16 @@ -export([format_reason/1]). +-export([gen_sequence/2]). +-export([gen_sequence/3]). + +-define(APP, bender_client). + +-type sequence_params() :: #{minimum => integer()}. +-type woody_context() :: woody_context:ctx(). + +-include_lib("bender_proto/include/bender_thrift.hrl"). + %% -spec unique_id() -> dmsl_base_thrift:'ID'(). @@ -89,3 +99,13 @@ unwrap_result({error, E}) -> %% TODO: fix this dirty hack format_reason(V) -> genlib:to_binary(V). + +-spec gen_sequence(binary(), woody_context()) -> integer(). +gen_sequence(SequenceID, WoodyContext) -> + gen_sequence(SequenceID, WoodyContext, #{}). + +-spec gen_sequence(binary(), woody_context(), sequence_params()) -> integer(). +gen_sequence(SequenceID, WoodyContext, Params) -> + case bender_generator_client:gen_sequence(SequenceID, WoodyContext, Params) of + {ok, {_, ID}} -> ID + end. From e46bb1728585a665098f9a37160b51cc6e25660d Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Thu, 26 Nov 2020 02:31:39 +0300 Subject: [PATCH 07/40] add new accounting module --- apps/hellgate/src/hg_accounting_new.erl | 319 ++++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 apps/hellgate/src/hg_accounting_new.erl diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl new file mode 100644 index 000000000..018c5e32b --- /dev/null +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -0,0 +1,319 @@ +%%% Accounting for shumaich +%%% +%%% TODO +%%% - Brittle posting id assignment, it should be a level upper, maybe even in +%%% `hg_cashflow`. +%%% - Stuff cash flow details in the posting description fields. + +-module(hg_accounting_new). + +-export([get_account/1]). +-export([get_account/2]). + +-export([get_balance/1]). +-export([get_balance/2]). + +-export([create_account/1]). + +-export([collect_account_map/6]). +-export([collect_merchant_account_map/2]). +-export([collect_provider_account_map/3]). +-export([collect_system_account_map/5]). +-export([collect_external_account_map/4]). + +-export([hold/3]). +-export([hold/4]). + +-export([plan/3]). +-export([plan/4]). + +-export([commit/3]). +-export([commit/4]). + +-export([rollback/3]). +-export([rollback/4]). + +-include_lib("damsel/include/dmsl_payment_processing_thrift.hrl"). +-include_lib("damsel/include/dmsl_domain_thrift.hrl"). +-include_lib("shumaich_proto/include/shumaich_shumaich_thrift.hrl"). + +-type amount() :: dmsl_domain_thrift:'Amount'(). +-type currency_code() :: dmsl_domain_thrift:'CurrencySymbolicCode'(). +-type account_id() :: dmsl_accounter_thrift:'AccountID'(). +-type plan_id() :: dmsl_accounter_thrift:'PlanID'(). +-type batch_id() :: dmsl_accounter_thrift:'BatchID'(). +-type final_cash_flow() :: dmsl_domain_thrift:'FinalCashFlow'(). +-type batch() :: {batch_id(), final_cash_flow()}. +-type clock() :: domain_thrift:'AccounterClock'(). + +-type payment() :: dmsl_domain_thrift:'InvoicePayment'(). +-type shop() :: dmsl_domain_thrift:'Shop'(). +-type payment_institution() :: dmsl_domain_thrift:'PaymentInstitution'(). +-type provider() :: dmsl_domain_thrift:'Provider'(). +-type varset() :: pm_selector:varset(). +-type revision() :: hg_domain:revision(). + +-export_type([batch/0]). + +-type account() :: #{ + account_id => account_id(), + currency_code => currency_code() +}. + +-type balance() :: #{ + account_id => account_id(), + own_amount => amount(), + min_available_amount => amount(), + max_available_amount => amount(), + clock => clock() +}. + +-spec get_account(account_id()) -> account(). +get_account(AccountID) -> + get_account(AccountID, undefined). + +-spec get_account(account_id(), clock()) -> account(). +get_account(AccountID, Clock) -> + case call_accounter('GetAccountByID', {AccountID, to_accounter_clock(Clock)}) of + {ok, Result} -> + construct_account(AccountID, Result); + {exception, #shumaich_AccountNotFound{}} -> + hg_woody_wrapper:raise(#payproc_AccountNotFound{}); + {exception, #shumaich_NotReady{}} -> + % FIXME: this should probably work differently, hg_retry? + timer:sleep(100), + get_account(AccountID, Clock) + end. + +-spec get_balance(account_id()) -> balance(). +get_balance(AccountID) -> + get_balance(AccountID, undefined). + +-spec get_balance(account_id(), clock()) -> balance(). +get_balance(AccountID, Clock) -> + case call_accounter('GetBalanceByID', {AccountID, to_accounter_clock(Clock)}) of + {ok, Result} -> + construct_balance(AccountID, Result); + {exception, #shumaich_AccountNotFound{}} -> + hg_woody_wrapper:raise(#payproc_AccountNotFound{}); + % FIXME: this should probably work differently, hg_retry? + {exception, #shumaich_NotReady{}} -> + get_balance(AccountID, Clock) + end. + +-spec create_account(currency_code()) -> account_id(). +create_account(_CurrencyCode) -> + WoodyCtx = hg_context:get_woody_context(hg_context:load()), + % FIXME: placeholder, the sequence id should probably be passed externally + % not sure about the minimum too + hg_utils:gen_sequence(<<"create_shumaich_account">>, WoodyCtx, #{minimum => 10000}). + +-spec collect_account_map(payment(), shop(), payment_institution(), provider(), varset(), revision()) -> map(). +collect_account_map(Payment, Shop, PaymentInstitution, Provider, VS, Revision) -> + Map0 = collect_merchant_account_map(Shop, #{}), + Map1 = collect_provider_account_map(Payment, Provider, Map0), + Map2 = collect_system_account_map(Payment, PaymentInstitution, VS, Revision, Map1), + collect_external_account_map(Payment, VS, Revision, Map2). + +-spec collect_merchant_account_map(shop(), map()) -> map(). +collect_merchant_account_map(#domain_Shop{account = MerchantAccount}, Acc) -> + Acc#{ + {merchant, settlement} => MerchantAccount#domain_ShopAccount.settlement, + {merchant, guarantee} => MerchantAccount#domain_ShopAccount.guarantee + }. + +-spec collect_provider_account_map(payment(), provider(), map()) -> map(). +collect_provider_account_map(Payment, #domain_Provider{accounts = ProviderAccounts}, Acc) -> + Currency = get_currency(get_payment_cost(Payment)), + ProviderAccount = hg_payment_institution:choose_provider_account(Currency, ProviderAccounts), + Acc#{ + {provider, settlement} => ProviderAccount#domain_ProviderAccount.settlement + }. + +-spec collect_system_account_map(payment(), payment_institution(), varset(), revision(), map()) -> map(). +collect_system_account_map(Payment, PaymentInstitution, VS, Revision, Acc) -> + Currency = get_currency(get_payment_cost(Payment)), + SystemAccount = hg_payment_institution:get_system_account(Currency, VS, Revision, PaymentInstitution), + Acc#{ + {system, settlement} => SystemAccount#domain_SystemAccount.settlement, + {system, subagent} => SystemAccount#domain_SystemAccount.subagent + }. + +-spec collect_external_account_map(payment(), varset(), revision(), map()) -> map(). +collect_external_account_map(Payment, VS, Revision, Acc) -> + Currency = get_currency(get_payment_cost(Payment)), + case hg_payment_institution:choose_external_account(Currency, VS, Revision) of + #domain_ExternalAccount{income = Income, outcome = Outcome} -> + Acc#{ + {external, income} => Income, + {external, outcome} => Outcome + }; + undefined -> + Acc + end. + +%% +-spec plan(plan_id(), [batch()], hg_datetime:timestamp()) -> clock(). +plan(_PlanID, [], _Timestamp) -> + error(badarg); +plan(_PlanID, Batches, _Timestamp) when not is_list(Batches) -> + error(badarg); +plan(PlanID, Batches, Timestamp) -> + lists:foldl( + fun(Batch, _) -> hold(PlanID, Batch, Timestamp) end, + undefined, + Batches + ). + +-spec plan(plan_id(), [batch()], hg_datetime:timestamp(), clock()) -> clock(). +plan(_PlanID, [], _Timestamp, _Clock) -> + error(badarg); +plan(_PlanID, Batches, _Timestamp, _Clock) when not is_list(Batches) -> + error(badarg); +plan(PlanID, Batches, Timestamp, Clock) -> + lists:foldl( + fun(Batch, _) -> hold(PlanID, Batch, Timestamp, Clock) end, + undefined, + Batches + ). + +-spec hold(plan_id(), batch(), hg_datetime:timestamp()) -> clock(). +hold(PlanID, Batch, Timestamp) -> + do('Hold', construct_plan_change(PlanID, Batch, Timestamp)). + +-spec hold(plan_id(), batch(), hg_datetime:timestamp(), clock()) -> clock(). +hold(PlanID, Batches, Timestamp, Clock) -> + AccounterClock = to_accounter_clock(Clock), + do('Hold', construct_plan_change(PlanID, Batches, Timestamp), AccounterClock). + +-spec commit(plan_id(), [batch()], hg_datetime:timestamp()) -> clock(). +commit(PlanID, Batches, Timestamp) -> + do('CommitPlan', construct_plan(PlanID, Batches, Timestamp)). + +-spec commit(plan_id(), [batch()], hg_datetime:timestamp(), clock()) -> clock(). +commit(PlanID, Batches, Timestamp, Clock) -> + AccounterClock = to_accounter_clock(Clock), + do('CommitPlan', construct_plan(PlanID, Batches, Timestamp), AccounterClock). + +-spec rollback(plan_id(), [batch()], hg_datetime:timestamp()) -> clock(). +rollback(PlanID, Batches, Timestamp) -> + do('RollbackPlan', construct_plan(PlanID, Batches, Timestamp)). + +-spec rollback(plan_id(), [batch()], hg_datetime:timestamp(), clock()) -> clock(). +rollback(PlanID, Batches, Timestamp, Clock) -> + AccounterClock = to_accounter_clock(Clock), + do('RollbackPlan', construct_plan(PlanID, Batches, Timestamp), AccounterClock). + +do(Op, Plan) -> + do(Op, Plan, {latest, #shumaich_LatestClock{}}). + +do(Op, Plan, PreviousClock) -> + case call_accounter(Op, {Plan, PreviousClock}) of + {ok, Clock} -> + to_domain_clock(Clock); + {exception, #shumaich_NotReady{}} -> + erlang:display({'NOT READY', Op, Plan#shumaich_PostingPlan.id}), + % FIXME: maybe some other way? + ok = timer:sleep(200), + do(Op, Plan, PreviousClock); + {exception, Exception} -> + % FIXME + error({accounting, Exception}) + end. + +construct_plan_change(PlanID, {BatchID, Cashflow}, Timestamp) -> + #shumaich_PostingPlanChange{ + id = PlanID, + creation_time = Timestamp, + batch = #shumaich_PostingBatch{ + id = BatchID, + postings = collect_postings(Cashflow) + } + }. + +construct_plan(PlanID, Batches, Timestamp) -> + #shumaich_PostingPlan{ + id = PlanID, + creation_time = Timestamp, + batch_list = [ + #shumaich_PostingBatch{ + id = BatchID, + postings = collect_postings(Cashflow) + } + || {BatchID, Cashflow} <- Batches + ] + }. + +collect_postings(Cashflow) -> + [ + #shumaich_Posting{ + from_account = #shumaich_Account{id = Source, currency_symbolic_code = CurrencyCode}, + to_account = #shumaich_Account{id = Destination, currency_symbolic_code = CurrencyCode}, + amount = Amount, + currency_symbolic_code = CurrencyCode, + description = construct_posting_description(Details) + } + || #domain_FinalCashFlowPosting{ + source = #domain_FinalCashFlowAccount{account_id = Source}, + destination = #domain_FinalCashFlowAccount{account_id = Destination}, + details = Details, + volume = #domain_Cash{ + amount = Amount, + currency = #domain_CurrencyRef{symbolic_code = CurrencyCode} + } + } <- Cashflow + ]. + +construct_posting_description(Details) when is_binary(Details) -> + Details; +construct_posting_description(undefined) -> + <<>>. + +%% + +construct_account( + AccountID, + #shumaich_Account{ + currency_symbolic_code = CurrencyCode + } +) -> + #{ + account_id => AccountID, + currency_code => CurrencyCode + }. + +construct_balance( + AccountID, + #shumaich_Balance{ + own_amount = OwnAmount, + min_available_amount = MinAvailableAmount, + max_available_amount = MaxAvailableAmount + % clock = Clock + } +) -> + #{ + account_id => AccountID, + own_amount => OwnAmount, + min_available_amount => MinAvailableAmount, + max_available_amount => MaxAvailableAmount + % clock => to_domain_clock(Clock) + }. + +%% + +call_accounter(Function, Args) -> + hg_woody_wrapper:call(accounter, Function, Args). + +get_payment_cost(#domain_InvoicePayment{cost = Cost}) -> + Cost. + +get_currency(#domain_Cash{currency = Currency}) -> + Currency. + +to_domain_clock({vector, #shumaich_VectorClock{state = State}}) -> + {vector, #domain_VectorClock{state = State}}. + +to_accounter_clock(undefined) -> + {latest, #shumaich_LatestClock{}}; +to_accounter_clock({vector, #domain_VectorClock{state = State}}) -> + {vector, #shumaich_VectorClock{state = State}}. From 7b6fd9c6ca9c8f4ca970287f38fef710eb5eb1ea Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Thu, 26 Nov 2020 02:40:08 +0300 Subject: [PATCH 08/40] fix thrift compiler version --- rebar.config | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index c6f6c6e63..4359cfcea 100644 --- a/rebar.config +++ b/rebar.config @@ -111,7 +111,9 @@ {plugins, [ - {erlfmt, "0.7.0"} + {erlfmt, "0.7.0"}, + {rebar3_thrift_compiler, + {git, "https://github.com/rbkmoney/rebar3_thrift_compiler.git", {tag, "0.3.1"}}} ]}. {erlfmt, [ From d88626aa2500df5038aba2d403759709774e3373 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Thu, 26 Nov 2020 03:14:45 +0300 Subject: [PATCH 09/40] add shumaich service handling --- apps/hellgate/test/hg_ct_helper.erl | 1 + apps/hg_proto/src/hg_proto.erl | 2 ++ config/sys.config | 1 + 3 files changed, 4 insertions(+) diff --git a/apps/hellgate/test/hg_ct_helper.erl b/apps/hellgate/test/hg_ct_helper.erl index f6225fc13..05246f385 100644 --- a/apps/hellgate/test/hg_ct_helper.erl +++ b/apps/hellgate/test/hg_ct_helper.erl @@ -128,6 +128,7 @@ start_app(hellgate = AppName) -> }}, {services, #{ accounter => <<"http://shumway:8022/shumpune">>, + accounter_new => <<"http://shumway:8022/shumaich">>, automaton => <<"http://machinegun:8022/v1/automaton">>, customer_management => #{ url => <<"http://hellgate:8022/v1/processing/customer_management">>, diff --git a/apps/hg_proto/src/hg_proto.erl b/apps/hg_proto/src/hg_proto.erl index eef373e51..93fbfe542 100644 --- a/apps/hg_proto/src/hg_proto.erl +++ b/apps/hg_proto/src/hg_proto.erl @@ -40,6 +40,8 @@ get_service(proxy_host_provider) -> {dmsl_proxy_provider_thrift, 'ProviderProxyHost'}; get_service(accounter) -> {shumpune_shumpune_thrift, 'Accounter'}; +get_service(accounter_new) -> + {shumpune_shumpune_thrift, 'Accounter'}; get_service(automaton) -> {mg_proto_state_processing_thrift, 'Automaton'}; get_service(processor) -> diff --git a/config/sys.config b/config/sys.config index 0623a4483..8404935f3 100644 --- a/config/sys.config +++ b/config/sys.config @@ -51,6 +51,7 @@ automaton => "http://machinegun:8022/v1/automaton", eventsink => "http://machinegun:8022/v1/event_sink", accounter => "http://shumway:8022/shumpune", + accounter_new => "http://shumway:8022/shumaich", party_management => "http://hellgate:8022/v1/processing/partymgmt", customer_management => "http://hellgate:8022/v1/processing/customer_management", % TODO make more consistent From c92dc7fcb21a8920cfeab897e0fa2ed9e49e3786 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Mon, 30 Nov 2020 02:56:05 +0300 Subject: [PATCH 10/40] update new accounter --- apps/hellgate/src/hg_accounting_new.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 018c5e32b..13f09866a 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -302,7 +302,7 @@ construct_balance( %% call_accounter(Function, Args) -> - hg_woody_wrapper:call(accounter, Function, Args). + hg_woody_wrapper:call(accounter_new, Function, Args). get_payment_cost(#domain_InvoicePayment{cost = Cost}) -> Cost. From 21539f0f0d615db653279683b8d610e3c6f9fbc6 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Mon, 30 Nov 2020 02:56:22 +0300 Subject: [PATCH 11/40] update hg_proto --- apps/hg_proto/src/hg_proto.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/hg_proto/src/hg_proto.erl b/apps/hg_proto/src/hg_proto.erl index 93fbfe542..44dc9b669 100644 --- a/apps/hg_proto/src/hg_proto.erl +++ b/apps/hg_proto/src/hg_proto.erl @@ -41,7 +41,7 @@ get_service(proxy_host_provider) -> get_service(accounter) -> {shumpune_shumpune_thrift, 'Accounter'}; get_service(accounter_new) -> - {shumpune_shumpune_thrift, 'Accounter'}; + {shumaich_shumaich_thrift, 'Accounter'}; get_service(automaton) -> {mg_proto_state_processing_thrift, 'Automaton'}; get_service(processor) -> From 784168d8506053cc570770ca22410d684e9dd606 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Mon, 30 Nov 2020 03:32:24 +0300 Subject: [PATCH 12/40] update account balances test --- apps/hellgate/test/hg_invoice_tests_SUITE.erl | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/apps/hellgate/test/hg_invoice_tests_SUITE.erl b/apps/hellgate/test/hg_invoice_tests_SUITE.erl index 94849b9b4..8a8faa228 100644 --- a/apps/hellgate/test/hg_invoice_tests_SUITE.erl +++ b/apps/hellgate/test/hg_invoice_tests_SUITE.erl @@ -4805,18 +4805,35 @@ consistent_account_balances(C) -> Party = hg_client_party:get(PartyClient), Shops = maps:values(Party#domain_Party.shops), _ = [ - consistent_account_balance(AccountID, Shop) + { + consistent_account_balance(AccountID, Shop), + consistent_account_balance_new(AccountID, Shop) + } || #domain_Shop{account = ShopAccount} = Shop <- Shops, #domain_ShopAccount{settlement = AccountID1, guarantee = AccountID2} <- [ShopAccount], AccountID <- [AccountID1, AccountID2] ]. consistent_account_balance(AccountID, Comment) -> - case hg_ct_helper:get_balance(AccountID) of + try hg_accounting:get_balance(AccountID) of #{own_amount := V, min_available_amount := V, max_available_amount := V} -> ok; #{} = Account -> erlang:error({"Inconsistent account balance", Account, Comment}) + catch + #payproc_AccountNotFound{} -> + ok + end. + +consistent_account_balance_new(AccountID, Comment) -> + try hg_accounting:get_balance(AccountID) of + #{own_amount := V, min_available_amount := V, max_available_amount := V} -> + ok; + #{} = Account -> + erlang:error({"Inconsistent account balance", Account, Comment}) + catch + #payproc_AccountNotFound{} -> + ok end. %% From aaba8723c9f1787b9f2915074982a09c58da2489 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Mon, 30 Nov 2020 11:26:51 +0300 Subject: [PATCH 13/40] fix format --- apps/hellgate/test/hg_invoice_tests_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/hellgate/test/hg_invoice_tests_SUITE.erl b/apps/hellgate/test/hg_invoice_tests_SUITE.erl index 8a8faa228..097ab044d 100644 --- a/apps/hellgate/test/hg_invoice_tests_SUITE.erl +++ b/apps/hellgate/test/hg_invoice_tests_SUITE.erl @@ -4805,10 +4805,10 @@ consistent_account_balances(C) -> Party = hg_client_party:get(PartyClient), Shops = maps:values(Party#domain_Party.shops), _ = [ - { + { consistent_account_balance(AccountID, Shop), consistent_account_balance_new(AccountID, Shop) - } + } || #domain_Shop{account = ShopAccount} = Shop <- Shops, #domain_ShopAccount{settlement = AccountID1, guarantee = AccountID2} <- [ShopAccount], AccountID <- [AccountID1, AccountID2] From 8562e08971923a7d07c8fd665c7bee91e376a291 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Mon, 30 Nov 2020 11:35:54 +0300 Subject: [PATCH 14/40] fix xref --- apps/hellgate/src/hg_accounting_new.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 13f09866a..8613fe401 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -130,10 +130,10 @@ collect_provider_account_map(Payment, #domain_Provider{accounts = ProviderAccoun {provider, settlement} => ProviderAccount#domain_ProviderAccount.settlement }. --spec collect_system_account_map(payment(), payment_institution(), varset(), revision(), map()) -> map(). -collect_system_account_map(Payment, PaymentInstitution, VS, Revision, Acc) -> +-spec collect_system_account_map(payment(), payment_institution(), revision(), map()) -> map(). +collect_system_account_map(Payment, PaymentInstitution, Revision, Acc) -> Currency = get_currency(get_payment_cost(Payment)), - SystemAccount = hg_payment_institution:get_system_account(Currency, VS, Revision, PaymentInstitution), + SystemAccount = hg_payment_institution:get_system_account(Currency, Revision, PaymentInstitution), Acc#{ {system, settlement} => SystemAccount#domain_SystemAccount.settlement, {system, subagent} => SystemAccount#domain_SystemAccount.subagent From fef9a7b70018dcf6c58baa6d2c657689db741579 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Mon, 30 Nov 2020 11:51:17 +0300 Subject: [PATCH 15/40] some fixes --- apps/hellgate/src/hg_accounting_new.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 8613fe401..4c2ea0fd1 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -18,7 +18,7 @@ -export([collect_account_map/6]). -export([collect_merchant_account_map/2]). -export([collect_provider_account_map/3]). --export([collect_system_account_map/5]). +-export([collect_system_account_map/4]). -export([collect_external_account_map/4]). -export([hold/3]). @@ -112,7 +112,7 @@ create_account(_CurrencyCode) -> collect_account_map(Payment, Shop, PaymentInstitution, Provider, VS, Revision) -> Map0 = collect_merchant_account_map(Shop, #{}), Map1 = collect_provider_account_map(Payment, Provider, Map0), - Map2 = collect_system_account_map(Payment, PaymentInstitution, VS, Revision, Map1), + Map2 = collect_system_account_map(Payment, PaymentInstitution, Revision, Map1), collect_external_account_map(Payment, VS, Revision, Map2). -spec collect_merchant_account_map(shop(), map()) -> map(). From b46924d059ad190eb481948f03fc340a16f8ea64 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Mon, 30 Nov 2020 12:43:07 +0300 Subject: [PATCH 16/40] type fix --- apps/hellgate/src/hg_accounting_new.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 4c2ea0fd1..9fc76b7cb 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -44,7 +44,7 @@ -type batch_id() :: dmsl_accounter_thrift:'BatchID'(). -type final_cash_flow() :: dmsl_domain_thrift:'FinalCashFlow'(). -type batch() :: {batch_id(), final_cash_flow()}. --type clock() :: domain_thrift:'AccounterClock'(). +-type clock() :: dmsl_domain_thrift:'AccounterClock'(). -type payment() :: dmsl_domain_thrift:'InvoicePayment'(). -type shop() :: dmsl_domain_thrift:'Shop'(). From 27b264fcfcf19c07af4d9dfb96ea3add3b35c214 Mon Sep 17 00:00:00 2001 From: Roman Pushkov Date: Mon, 30 Nov 2020 15:59:47 +0300 Subject: [PATCH 17/40] type fix --- apps/hellgate/src/hg_accounting_new.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 9fc76b7cb..edf16af98 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -72,7 +72,7 @@ get_account(AccountID) -> get_account(AccountID, undefined). --spec get_account(account_id(), clock()) -> account(). +-spec get_account(account_id(), undefined | clock()) -> account(). get_account(AccountID, Clock) -> case call_accounter('GetAccountByID', {AccountID, to_accounter_clock(Clock)}) of {ok, Result} -> @@ -89,7 +89,7 @@ get_account(AccountID, Clock) -> get_balance(AccountID) -> get_balance(AccountID, undefined). --spec get_balance(account_id(), clock()) -> balance(). +-spec get_balance(account_id(), undefined | clock()) -> balance(). get_balance(AccountID, Clock) -> case call_accounter('GetBalanceByID', {AccountID, to_accounter_clock(Clock)}) of {ok, Result} -> From 186fb653838f1df5e8ca7f3dc0b49d88303ad2c8 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Wed, 23 Jun 2021 16:33:59 +0300 Subject: [PATCH 18/40] fix format --- rebar.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index f5e96e88d..f248bff85 100644 --- a/rebar.config +++ b/rebar.config @@ -101,8 +101,7 @@ {plugins, [ {erlfmt, "0.10.0"}, - {rebar3_thrift_compiler, - {git, "https://github.com/rbkmoney/rebar3_thrift_compiler.git", {tag, "0.3.1"}}} + {rebar3_thrift_compiler, {git, "https://github.com/rbkmoney/rebar3_thrift_compiler.git", {tag, "0.3.1"}}} ]}. {erlfmt, [ From a6b0dc38212a66beac085a62d92132051273064f Mon Sep 17 00:00:00 2001 From: Kehitt Date: Mon, 12 Jul 2021 14:22:28 +0300 Subject: [PATCH 19/40] accounter retries via hg_retry --- apps/hellgate/src/hg_accounting_new.erl | 42 ++++++++++--------- apps/hellgate/src/hg_retry.erl | 21 +++++++++- apps/hellgate/src/hg_utils.erl | 4 -- apps/hellgate/test/hg_invoice_tests_SUITE.erl | 12 ++---- docker-compose.sh | 12 +++--- 5 files changed, 53 insertions(+), 38 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index edf16af98..216b890e2 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -78,11 +78,7 @@ get_account(AccountID, Clock) -> {ok, Result} -> construct_account(AccountID, Result); {exception, #shumaich_AccountNotFound{}} -> - hg_woody_wrapper:raise(#payproc_AccountNotFound{}); - {exception, #shumaich_NotReady{}} -> - % FIXME: this should probably work differently, hg_retry? - timer:sleep(100), - get_account(AccountID, Clock) + hg_woody_wrapper:raise(#payproc_AccountNotFound{}) end. -spec get_balance(account_id()) -> balance(). @@ -95,10 +91,7 @@ get_balance(AccountID, Clock) -> {ok, Result} -> construct_balance(AccountID, Result); {exception, #shumaich_AccountNotFound{}} -> - hg_woody_wrapper:raise(#payproc_AccountNotFound{}); - % FIXME: this should probably work differently, hg_retry? - {exception, #shumaich_NotReady{}} -> - get_balance(AccountID, Clock) + hg_woody_wrapper:raise(#payproc_AccountNotFound{}) end. -spec create_account(currency_code()) -> account_id(). @@ -211,11 +204,6 @@ do(Op, Plan, PreviousClock) -> case call_accounter(Op, {Plan, PreviousClock}) of {ok, Clock} -> to_domain_clock(Clock); - {exception, #shumaich_NotReady{}} -> - erlang:display({'NOT READY', Op, Plan#shumaich_PostingPlan.id}), - % FIXME: maybe some other way? - ok = timer:sleep(200), - do(Op, Plan, PreviousClock); {exception, Exception} -> % FIXME error({accounting, Exception}) @@ -287,22 +275,34 @@ construct_balance( #shumaich_Balance{ own_amount = OwnAmount, min_available_amount = MinAvailableAmount, - max_available_amount = MaxAvailableAmount - % clock = Clock + max_available_amount = MaxAvailableAmount, + clock = Clock } ) -> #{ account_id => AccountID, own_amount => OwnAmount, min_available_amount => MinAvailableAmount, - max_available_amount => MaxAvailableAmount - % clock => to_domain_clock(Clock) + max_available_amount => MaxAvailableAmount, + clock => to_domain_clock(Clock) }. %% call_accounter(Function, Args) -> - hg_woody_wrapper:call(accounter_new, Function, Args). + hg_retry:call_with_retry( + fun() -> + case hg_woody_wrapper:call(accounter_new, Function, Args) of + {ok, _} = Ok -> + {return, Ok}; + {exception, #shumaich_NotReady{}} -> + retry; + {exception, _} = Exception -> + {return, Exception} + end + end, + get_retry_strategy(Function) + ). get_payment_cost(#domain_InvoicePayment{cost = Cost}) -> Cost. @@ -317,3 +317,7 @@ to_accounter_clock(undefined) -> {latest, #shumaich_LatestClock{}}; to_accounter_clock({vector, #domain_VectorClock{state = State}}) -> {vector, #shumaich_VectorClock{state = State}}. + +get_retry_strategy(Function) -> + PolicyConfig = genlib_app:env(hellgate, accounter_retry_policy, #{}), + hg_retry:new_strategy(maps:get(Function, PolicyConfig, no_retry)). diff --git a/apps/hellgate/src/hg_retry.erl b/apps/hellgate/src/hg_retry.erl index 4058cad9e..7086f508b 100644 --- a/apps/hellgate/src/hg_retry.erl +++ b/apps/hellgate/src/hg_retry.erl @@ -3,7 +3,8 @@ -export([ next_step/1, skip_steps/2, - new_strategy/1 + new_strategy/1, + call_with_retry/2 ]). -type retries_num() :: pos_integer() | infinity. @@ -58,4 +59,22 @@ skip_steps(Strategy, N) when N > 0 -> end, skip_steps(NewStrategy, N - 1). +-type retry_fun_return() :: term() | no_return(). +-type retry_fun() :: fun(() -> retry_fun_return()). + +-spec call_with_retry(retry_fun(), strategy()) -> retry_fun_return(). +call_with_retry(Fun, Strategy) -> + case Fun() of + {return, Result} -> + Result; + retry -> + case next_step(Strategy) of + {wait, Timeout, NextStrategy} -> + _ = timer:sleep(Timeout), + call_with_retry(Fun, NextStrategy); + finish -> + throw({error, no_more_retries}) + end + end. + %%% Internal functions diff --git a/apps/hellgate/src/hg_utils.erl b/apps/hellgate/src/hg_utils.erl index b722d654f..318461da0 100644 --- a/apps/hellgate/src/hg_utils.erl +++ b/apps/hellgate/src/hg_utils.erl @@ -15,13 +15,9 @@ -export([gen_sequence/2]). -export([gen_sequence/3]). --define(APP, bender_client). - -type sequence_params() :: #{minimum => integer()}. -type woody_context() :: woody_context:ctx(). --include_lib("bender_proto/include/bender_thrift.hrl"). - %% -spec unique_id() -> dmsl_base_thrift:'ID'(). diff --git a/apps/hellgate/test/hg_invoice_tests_SUITE.erl b/apps/hellgate/test/hg_invoice_tests_SUITE.erl index 7b667c100..02773d9a5 100644 --- a/apps/hellgate/test/hg_invoice_tests_SUITE.erl +++ b/apps/hellgate/test/hg_invoice_tests_SUITE.erl @@ -4674,25 +4674,19 @@ consistent_account_balances(C) -> ]. consistent_account_balance(AccountID, Comment) -> - try hg_accounting:get_balance(AccountID) of + case hg_accounting:get_balance(AccountID) of #{own_amount := V, min_available_amount := V, max_available_amount := V} -> ok; #{} = Account -> erlang:error({"Inconsistent account balance", Account, Comment}) - catch - #payproc_AccountNotFound{} -> - ok end. consistent_account_balance_new(AccountID, Comment) -> - try hg_accounting:get_balance(AccountID) of + case hg_accounting_new:get_balance(AccountID) of #{own_amount := V, min_available_amount := V, max_available_amount := V} -> ok; #{} = Account -> - erlang:error({"Inconsistent account balance", Account, Comment}) - catch - #payproc_AccountNotFound{} -> - ok + erlang:error({"Inconsistent account balance (new)", Account, Comment}) end. %% diff --git a/docker-compose.sh b/docker-compose.sh index c286edfc8..a8e9d5582 100755 --- a/docker-compose.sh +++ b/docker-compose.sh @@ -59,7 +59,7 @@ services: condition: service_healthy shumway: - image: dr2.rbkmoney.com/rbkmoney/shumway:658c9aec229b5a70d745a49cb938bb1a132b5ca2 + image: dr2.rbkmoney.com/rbkmoney/shumway:2f7d381d36ec69cfc90c77996f7e82b79d89e80b hostname: shumway container_name: shumway ports: @@ -70,7 +70,7 @@ services: "spring.datasource.username": "postgres", "spring.datasource.password": "postgres", "management.metrics.export.statsd.enabled": "false", - "service.shumaich.url": "http://shumaich:8033/shumaich" + "service.shumaich.url": "http://shumaich:8022/shumaich" }' depends_on: - postgres @@ -140,12 +140,9 @@ services: image: dr2.rbkmoney.com/rbkmoney/shumaich:1e4ebe41a9aaae0c46b1c41edffb95f7d93c5f48 hostname: shumaich container_name: shumaich - ports: - - "8033:8033" restart: on-failure environment: SPRING_APPLICATION_JSON: '{ - "server.port": "8033", "rocksdb.name": "shumaich", "rocksdb.dir": "/temp/rocksdb", "kafka.bootstrap-servers": "broker:9092", @@ -160,6 +157,11 @@ services: target: /temp/rocksdb/shumaich volume: nocopy: true + healthcheck: + test: "curl http://localhost:8022/" + interval: 5s + timeout: 1s + retries: 20 holmes: image: dr2.rbkmoney.com/rbkmoney/holmes:7d496d0886a1489044c57eee4ba4bfcf8f8b6a48 From 6f9b7ae120745b4333366a43d2b5514390b89427 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Mon, 12 Jul 2021 14:47:25 +0300 Subject: [PATCH 20/40] fix test --- apps/hellgate/test/hg_invoice_tests_SUITE.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/hellgate/test/hg_invoice_tests_SUITE.erl b/apps/hellgate/test/hg_invoice_tests_SUITE.erl index 02773d9a5..0c3f1d640 100644 --- a/apps/hellgate/test/hg_invoice_tests_SUITE.erl +++ b/apps/hellgate/test/hg_invoice_tests_SUITE.erl @@ -4682,11 +4682,14 @@ consistent_account_balance(AccountID, Comment) -> end. consistent_account_balance_new(AccountID, Comment) -> - case hg_accounting_new:get_balance(AccountID) of + try hg_accounting_new:get_balance(AccountID) of #{own_amount := V, min_available_amount := V, max_available_amount := V} -> ok; #{} = Account -> erlang:error({"Inconsistent account balance (new)", Account, Comment}) + catch + #payproc_AccountNotFound{} -> + ok end. %% From cb86a3cd0bdf7e6474aadf920d7495d4bacfd28a Mon Sep 17 00:00:00 2001 From: Kehitt Date: Mon, 12 Jul 2021 15:48:43 +0300 Subject: [PATCH 21/40] fix clock decoding, relax retry strategy --- apps/hellgate/src/hg_accounting_new.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 216b890e2..24ceb3513 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -68,6 +68,8 @@ clock => clock() }. +-define(DEFAULT_RETRY_STRATEGY, {linear, 5, 100}). + -spec get_account(account_id()) -> account(). get_account(AccountID) -> get_account(AccountID, undefined). @@ -279,17 +281,18 @@ construct_balance( clock = Clock } ) -> - #{ + genlib_map:compact(#{ account_id => AccountID, own_amount => OwnAmount, min_available_amount => MinAvailableAmount, max_available_amount => MaxAvailableAmount, clock => to_domain_clock(Clock) - }. + }). %% call_accounter(Function, Args) -> + %% Really not sure what to best do when we run out of retries hg_retry:call_with_retry( fun() -> case hg_woody_wrapper:call(accounter_new, Function, Args) of @@ -310,6 +313,8 @@ get_payment_cost(#domain_InvoicePayment{cost = Cost}) -> get_currency(#domain_Cash{currency = Currency}) -> Currency. +to_domain_clock({latest, #shumaich_LatestClock{}}) -> + undefined; to_domain_clock({vector, #shumaich_VectorClock{state = State}}) -> {vector, #domain_VectorClock{state = State}}. @@ -320,4 +325,4 @@ to_accounter_clock({vector, #domain_VectorClock{state = State}}) -> get_retry_strategy(Function) -> PolicyConfig = genlib_app:env(hellgate, accounter_retry_policy, #{}), - hg_retry:new_strategy(maps:get(Function, PolicyConfig, no_retry)). + hg_retry:new_strategy(maps:get(Function, PolicyConfig, ?DEFAULT_RETRY_STRATEGY)). From 0c9806a69677549ba06042ff38bc1d0692b21451 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Mon, 12 Jul 2021 17:45:59 +0300 Subject: [PATCH 22/40] fix balance consistency --- apps/hellgate/test/hg_invoice_tests_SUITE.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/hellgate/test/hg_invoice_tests_SUITE.erl b/apps/hellgate/test/hg_invoice_tests_SUITE.erl index 0c3f1d640..dce109c4e 100644 --- a/apps/hellgate/test/hg_invoice_tests_SUITE.erl +++ b/apps/hellgate/test/hg_invoice_tests_SUITE.erl @@ -4682,7 +4682,8 @@ consistent_account_balance(AccountID, Comment) -> end. consistent_account_balance_new(AccountID, Comment) -> - try hg_accounting_new:get_balance(AccountID) of + %% TODO: Switch to hg_accounting_new when all operations are migrated + try hg_accounting:get_balance(AccountID) of #{own_amount := V, min_available_amount := V, max_available_amount := V} -> ok; #{} = Account -> From 0c66e4c6f003c142ffb4e2b05c3b0b61d1aeb152 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Wed, 14 Jul 2021 22:05:51 +0300 Subject: [PATCH 23/40] relax default retry strategy --- apps/hellgate/src/hg_accounting_new.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 24ceb3513..1f910fc42 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -68,7 +68,7 @@ clock => clock() }. --define(DEFAULT_RETRY_STRATEGY, {linear, 5, 100}). +-define(DEFAULT_RETRY_STRATEGY, {exponential, 6, 1.5, 100}). -spec get_account(account_id()) -> account(). get_account(AccountID) -> From 42831aea4d61b53aa64b0081037ec1b46f694492 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Thu, 15 Jul 2021 13:46:40 +0300 Subject: [PATCH 24/40] tighter default retry strategy --- apps/hellgate/src/hg_accounting_new.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 1f910fc42..77d3daa8c 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -68,7 +68,7 @@ clock => clock() }. --define(DEFAULT_RETRY_STRATEGY, {exponential, 6, 1.5, 100}). +-define(DEFAULT_RETRY_STRATEGY, {exponential, 10, 1.1, 100}). -spec get_account(account_id()) -> account(). get_account(AccountID) -> From 4e65175f97506619a1240bcd161c735a351bc9cc Mon Sep 17 00:00:00 2001 From: Kehitt Date: Mon, 6 Sep 2021 12:47:23 +0300 Subject: [PATCH 25/40] fail machine (somewhat) properly when out of retries to accounter --- apps/hellgate/src/hg_accounting_new.erl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 77d3daa8c..764d2ec66 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -293,6 +293,14 @@ construct_balance( call_accounter(Function, Args) -> %% Really not sure what to best do when we run out of retries + try + call_with_retry(Function, Args) + catch + throw:{error, no_more_retries} -> + error({accounter, not_ready}) + end. + +call_with_retry(Function, Args) -> hg_retry:call_with_retry( fun() -> case hg_woody_wrapper:call(accounter_new, Function, Args) of From e1daeb5ad72202f2772930f6f6ab0b2da5c9e42c Mon Sep 17 00:00:00 2001 From: Kehitt Date: Fri, 10 Sep 2021 16:11:20 +0300 Subject: [PATCH 26/40] fix plan function, remove duplicate code --- apps/hellgate/src/hg_accounting_new.erl | 50 +++++-------------------- 1 file changed, 10 insertions(+), 40 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 764d2ec66..b73e51190 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -105,47 +105,23 @@ create_account(_CurrencyCode) -> -spec collect_account_map(payment(), shop(), payment_institution(), provider(), varset(), revision()) -> map(). collect_account_map(Payment, Shop, PaymentInstitution, Provider, VS, Revision) -> - Map0 = collect_merchant_account_map(Shop, #{}), - Map1 = collect_provider_account_map(Payment, Provider, Map0), - Map2 = collect_system_account_map(Payment, PaymentInstitution, Revision, Map1), - collect_external_account_map(Payment, VS, Revision, Map2). + hg_accounting:collect_account_map(Payment, Shop, PaymentInstitution, Provider, VS, Revision). -spec collect_merchant_account_map(shop(), map()) -> map(). -collect_merchant_account_map(#domain_Shop{account = MerchantAccount}, Acc) -> - Acc#{ - {merchant, settlement} => MerchantAccount#domain_ShopAccount.settlement, - {merchant, guarantee} => MerchantAccount#domain_ShopAccount.guarantee - }. +collect_merchant_account_map(Shop, Acc) -> + hg_accounting:collect_merchant_account_map(Shop, Acc). -spec collect_provider_account_map(payment(), provider(), map()) -> map(). -collect_provider_account_map(Payment, #domain_Provider{accounts = ProviderAccounts}, Acc) -> - Currency = get_currency(get_payment_cost(Payment)), - ProviderAccount = hg_payment_institution:choose_provider_account(Currency, ProviderAccounts), - Acc#{ - {provider, settlement} => ProviderAccount#domain_ProviderAccount.settlement - }. +collect_provider_account_map(Payment, Provider, Acc) -> + hg_accounting:collect_provider_account_map(Payment, Provider, Acc). -spec collect_system_account_map(payment(), payment_institution(), revision(), map()) -> map(). collect_system_account_map(Payment, PaymentInstitution, Revision, Acc) -> - Currency = get_currency(get_payment_cost(Payment)), - SystemAccount = hg_payment_institution:get_system_account(Currency, Revision, PaymentInstitution), - Acc#{ - {system, settlement} => SystemAccount#domain_SystemAccount.settlement, - {system, subagent} => SystemAccount#domain_SystemAccount.subagent - }. + hg_accounting:collect_system_account_map(Payment, PaymentInstitution, Revision, Acc). -spec collect_external_account_map(payment(), varset(), revision(), map()) -> map(). collect_external_account_map(Payment, VS, Revision, Acc) -> - Currency = get_currency(get_payment_cost(Payment)), - case hg_payment_institution:choose_external_account(Currency, VS, Revision) of - #domain_ExternalAccount{income = Income, outcome = Outcome} -> - Acc#{ - {external, income} => Income, - {external, outcome} => Outcome - }; - undefined -> - Acc - end. + hg_accounting:collect_external_account_map(Payment, VS, Revision, Acc). %% -spec plan(plan_id(), [batch()], hg_datetime:timestamp()) -> clock(). @@ -155,7 +131,7 @@ plan(_PlanID, Batches, _Timestamp) when not is_list(Batches) -> error(badarg); plan(PlanID, Batches, Timestamp) -> lists:foldl( - fun(Batch, _) -> hold(PlanID, Batch, Timestamp) end, + fun(Batch, ClockAcc) -> hold(PlanID, Batch, Timestamp, ClockAcc) end, undefined, Batches ). @@ -167,8 +143,8 @@ plan(_PlanID, Batches, _Timestamp, _Clock) when not is_list(Batches) -> error(badarg); plan(PlanID, Batches, Timestamp, Clock) -> lists:foldl( - fun(Batch, _) -> hold(PlanID, Batch, Timestamp, Clock) end, - undefined, + fun(Batch, ClockAcc) -> hold(PlanID, Batch, Timestamp, ClockAcc) end, + Clock, Batches ). @@ -315,12 +291,6 @@ call_with_retry(Function, Args) -> get_retry_strategy(Function) ). -get_payment_cost(#domain_InvoicePayment{cost = Cost}) -> - Cost. - -get_currency(#domain_Cash{currency = Currency}) -> - Currency. - to_domain_clock({latest, #shumaich_LatestClock{}}) -> undefined; to_domain_clock({vector, #shumaich_VectorClock{state = State}}) -> From cb7c878a9ee131daa5ff1aef263f5ea13314f11a Mon Sep 17 00:00:00 2001 From: Kehitt Date: Fri, 10 Sep 2021 17:54:03 +0300 Subject: [PATCH 27/40] fix types --- apps/hellgate/src/hg_accounting_new.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index b73e51190..c611b8b82 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -152,7 +152,7 @@ plan(PlanID, Batches, Timestamp, Clock) -> hold(PlanID, Batch, Timestamp) -> do('Hold', construct_plan_change(PlanID, Batch, Timestamp)). --spec hold(plan_id(), batch(), hg_datetime:timestamp(), clock()) -> clock(). +-spec hold(plan_id(), batch(), hg_datetime:timestamp(), clock() | undefined) -> clock(). hold(PlanID, Batches, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('Hold', construct_plan_change(PlanID, Batches, Timestamp), AccounterClock). @@ -161,7 +161,7 @@ hold(PlanID, Batches, Timestamp, Clock) -> commit(PlanID, Batches, Timestamp) -> do('CommitPlan', construct_plan(PlanID, Batches, Timestamp)). --spec commit(plan_id(), [batch()], hg_datetime:timestamp(), clock()) -> clock(). +-spec commit(plan_id(), [batch()], hg_datetime:timestamp(), clock() | undefined) -> clock(). commit(PlanID, Batches, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('CommitPlan', construct_plan(PlanID, Batches, Timestamp), AccounterClock). @@ -170,7 +170,7 @@ commit(PlanID, Batches, Timestamp, Clock) -> rollback(PlanID, Batches, Timestamp) -> do('RollbackPlan', construct_plan(PlanID, Batches, Timestamp)). --spec rollback(plan_id(), [batch()], hg_datetime:timestamp(), clock()) -> clock(). +-spec rollback(plan_id(), [batch()], hg_datetime:timestamp(), clock() | undefined) -> clock(). rollback(PlanID, Batches, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('RollbackPlan', construct_plan(PlanID, Batches, Timestamp), AccounterClock). From 8b2b9fa2ede7eeba537add55dbabee1347a394d6 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Tue, 14 Sep 2021 17:41:40 +0300 Subject: [PATCH 28/40] only retry sometimes --- apps/hellgate/src/hg_accounting_new.erl | 29 ++++++++++++++----------- apps/hellgate/src/hg_retry.erl | 4 ++-- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index c611b8b82..edf381eda 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -183,8 +183,7 @@ do(Op, Plan, PreviousClock) -> {ok, Clock} -> to_domain_clock(Clock); {exception, Exception} -> - % FIXME - error({accounting, Exception}) + erlang:throw(Exception) end. construct_plan_change(PlanID, {BatchID, Cashflow}, Timestamp) -> @@ -267,23 +266,24 @@ construct_balance( %% +call_accounter(Function, Args) when + Function =:= 'CommitPlan'; + Function =:= 'RollbackPlan'; + Function =:= 'GetBalanceByID'; + Function =:= 'GetAccountByID' +-> + call_service_with_retry(Function, Args); call_accounter(Function, Args) -> - %% Really not sure what to best do when we run out of retries - try - call_with_retry(Function, Args) - catch - throw:{error, no_more_retries} -> - error({accounter, not_ready}) - end. + call_service(Function, Args). -call_with_retry(Function, Args) -> +call_service_with_retry(Function, Args) -> hg_retry:call_with_retry( fun() -> - case hg_woody_wrapper:call(accounter_new, Function, Args) of + case call_service(Function, Args) of {ok, _} = Ok -> {return, Ok}; - {exception, #shumaich_NotReady{}} -> - retry; + {exception, #shumaich_NotReady{}} = Exception -> + {retry, Exception}; {exception, _} = Exception -> {return, Exception} end @@ -291,6 +291,9 @@ call_with_retry(Function, Args) -> get_retry_strategy(Function) ). +call_service(Function, Args) -> + hg_woody_wrapper:call(accounter_new, Function, Args). + to_domain_clock({latest, #shumaich_LatestClock{}}) -> undefined; to_domain_clock({vector, #shumaich_VectorClock{state = State}}) -> diff --git a/apps/hellgate/src/hg_retry.erl b/apps/hellgate/src/hg_retry.erl index 7086f508b..0e6a6d7ef 100644 --- a/apps/hellgate/src/hg_retry.erl +++ b/apps/hellgate/src/hg_retry.erl @@ -67,13 +67,13 @@ call_with_retry(Fun, Strategy) -> case Fun() of {return, Result} -> Result; - retry -> + {retry, Error} -> case next_step(Strategy) of {wait, Timeout, NextStrategy} -> _ = timer:sleep(Timeout), call_with_retry(Fun, NextStrategy); finish -> - throw({error, no_more_retries}) + Error end end. From a821836bc86c81ec2549409a48ef4f0f08c6057c Mon Sep 17 00:00:00 2001 From: Kehitt Date: Mon, 20 Sep 2021 18:52:36 +0300 Subject: [PATCH 29/40] change hg_accounting_new api --- apps/hellgate/src/hg_accounting_new.erl | 102 +++++++++++++++--------- 1 file changed, 64 insertions(+), 38 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index edf381eda..39982ff35 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -54,6 +54,7 @@ -type revision() :: hg_domain:revision(). -export_type([batch/0]). +-export_type([clock/0]). -type account() :: #{ account_id => account_id(), @@ -70,30 +71,30 @@ -define(DEFAULT_RETRY_STRATEGY, {exponential, 10, 1.1, 100}). --spec get_account(account_id()) -> account(). +-spec get_account(account_id()) -> {ok, account()} | {error, not_ready | {account_not_found, _}}. get_account(AccountID) -> get_account(AccountID, undefined). --spec get_account(account_id(), undefined | clock()) -> account(). +-spec get_account(account_id(), undefined | clock()) -> {ok, account()} | {error, not_ready | {account_not_found, _}}. get_account(AccountID, Clock) -> case call_accounter('GetAccountByID', {AccountID, to_accounter_clock(Clock)}) of {ok, Result} -> - construct_account(AccountID, Result); - {exception, #shumaich_AccountNotFound{}} -> - hg_woody_wrapper:raise(#payproc_AccountNotFound{}) + {ok, construct_account(AccountID, Result)}; + {error, _} = Error -> + Error end. --spec get_balance(account_id()) -> balance(). +-spec get_balance(account_id()) -> {ok, balance()} | {error, not_ready | {account_not_found, _}}. get_balance(AccountID) -> get_balance(AccountID, undefined). --spec get_balance(account_id(), undefined | clock()) -> balance(). +-spec get_balance(account_id(), undefined | clock()) -> {ok, balance()} | {error, not_ready | {account_not_found, _}}. get_balance(AccountID, Clock) -> case call_accounter('GetBalanceByID', {AccountID, to_accounter_clock(Clock)}) of {ok, Result} -> - construct_balance(AccountID, Result); - {exception, #shumaich_AccountNotFound{}} -> - hg_woody_wrapper:raise(#payproc_AccountNotFound{}) + {ok, construct_balance(AccountID, Result)}; + {error, _} = Error -> + Error end. -spec create_account(currency_code()) -> account_id(). @@ -124,53 +125,51 @@ collect_external_account_map(Payment, VS, Revision, Acc) -> hg_accounting:collect_external_account_map(Payment, VS, Revision, Acc). %% --spec plan(plan_id(), [batch()], hg_datetime:timestamp()) -> clock(). +-spec plan(plan_id(), [batch()], hg_datetime:timestamp()) -> {ok, clock()} | {error, {invalid_posting_params, _}}. plan(_PlanID, [], _Timestamp) -> error(badarg); plan(_PlanID, Batches, _Timestamp) when not is_list(Batches) -> error(badarg); plan(PlanID, Batches, Timestamp) -> - lists:foldl( - fun(Batch, ClockAcc) -> hold(PlanID, Batch, Timestamp, ClockAcc) end, - undefined, - Batches - ). + execute_plan(PlanID, Batches, Timestamp, undefined). --spec plan(plan_id(), [batch()], hg_datetime:timestamp(), clock()) -> clock(). +-spec plan(plan_id(), [batch()], hg_datetime:timestamp(), clock()) -> + {ok, clock()} | {error, {invalid_posting_params, _}}. plan(_PlanID, [], _Timestamp, _Clock) -> error(badarg); plan(_PlanID, Batches, _Timestamp, _Clock) when not is_list(Batches) -> error(badarg); plan(PlanID, Batches, Timestamp, Clock) -> - lists:foldl( - fun(Batch, ClockAcc) -> hold(PlanID, Batch, Timestamp, ClockAcc) end, - Clock, - Batches - ). + execute_plan(PlanID, Batches, Timestamp, Clock). --spec hold(plan_id(), batch(), hg_datetime:timestamp()) -> clock(). +-spec hold(plan_id(), batch(), hg_datetime:timestamp()) -> {ok, clock()} | {error, {invalid_posting_params, _}}. hold(PlanID, Batch, Timestamp) -> do('Hold', construct_plan_change(PlanID, Batch, Timestamp)). --spec hold(plan_id(), batch(), hg_datetime:timestamp(), clock() | undefined) -> clock(). -hold(PlanID, Batches, Timestamp, Clock) -> +-spec hold(plan_id(), batch(), hg_datetime:timestamp(), clock() | undefined) -> + {ok, clock()} | {error, {invalid_posting_params, _}}. +hold(PlanID, Batch, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), - do('Hold', construct_plan_change(PlanID, Batches, Timestamp), AccounterClock). + do('Hold', construct_plan_change(PlanID, Batch, Timestamp), AccounterClock). --spec commit(plan_id(), [batch()], hg_datetime:timestamp()) -> clock(). +-spec commit(plan_id(), [batch()], hg_datetime:timestamp()) -> + {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. commit(PlanID, Batches, Timestamp) -> do('CommitPlan', construct_plan(PlanID, Batches, Timestamp)). --spec commit(plan_id(), [batch()], hg_datetime:timestamp(), clock() | undefined) -> clock(). +-spec commit(plan_id(), [batch()], hg_datetime:timestamp(), clock() | undefined) -> + {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. commit(PlanID, Batches, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('CommitPlan', construct_plan(PlanID, Batches, Timestamp), AccounterClock). --spec rollback(plan_id(), [batch()], hg_datetime:timestamp()) -> clock(). +-spec rollback(plan_id(), [batch()], hg_datetime:timestamp()) -> + {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. rollback(PlanID, Batches, Timestamp) -> do('RollbackPlan', construct_plan(PlanID, Batches, Timestamp)). --spec rollback(plan_id(), [batch()], hg_datetime:timestamp(), clock() | undefined) -> clock(). +-spec rollback(plan_id(), [batch()], hg_datetime:timestamp(), clock() | undefined) -> + {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. rollback(PlanID, Batches, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('RollbackPlan', construct_plan(PlanID, Batches, Timestamp), AccounterClock). @@ -181,9 +180,19 @@ do(Op, Plan) -> do(Op, Plan, PreviousClock) -> case call_accounter(Op, {Plan, PreviousClock}) of {ok, Clock} -> - to_domain_clock(Clock); - {exception, Exception} -> - erlang:throw(Exception) + {ok, to_domain_clock(Clock)}; + {error, _} = Error -> + Error + end. + +execute_plan(_PlanID, [], _Timestamp, Clock) -> + {ok, Clock}; +execute_plan(PlanID, [Batch | Rest], Timestamp, Clock) -> + case hold(PlanID, Batch, Timestamp, Clock) of + {ok, NewClock} -> + execute_plan(PlanID, Rest, Timestamp, NewClock); + {error, _} = Error -> + Error end. construct_plan_change(PlanID, {BatchID, Cashflow}, Timestamp) -> @@ -282,17 +291,34 @@ call_service_with_retry(Function, Args) -> case call_service(Function, Args) of {ok, _} = Ok -> {return, Ok}; - {exception, #shumaich_NotReady{}} = Exception -> - {retry, Exception}; - {exception, _} = Exception -> - {return, Exception} + {error, ErrorType} = Error -> + {map_error_action(ErrorType), Error} end end, get_retry_strategy(Function) ). call_service(Function, Args) -> - hg_woody_wrapper:call(accounter_new, Function, Args). + case hg_woody_wrapper:call(accounter_new, Function, Args) of + {ok, _} = Ok -> + Ok; + {exception, Exception} -> + {error, map_exception(Exception)} + end. + +map_error_action(not_ready) -> + retry; +map_error_action(_) -> + return. + +map_exception(#shumaich_NotReady{}) -> + not_ready; +map_exception(#shumaich_AccountNotFound{account_id = AccountID}) -> + {account_not_found, AccountID}; +map_exception(#shumaich_PlanNotFound{plan_id = PlanID}) -> + {plan_not_found, PlanID}; +map_exception(#shumaich_InvalidPostingParams{wrong_postings = WrongPostings}) -> + {invalid_posting_params, WrongPostings}. to_domain_clock({latest, #shumaich_LatestClock{}}) -> undefined; From 230cae8736a55a26ab4510a7fbdb24e5362b06bf Mon Sep 17 00:00:00 2001 From: Kehitt Date: Tue, 21 Sep 2021 17:19:07 +0300 Subject: [PATCH 30/40] work around a shumway bug --- apps/hellgate/src/hg_accounting_new.erl | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 39982ff35..e859f501e 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -275,24 +275,18 @@ construct_balance( %% -call_accounter(Function, Args) when - Function =:= 'CommitPlan'; - Function =:= 'RollbackPlan'; - Function =:= 'GetBalanceByID'; - Function =:= 'GetAccountByID' --> - call_service_with_retry(Function, Args); call_accounter(Function, Args) -> - call_service(Function, Args). - -call_service_with_retry(Function, Args) -> hg_retry:call_with_retry( fun() -> - case call_service(Function, Args) of + try call_service(Function, Args) of {ok, _} = Ok -> {return, Ok}; {error, ErrorType} = Error -> {map_error_action(ErrorType), Error} + catch + %% @FIXME: Because apparently `Hold` still throws NotReady exception internally + error:{woody_error, {external, result_unexpected, <<"internal thrift application error">>}} -> + {retry, {error, not_ready}} end end, get_retry_strategy(Function) From 2ffa2886bb0db8ffb82b715862cd8a33fc26f7f7 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Tue, 21 Sep 2021 17:31:35 +0300 Subject: [PATCH 31/40] add some logging --- apps/hellgate/src/hg_accounting_new.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index e859f501e..c1e7ac6ba 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -286,6 +286,7 @@ call_accounter(Function, Args) -> catch %% @FIXME: Because apparently `Hold` still throws NotReady exception internally error:{woody_error, {external, result_unexpected, <<"internal thrift application error">>}} -> + _ = logger:warning("Accounter has crashed, retrying"), {retry, {error, not_ready}} end end, From 8a1e73cd34f04654c9f44542b665dd3ad78bc06f Mon Sep 17 00:00:00 2001 From: Kehitt Date: Wed, 22 Sep 2021 19:11:10 +0300 Subject: [PATCH 32/40] fix types --- apps/hellgate/src/hg_accounting_new.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index c1e7ac6ba..dbed8ce0e 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -125,7 +125,8 @@ collect_external_account_map(Payment, VS, Revision, Acc) -> hg_accounting:collect_external_account_map(Payment, VS, Revision, Acc). %% --spec plan(plan_id(), [batch()], hg_datetime:timestamp()) -> {ok, clock()} | {error, {invalid_posting_params, _}}. +-spec plan(plan_id(), [batch()], hg_datetime:timestamp()) -> + {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. plan(_PlanID, [], _Timestamp) -> error(badarg); plan(_PlanID, Batches, _Timestamp) when not is_list(Batches) -> @@ -134,7 +135,7 @@ plan(PlanID, Batches, Timestamp) -> execute_plan(PlanID, Batches, Timestamp, undefined). -spec plan(plan_id(), [batch()], hg_datetime:timestamp(), clock()) -> - {ok, clock()} | {error, {invalid_posting_params, _}}. + {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. plan(_PlanID, [], _Timestamp, _Clock) -> error(badarg); plan(_PlanID, Batches, _Timestamp, _Clock) when not is_list(Batches) -> @@ -142,7 +143,8 @@ plan(_PlanID, Batches, _Timestamp, _Clock) when not is_list(Batches) -> plan(PlanID, Batches, Timestamp, Clock) -> execute_plan(PlanID, Batches, Timestamp, Clock). --spec hold(plan_id(), batch(), hg_datetime:timestamp()) -> {ok, clock()} | {error, {invalid_posting_params, _}}. +-spec hold(plan_id(), batch(), hg_datetime:timestamp()) -> + {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. hold(PlanID, Batch, Timestamp) -> do('Hold', construct_plan_change(PlanID, Batch, Timestamp)). From 42efffae324201b8762c1cae0841596906541612 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Wed, 22 Sep 2021 21:59:09 +0300 Subject: [PATCH 33/40] another one --- apps/hellgate/src/hg_accounting_new.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index dbed8ce0e..774604f6c 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -149,7 +149,7 @@ hold(PlanID, Batch, Timestamp) -> do('Hold', construct_plan_change(PlanID, Batch, Timestamp)). -spec hold(plan_id(), batch(), hg_datetime:timestamp(), clock() | undefined) -> - {ok, clock()} | {error, {invalid_posting_params, _}}. + {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. hold(PlanID, Batch, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('Hold', construct_plan_change(PlanID, Batch, Timestamp), AccounterClock). From 8d70d1427bf0ff8f7b894b0b7e0ad649686d2bbc Mon Sep 17 00:00:00 2001 From: Kehitt Date: Fri, 24 Sep 2021 12:45:52 +0300 Subject: [PATCH 34/40] update shumaich to fix the Hold bug --- apps/hellgate/src/hg_accounting_new.erl | 7 +------ docker-compose.sh | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 774604f6c..609a12fd8 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -280,16 +280,11 @@ construct_balance( call_accounter(Function, Args) -> hg_retry:call_with_retry( fun() -> - try call_service(Function, Args) of + case call_service(Function, Args) of {ok, _} = Ok -> {return, Ok}; {error, ErrorType} = Error -> {map_error_action(ErrorType), Error} - catch - %% @FIXME: Because apparently `Hold` still throws NotReady exception internally - error:{woody_error, {external, result_unexpected, <<"internal thrift application error">>}} -> - _ = logger:warning("Accounter has crashed, retrying"), - {retry, {error, not_ready}} end end, get_retry_strategy(Function) diff --git a/docker-compose.sh b/docker-compose.sh index a8e9d5582..f949f57c2 100755 --- a/docker-compose.sh +++ b/docker-compose.sh @@ -137,7 +137,7 @@ services: kafka-topics --create --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic operation_log'" shumaich: - image: dr2.rbkmoney.com/rbkmoney/shumaich:1e4ebe41a9aaae0c46b1c41edffb95f7d93c5f48 + image: dr2.rbkmoney.com/rbkmoney/shumaich:3be4048303d9a649027faa95d87a5ecd99af1e6b hostname: shumaich container_name: shumaich restart: on-failure From 55eca2f2a7c22be90147a997919f0c884aa25922 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Tue, 28 Sep 2021 11:25:26 +0300 Subject: [PATCH 35/40] bump proto --- rebar.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.lock b/rebar.lock index af2237d92..33e73bacb 100644 --- a/rebar.lock +++ b/rebar.lock @@ -86,7 +86,7 @@ 0}, {<<"shumaich_proto">>, {git,"git@github.com:rbkmoney/shumaich-proto.git", - {ref,"be2c835aff55c90927df45d6fb85597b16cff747"}}, + {ref,"32f6ad35d6ed9e432aadbf66cbb075c23d71fbc1"}}, 0}, {<<"shumpune_proto">>, {git,"https://github.com/rbkmoney/shumpune-proto.git", From 91f62ac62d33749cd474ea933c058d168f594523 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Wed, 29 Sep 2021 13:37:00 +0300 Subject: [PATCH 36/40] bump shumway in tests --- docker-compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.sh b/docker-compose.sh index f949f57c2..566e1562d 100755 --- a/docker-compose.sh +++ b/docker-compose.sh @@ -59,7 +59,7 @@ services: condition: service_healthy shumway: - image: dr2.rbkmoney.com/rbkmoney/shumway:2f7d381d36ec69cfc90c77996f7e82b79d89e80b + image: dr2.rbkmoney.com/rbkmoney/shumway:8ab8dae452106acfd33c855ca20c0930f4ce2f7c hostname: shumway container_name: shumway ports: From b9298a461fac7488ece72e21fed46a182bc100a0 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Tue, 5 Oct 2021 18:21:53 +0300 Subject: [PATCH 37/40] some review fixes --- apps/hellgate/src/hg_accounting_new.erl | 14 +------------- apps/hellgate/src/hg_retry.erl | 15 +++++++++------ docker-compose.sh | 14 ++++---------- rebar.config | 3 +-- 4 files changed, 15 insertions(+), 31 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index 609a12fd8..fdafcfe3e 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -13,24 +13,19 @@ -export([get_balance/1]). -export([get_balance/2]). --export([create_account/1]). - -export([collect_account_map/6]). -export([collect_merchant_account_map/2]). -export([collect_provider_account_map/3]). -export([collect_system_account_map/4]). -export([collect_external_account_map/4]). --export([hold/3]). -export([hold/4]). -export([plan/3]). -export([plan/4]). --export([commit/3]). -export([commit/4]). --export([rollback/3]). -export([rollback/4]). -include_lib("damsel/include/dmsl_payment_processing_thrift.hrl"). @@ -97,13 +92,6 @@ get_balance(AccountID, Clock) -> Error end. --spec create_account(currency_code()) -> account_id(). -create_account(_CurrencyCode) -> - WoodyCtx = hg_context:get_woody_context(hg_context:load()), - % FIXME: placeholder, the sequence id should probably be passed externally - % not sure about the minimum too - hg_utils:gen_sequence(<<"create_shumaich_account">>, WoodyCtx, #{minimum => 10000}). - -spec collect_account_map(payment(), shop(), payment_institution(), provider(), varset(), revision()) -> map(). collect_account_map(Payment, Shop, PaymentInstitution, Provider, VS, Revision) -> hg_accounting:collect_account_map(Payment, Shop, PaymentInstitution, Provider, VS, Revision). @@ -278,7 +266,7 @@ construct_balance( %% call_accounter(Function, Args) -> - hg_retry:call_with_retry( + hg_retry:apply( fun() -> case call_service(Function, Args) of {ok, _} = Ok -> diff --git a/apps/hellgate/src/hg_retry.erl b/apps/hellgate/src/hg_retry.erl index 0e6a6d7ef..fdc5d1180 100644 --- a/apps/hellgate/src/hg_retry.erl +++ b/apps/hellgate/src/hg_retry.erl @@ -4,7 +4,7 @@ next_step/1, skip_steps/2, new_strategy/1, - call_with_retry/2 + apply/2 ]). -type retries_num() :: pos_integer() | infinity. @@ -59,11 +59,14 @@ skip_steps(Strategy, N) when N > 0 -> end, skip_steps(NewStrategy, N - 1). --type retry_fun_return() :: term() | no_return(). --type retry_fun() :: fun(() -> retry_fun_return()). +-type retry_fun_result() :: any(). +-type retry_fun_action() :: retry | return. +-type retry_fun() :: fun(() -> {retry_fun_action(), retry_fun_result()}). --spec call_with_retry(retry_fun(), strategy()) -> retry_fun_return(). -call_with_retry(Fun, Strategy) -> +-compile({no_auto_import, [apply/2]}). + +-spec apply(retry_fun(), strategy()) -> retry_fun_result(). +apply(Fun, Strategy) -> case Fun() of {return, Result} -> Result; @@ -71,7 +74,7 @@ call_with_retry(Fun, Strategy) -> case next_step(Strategy) of {wait, Timeout, NextStrategy} -> _ = timer:sleep(Timeout), - call_with_retry(Fun, NextStrategy); + apply(Fun, NextStrategy); finish -> Error end diff --git a/docker-compose.sh b/docker-compose.sh index 566e1562d..496f86a14 100755 --- a/docker-compose.sh +++ b/docker-compose.sh @@ -132,9 +132,10 @@ services: container_name: kafka-setup depends_on: - broker - command: "bash -c 'echo Waiting for Kafka to be ready... && \ - cub kafka-ready -b broker:9092 1 60 && \ - kafka-topics --create --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic operation_log'" + command: > + bash -c 'echo Waiting for Kafka to be ready... && + cub kafka-ready -b broker:9092 1 60 && + kafka-topics --create --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic operation_log'" shumaich: image: dr2.rbkmoney.com/rbkmoney/shumaich:3be4048303d9a649027faa95d87a5ecd99af1e6b @@ -163,13 +164,6 @@ services: timeout: 1s retries: 20 - holmes: - image: dr2.rbkmoney.com/rbkmoney/holmes:7d496d0886a1489044c57eee4ba4bfcf8f8b6a48 - hostname: holmes - container_name: holmes - volumes: - - ./_build/default/lib/shumpune-proto/proto/:/opt/holmes/shumaich-proto - postgres: image: postgres:9.6 environment: diff --git a/rebar.config b/rebar.config index f248bff85..d3b489ed8 100644 --- a/rebar.config +++ b/rebar.config @@ -100,8 +100,7 @@ ]}. {plugins, [ - {erlfmt, "0.10.0"}, - {rebar3_thrift_compiler, {git, "https://github.com/rbkmoney/rebar3_thrift_compiler.git", {tag, "0.3.1"}}} + {erlfmt, "0.10.0"} ]}. {erlfmt, [ From bc9deda873c1a23813f64672331b8c8145fa2ed3 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Tue, 5 Oct 2021 19:03:22 +0300 Subject: [PATCH 38/40] fixes for review fixes --- apps/hellgate/src/hg_accounting_new.erl | 18 ------------------ rebar.config | 3 ++- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/apps/hellgate/src/hg_accounting_new.erl b/apps/hellgate/src/hg_accounting_new.erl index fdafcfe3e..6ae932b1e 100644 --- a/apps/hellgate/src/hg_accounting_new.erl +++ b/apps/hellgate/src/hg_accounting_new.erl @@ -131,42 +131,24 @@ plan(_PlanID, Batches, _Timestamp, _Clock) when not is_list(Batches) -> plan(PlanID, Batches, Timestamp, Clock) -> execute_plan(PlanID, Batches, Timestamp, Clock). --spec hold(plan_id(), batch(), hg_datetime:timestamp()) -> - {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. -hold(PlanID, Batch, Timestamp) -> - do('Hold', construct_plan_change(PlanID, Batch, Timestamp)). - -spec hold(plan_id(), batch(), hg_datetime:timestamp(), clock() | undefined) -> {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. hold(PlanID, Batch, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('Hold', construct_plan_change(PlanID, Batch, Timestamp), AccounterClock). --spec commit(plan_id(), [batch()], hg_datetime:timestamp()) -> - {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. -commit(PlanID, Batches, Timestamp) -> - do('CommitPlan', construct_plan(PlanID, Batches, Timestamp)). - -spec commit(plan_id(), [batch()], hg_datetime:timestamp(), clock() | undefined) -> {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. commit(PlanID, Batches, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('CommitPlan', construct_plan(PlanID, Batches, Timestamp), AccounterClock). --spec rollback(plan_id(), [batch()], hg_datetime:timestamp()) -> - {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. -rollback(PlanID, Batches, Timestamp) -> - do('RollbackPlan', construct_plan(PlanID, Batches, Timestamp)). - -spec rollback(plan_id(), [batch()], hg_datetime:timestamp(), clock() | undefined) -> {ok, clock()} | {error, not_ready | {invalid_posting_params, _}}. rollback(PlanID, Batches, Timestamp, Clock) -> AccounterClock = to_accounter_clock(Clock), do('RollbackPlan', construct_plan(PlanID, Batches, Timestamp), AccounterClock). -do(Op, Plan) -> - do(Op, Plan, {latest, #shumaich_LatestClock{}}). - do(Op, Plan, PreviousClock) -> case call_accounter(Op, {Plan, PreviousClock}) of {ok, Clock} -> diff --git a/rebar.config b/rebar.config index d3b489ed8..f248bff85 100644 --- a/rebar.config +++ b/rebar.config @@ -100,7 +100,8 @@ ]}. {plugins, [ - {erlfmt, "0.10.0"} + {erlfmt, "0.10.0"}, + {rebar3_thrift_compiler, {git, "https://github.com/rbkmoney/rebar3_thrift_compiler.git", {tag, "0.3.1"}}} ]}. {erlfmt, [ From a8afcf9046015e0217923e57ee8811e1f246ccd1 Mon Sep 17 00:00:00 2001 From: Kehitt Date: Tue, 5 Oct 2021 19:36:04 +0300 Subject: [PATCH 39/40] fix docker-compose --- docker-compose.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.sh b/docker-compose.sh index 496f86a14..16187d518 100755 --- a/docker-compose.sh +++ b/docker-compose.sh @@ -135,7 +135,7 @@ services: command: > bash -c 'echo Waiting for Kafka to be ready... && cub kafka-ready -b broker:9092 1 60 && - kafka-topics --create --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic operation_log'" + kafka-topics --create --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 --topic operation_log' shumaich: image: dr2.rbkmoney.com/rbkmoney/shumaich:3be4048303d9a649027faa95d87a5ecd99af1e6b From d74d2d1154188b0c8a3f0daf9f9c659ce463d1aa Mon Sep 17 00:00:00 2001 From: Kehitt Date: Wed, 6 Oct 2021 14:17:41 +0300 Subject: [PATCH 40/40] remove constant container names --- docker-compose.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docker-compose.sh b/docker-compose.sh index 16187d518..bd5009b53 100755 --- a/docker-compose.sh +++ b/docker-compose.sh @@ -61,7 +61,6 @@ services: shumway: image: dr2.rbkmoney.com/rbkmoney/shumway:8ab8dae452106acfd33c855ca20c0930f4ce2f7c hostname: shumway - container_name: shumway ports: - "8022:8022" environment: @@ -87,7 +86,6 @@ services: zookeeper: image: confluentinc/cp-zookeeper:5.0.1 hostname: zookeeper - container_name: zookeeper environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 @@ -104,7 +102,6 @@ services: broker: image: confluentinc/cp-enterprise-kafka:5.0.1 hostname: broker - container_name: broker depends_on: - zookeeper environment: @@ -129,7 +126,6 @@ services: kafka-setup: image: confluentinc/cp-kafka:5.1.1 hostname: kafka-setup - container_name: kafka-setup depends_on: - broker command: > @@ -140,7 +136,6 @@ services: shumaich: image: dr2.rbkmoney.com/rbkmoney/shumaich:3be4048303d9a649027faa95d87a5ecd99af1e6b hostname: shumaich - container_name: shumaich restart: on-failure environment: SPRING_APPLICATION_JSON: '{