From 51d12b669647c318f8a4388fcf52c7533e7d2e1f Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 12:24:27 +0300 Subject: [PATCH 01/11] TD-686: Adopts opentelemetry API --- apps/kds/src/kds.app.src | 5 +++- rebar.config | 8 +++++-- rebar.lock | 52 +++++++++++++++++++++++++++++++++------- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/apps/kds/src/kds.app.src b/apps/kds/src/kds.app.src index 6a070b1..80728bb 100644 --- a/apps/kds/src/kds.app.src +++ b/apps/kds/src/kds.app.src @@ -14,7 +14,10 @@ msgpack, lib_combin, jose, - cds_proto + cds_proto, + opentelemetry_api, + opentelemetry_exporter, + opentelemetry ]}, {env, []}, {modules, []}, diff --git a/rebar.config b/rebar.config index 8985de7..133657f 100644 --- a/rebar.config +++ b/rebar.config @@ -37,7 +37,10 @@ {erl_health, {git, "https://github.com/valitydev/erlang-health.git", {branch, master}}}, {cds_proto, {git, "https://github.com/valitydev/cds-proto.git", {branch, master}}}, {msgpack, {git, "https://github.com/valitydev/msgpack-erlang", {branch, master}}}, - {scoper, {git, "https://github.com/valitydev/scoper.git", {branch, master}}} + {scoper, {git, "https://github.com/valitydev/scoper.git", {branch, master}}}, + {opentelemetry_api, "1.2.1"}, + {opentelemetry, "1.3.0"}, + {opentelemetry_exporter, "1.3.0"} ]}. %% XRef checks @@ -73,7 +76,7 @@ {prometheus, "4.8.1"}, {prometheus_cowboy, "0.1.8"}, {logger_logstash_formatter, - {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "2c7b716"}}}, + {git, "https://github.com/valitydev/logger_logstash_formatter.git", {ref, "08a66a6"}}}, {iosetopts, {git, "https://github.com/valitydev/iosetopts.git", {ref, "edb445c"}}} ]}, {relx, [ @@ -82,6 +85,7 @@ {recon, load}, {runtime_tools, load}, {tools, load}, + {opentelemetry, temporary}, {logger_logstash_formatter, load}, prometheus, prometheus_cowboy, diff --git a/rebar.lock b/rebar.lock index 26fdd79..1f9881a 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,5 +1,6 @@ {"1.2.0", -[{<<"cache">>,{pkg,<<"cache">>,<<"2.3.3">>},1}, +[{<<"acceptor_pool">>,{pkg,<<"acceptor_pool">>,<<"1.0.0">>},2}, + {<<"cache">>,{pkg,<<"cache">>,<<"2.3.3">>},1}, {<<"cds_proto">>, {git,"https://github.com/valitydev/cds-proto.git", {ref,"ed9f9078049ebcd1439d2ada0479fc8f33ccacf1"}}, @@ -9,18 +10,22 @@ {git,"https://github.com/rbkmoney/cg_mon.git", {ref,"5a87a37694e42b6592d3b4164ae54e0e87e24e18"}}, 1}, + {<<"chatterbox">>,{pkg,<<"ts_chatterbox">>,<<"0.13.0">>},2}, {<<"cowboy">>,{pkg,<<"cowboy">>,<<"2.9.0">>},1}, {<<"cowlib">>,{pkg,<<"cowlib">>,<<"2.11.0">>},2}, + {<<"ctx">>,{pkg,<<"ctx">>,<<"0.6.0">>},2}, {<<"erl_health">>, {git,"https://github.com/valitydev/erlang-health.git", - {ref,"5958e2f35cd4d09f40685762b82b82f89b4d9333"}}, + {ref,"7ffbc855bdbe79e23efad1803b0b185c9ea8d2f1"}}, 0}, {<<"genlib">>, {git,"https://github.com/valitydev/genlib.git", - {ref,"82c5ff3866e3019eb347c7f1d8f1f847bed28c10"}}, + {ref,"f6074551d6586998e91a97ea20acb47241254ff3"}}, 0}, {<<"gproc">>,{pkg,<<"gproc">>,<<"0.9.0">>},1}, + {<<"grpcbox">>,{pkg,<<"grpcbox">>,<<"0.16.0">>},1}, {<<"hackney">>,{pkg,<<"hackney">>,<<"1.18.0">>},1}, + {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},3}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},2}, {<<"jose">>, {git,"https://github.com/potatosalad/erlang-jose.git", @@ -34,12 +39,20 @@ {git,"https://github.com/valitydev/msgpack-erlang", {ref,"9d56647ed77498c7655da39891c4985142697083"}}, 0}, + {<<"opentelemetry">>,{pkg,<<"opentelemetry">>,<<"1.3.0">>},0}, + {<<"opentelemetry_api">>,{pkg,<<"opentelemetry_api">>,<<"1.2.1">>},0}, + {<<"opentelemetry_exporter">>, + {pkg,<<"opentelemetry_exporter">>,<<"1.3.0">>}, + 0}, + {<<"opentelemetry_semantic_conventions">>, + {pkg,<<"opentelemetry_semantic_conventions">>,<<"0.2.0">>}, + 1}, {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.3.1">>},2}, {<<"proper">>,{pkg,<<"proper">>,<<"1.3.0">>},1}, {<<"ranch">>,{pkg,<<"ranch">>,<<"1.8.0">>},2}, {<<"scoper">>, {git,"https://github.com/valitydev/scoper.git", - {ref,"7f3183df279bc8181efe58dafd9cae164f495e6f"}}, + {ref,"41a14a558667316998af9f49149ee087ffa8bef2"}}, 0}, {<<"shamir">>, {git,"https://github.com/valitydev/shamir.git", @@ -49,49 +62,72 @@ {git,"https://github.com/valitydev/snowflake.git", {ref,"de159486ef40cec67074afe71882bdc7f7deab72"}}, 1}, - {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.6">>},2}, + {<<"ssl_verify_fun">>,{pkg,<<"ssl_verify_fun">>,<<"1.1.7">>},2}, {<<"thrift">>, {git,"https://github.com/valitydev/thrift_erlang.git", {ref,"c280ff266ae1c1906fb0dcee8320bb8d8a4a3c75"}}, 1}, + {<<"tls_certificate_check">>, + {pkg,<<"tls_certificate_check">>,<<"1.20.0">>}, + 1}, {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.7.0">>},2}, {<<"woody">>, {git,"https://github.com/valitydev/woody_erlang.git", - {ref,"3ddacb9296691aa8ddad05498d1fd34b078eda75"}}, + {ref,"5d46291a6bfcee0bae2a9346a7d927603a909249"}}, 0}]}. [ {pkg_hash,[ + {<<"acceptor_pool">>, <<"43C20D2ACAE35F0C2BCD64F9D2BDE267E459F0F3FD23DAB26485BF518C281B21">>}, {<<"cache">>, <<"B23A5FE7095445A88412A6E614C933377E0137B44FFED77C9B3FEF1A731A20B2">>}, {<<"certifi">>, <<"D4FB0A6BB20B7C9C3643E22507E42F356AC090A1DCEA9AB99E27E0376D695EBA">>}, + {<<"chatterbox">>, <<"6F059D97BCAA758B8EA6FFFE2B3B81362BD06B639D3EA2BB088335511D691EBF">>}, {<<"cowboy">>, <<"865DD8B6607E14CF03282E10E934023A1BD8BE6F6BACF921A7E2A96D800CD452">>}, {<<"cowlib">>, <<"0B9FF9C346629256C42EBE1EEB769A83C6CB771A6EE5960BD110AB0B9B872063">>}, + {<<"ctx">>, <<"8FF88B70E6400C4DF90142E7F130625B82086077A45364A78D208ED3ED53C7FE">>}, {<<"gproc">>, <<"853CCB7805E9ADA25D227A157BA966F7B34508F386A3E7E21992B1B484230699">>}, + {<<"grpcbox">>, <<"B83F37C62D6EECA347B77F9B1EC7E9F62231690CDFEB3A31BE07CD4002BA9C82">>}, {<<"hackney">>, <<"C4443D960BB9FBA6D01161D01CD81173089686717D9490E5D3606644C48D121F">>}, + {<<"hpack">>, <<"17670F83FF984AE6CD74B1C456EDDE906D27FF013740EE4D9EFAA4F1BF999633">>}, {<<"idna">>, <<"8A63070E9F7D0C62EB9D9FCB360A7DE382448200FBBD1B106CC96D3D8099DF8D">>}, {<<"jsx">>, <<"D12516BAA0BB23A59BB35DCCAF02A1BD08243FCBB9EFE24F2D9D056CCFF71268">>}, {<<"lib_combin">>, <<"00F241FDCB6AFFFC1DE109A61A216C07E4E51C52CCB165656565BF660E7C78EB">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, + {<<"opentelemetry">>, <<"988AC3C26ACAC9720A1D4FB8D9DC52E95B45ECFEC2D5B5583276A09E8936BC5E">>}, + {<<"opentelemetry_api">>, <<"7B69ED4F40025C005DE0B74FCE8C0549625D59CB4DF12D15C32FE6DC5076FF42">>}, + {<<"opentelemetry_exporter">>, <<"1D8809C0D4F4ACF986405F7700ED11992BCBDB6A4915DD11921E80777FFA7167">>}, + {<<"opentelemetry_semantic_conventions">>, <<"B67FE459C2938FCAB341CB0951C44860C62347C005ACE1B50F8402576F241435">>}, {<<"parse_trans">>, <<"16328AB840CC09919BD10DAB29E431DA3AF9E9E7E7E6F0089DD5A2D2820011D8">>}, {<<"proper">>, <<"C1ACD51C51DA17A2FE91D7A6FC6A0C25A6A9849D8DC77093533109D1218D8457">>}, {<<"ranch">>, <<"8C7A100A139FD57F17327B6413E4167AC559FBC04CA7448E9BE9057311597A1D">>}, - {<<"ssl_verify_fun">>, <<"CF344F5692C82D2CD7554F5EC8FD961548D4FD09E7D22F5B62482E5AEAEBD4B0">>}, + {<<"ssl_verify_fun">>, <<"354C321CF377240C7B8716899E182CE4890C5938111A1296ADD3EC74CF1715DF">>}, + {<<"tls_certificate_check">>, <<"1AC0C53F95E201FEB8D398EF9D764AE74175231289D89F166BA88A7F50CD8E73">>}, {<<"unicode_util_compat">>, <<"BC84380C9AB48177092F43AC89E4DFA2C6D62B40B8BD132B1059ECC7232F9A78">>}]}, {pkg_hash_ext,[ + {<<"acceptor_pool">>, <<"0CBCD83FDC8B9AD2EEE2067EF8B91A14858A5883CB7CD800E6FCD5803E158788">>}, {<<"cache">>, <<"44516CE6FA03594D3A2AF025DD3A87BFE711000EB730219E1DDEFC816E0AA2F4">>}, {<<"certifi">>, <<"6AC7EFC1C6F8600B08D625292D4BBF584E14847CE1B6B5C44D983D273E1097EA">>}, + {<<"chatterbox">>, <<"B93D19104D86AF0B3F2566C4CBA2A57D2E06D103728246BA1AC6C3C0FF010AA7">>}, {<<"cowboy">>, <<"2C729F934B4E1AA149AFF882F57C6372C15399A20D54F65C8D67BEF583021BDE">>}, {<<"cowlib">>, <<"2B3E9DA0B21C4565751A6D4901C20D1B4CC25CBB7FD50D91D2AB6DD287BC86A9">>}, + {<<"ctx">>, <<"A14ED2D1B67723DBEBBE423B28D7615EB0BDCBA6FF28F2D1F1B0A7E1D4AA5FC2">>}, {<<"gproc">>, <<"587E8AF698CCD3504CF4BA8D90F893EDE2B0F58CABB8A916E2BF9321DE3CF10B">>}, + {<<"grpcbox">>, <<"294DF743AE20A7E030889F00644001370A4F7CE0121F3BBDAF13CF3169C62913">>}, {<<"hackney">>, <<"9AFCDA620704D720DB8C6A3123E9848D09C87586DC1C10479C42627B905B5C5E">>}, + {<<"hpack">>, <<"06F580167C4B8B8A6429040DF36CC93BBA6D571FAEAEC1B28816523379CBB23A">>}, {<<"idna">>, <<"92376EB7894412ED19AC475E4A86F7B413C1B9FBB5BD16DCCD57934157944CEA">>}, {<<"jsx">>, <<"0C5CC8FDC11B53CC25CF65AC6705AD39E54ECC56D1C22E4ADB8F5A53FB9427F3">>}, {<<"lib_combin">>, <<"D565215B9E5B3B1EF0B1A5DEDD0CC41AEA0814D6EAD6191584B5961C9DB874D5">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>}, + {<<"opentelemetry">>, <<"8E09EDC26AAD11161509D7ECAD854A3285D88580F93B63B0B1CF0BAC332BFCC0">>}, + {<<"opentelemetry_api">>, <<"6D7A27B7CAD2AD69A09CABF6670514CAFCEC717C8441BEB5C96322BAC3D05350">>}, + {<<"opentelemetry_exporter">>, <<"2B40007F509D38361744882FD060A8841AF772AB83BB542AA5350908B303AD65">>}, + {<<"opentelemetry_semantic_conventions">>, <<"D61FA1F5639EE8668D74B527E6806E0503EFC55A42DB7B5F39939D84C07D6895">>}, {<<"parse_trans">>, <<"07CD9577885F56362D414E8C4C4E6BDF10D43A8767ABB92D24CBE8B24C54888B">>}, {<<"proper">>, <<"4AA192FCCDDD03FDBE50FEF620BE9D4D2F92635B54F55FB83AEC185994403CBC">>}, {<<"ranch">>, <<"49FBCFD3682FAB1F5D109351B61257676DA1A2FDBE295904176D5E521A2DDFE5">>}, - {<<"ssl_verify_fun">>, <<"BDB0D2471F453C88FF3908E7686F86F9BE327D065CC1EC16FA4540197EA04680">>}, + {<<"ssl_verify_fun">>, <<"FE4C190E8F37401D30167C8C405EDA19469F34577987C76DDE613E838BBC67F8">>}, + {<<"tls_certificate_check">>, <<"AB57B74B1A63DC5775650699A3EC032EC0065005EFF1F020818742B7312A8426">>}, {<<"unicode_util_compat">>, <<"25EEE6D67DF61960CF6A794239566599B09E17E668D3700247BC498638152521">>}]} ]. From b4a7377bfae7807b011ca83e3f9be765fc854cac Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 13:04:50 +0300 Subject: [PATCH 02/11] Tweaks sleep time to fit in rotation window --- apps/kds/test/kds_keyring_api_tests_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/kds/test/kds_keyring_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_api_tests_SUITE.erl index 3ad89c7..3557d8c 100644 --- a/apps/kds/test/kds_keyring_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_api_tests_SUITE.erl @@ -182,7 +182,7 @@ init(C) -> init_with_timeout(C) -> {Id, DecryptedMasterKeyShare} = partial_init(C), Timeout = genlib_app:env(kds, keyring_rotation_lifetime, 4000), - ok = timer:sleep(Timeout + 1500), + ok = timer:sleep(Timeout + 2000), _ = ?assertEqual( {error, {invalid_activity, {initialization, uninitialized}}}, kds_keyring_client:validate_init(Id, DecryptedMasterKeyShare, root_url(C)) From 2f74c9bd450224d8b004f67f9a81025cbe17599a Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 17:43:24 +0300 Subject: [PATCH 03/11] Adds 'await_*' helpers to keyring init tests --- apps/kds/test/kds_ct_utils.erl | 46 +++++++++++++++++++ apps/kds/test/kds_keyring_api_tests_SUITE.erl | 5 +- .../test/kds_keyring_meta_api_tests_SUITE.erl | 1 + .../kds_keyring_storage_api_tests_SUITE.erl | 1 + 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/apps/kds/test/kds_ct_utils.erl b/apps/kds/test/kds_ct_utils.erl index 87bf3af..b19ae55 100644 --- a/apps/kds/test/kds_ct_utils.erl +++ b/apps/kds/test/kds_ct_utils.erl @@ -11,6 +11,10 @@ -export([start_stash/1]). +-export([await_status/4]). +-export([await_initialization_phase/4]). +-export([await_matching_state/3]). + %% %% Types %% @@ -240,6 +244,48 @@ start_stash(C) -> {stash, kds_ct_stash:start()} ] ++ C. +-spec await_status(atom(), woody:url(), pos_integer(), pos_integer()) -> kds_keyring_manager:status(). +await_status(ExpectedStatus, RootUrl, Timeout, WaitTime) -> + await_matching_state(Timeout, RootUrl, fun + (#{status := Status}) when Status =:= ExpectedStatus -> + ok; + (_State) -> + {wait, WaitTime} + end). + +-spec await_initialization_phase(atom(), woody:url(), pos_integer(), pos_integer()) -> kds_keyring_manager:status(). +await_initialization_phase(ExpectedPhase, RootUrl, Timeout, WaitTime) -> + await_matching_state(Timeout, RootUrl, fun + (#{activities := #{initialization := #{phase := Phase}}}) when Phase =:= ExpectedPhase -> + ok; + (_State) -> + {wait, WaitTime} + end). + +-spec await_matching_state( + pos_integer(), woody:url(), fun((kds_keyring_manager:status()) -> ok | {wait, pos_integer()}) +) -> + kds_keyring_manager:status() | {error, timeout}. +await_matching_state(Timeout, RootUrl, F) -> + TimeoutTime = erlang:monotonic_time(millisecond) + Timeout, + {ok, State} = await_matching_state_(TimeoutTime, RootUrl, F), + State. + +await_matching_state_(TimeoutTime, RootUrl, F) -> + State = kds_keyring_client:get_state(RootUrl), + case erlang:monotonic_time(millisecond) of + Time when Time < TimeoutTime -> + case F(State) of + ok -> + {ok, State}; + {wait, WaitTime} -> + ok = timer:sleep(WaitTime), + await_matching_state(TimeoutTime, RootUrl, F) + end; + _ -> + {error, timeout} + end. + %% %% Internals %% diff --git a/apps/kds/test/kds_keyring_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_api_tests_SUITE.erl index 3557d8c..6d2824c 100644 --- a/apps/kds/test/kds_keyring_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_api_tests_SUITE.erl @@ -159,7 +159,7 @@ init(C) -> } } }, - kds_keyring_client:get_state(root_url(C)) + kds_ct_utils:await_initialization_phase(validation, root_url(C), 500, 100) ), ok = validate_init(DecryptedMasterKeyShares, C), _ = ?assertMatch( @@ -182,7 +182,7 @@ init(C) -> init_with_timeout(C) -> {Id, DecryptedMasterKeyShare} = partial_init(C), Timeout = genlib_app:env(kds, keyring_rotation_lifetime, 4000), - ok = timer:sleep(Timeout + 2000), + _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), Timeout + 2000, 200), _ = ?assertEqual( {error, {invalid_activity, {initialization, uninitialized}}}, kds_keyring_client:validate_init(Id, DecryptedMasterKeyShare, root_url(C)) @@ -192,6 +192,7 @@ init_with_timeout(C) -> init_with_cancel(C) -> {Id, DecryptedMasterKeyShare} = partial_init(C), ok = kds_keyring_client:cancel_init(root_url(C)), + _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 500, 100), _ = ?assertEqual( {error, {invalid_activity, {initialization, uninitialized}}}, kds_keyring_client:validate_init(Id, DecryptedMasterKeyShare, root_url(C)) diff --git a/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl index f3ceb7b..b7478e4 100644 --- a/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl @@ -79,6 +79,7 @@ init_check_meta(C) -> kds_keyring_client:get_keyring_meta(root_url(C)) ), _ = kds_ct_keyring:init(C), + _ = kds_ct_utils:await_status(unlocked, root_url(C), 500, 100), _ = ?assertMatch( #{keys := #{0 := #{retired := false}}, current_key_id := 0}, kds_keyring_client:get_keyring_meta(root_url(C)) diff --git a/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl index 2f9067d..8f3d864 100644 --- a/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl @@ -66,6 +66,7 @@ init_check_keyring(C) -> get_keyring(C) ), _ = kds_ct_keyring:init(C), + _ = kds_ct_utils:await_status(unlocked, management_root_url(C), 500, 100), _ = ?assertMatch( #{ meta := #{current_key_id := 0, version := 1, keys := #{0 := #{retired := false}}}, From 1d8e77433030f1e207c4e17d7e6e37f116770299 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 17:49:17 +0300 Subject: [PATCH 04/11] Fixes func call typo --- apps/kds/test/kds_ct_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/kds/test/kds_ct_utils.erl b/apps/kds/test/kds_ct_utils.erl index b19ae55..93fac9d 100644 --- a/apps/kds/test/kds_ct_utils.erl +++ b/apps/kds/test/kds_ct_utils.erl @@ -280,7 +280,7 @@ await_matching_state_(TimeoutTime, RootUrl, F) -> {ok, State}; {wait, WaitTime} -> ok = timer:sleep(WaitTime), - await_matching_state(TimeoutTime, RootUrl, F) + await_matching_state_(TimeoutTime, RootUrl, F) end; _ -> {error, timeout} From 08eb8ccfb0fb22f6e3ee056f476f06de8949bdb3 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 17:56:18 +0300 Subject: [PATCH 05/11] Adjusts timeouts in tests --- apps/kds/test/kds_keyring_api_tests_SUITE.erl | 4 ++-- apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/kds/test/kds_keyring_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_api_tests_SUITE.erl index 6d2824c..1471a03 100644 --- a/apps/kds/test/kds_keyring_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_api_tests_SUITE.erl @@ -159,7 +159,7 @@ init(C) -> } } }, - kds_ct_utils:await_initialization_phase(validation, root_url(C), 500, 100) + kds_ct_utils:await_initialization_phase(validation, root_url(C), 5000, 200) ), ok = validate_init(DecryptedMasterKeyShares, C), _ = ?assertMatch( @@ -192,7 +192,7 @@ init_with_timeout(C) -> init_with_cancel(C) -> {Id, DecryptedMasterKeyShare} = partial_init(C), ok = kds_keyring_client:cancel_init(root_url(C)), - _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 500, 100), + _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 2000, 200), _ = ?assertEqual( {error, {invalid_activity, {initialization, uninitialized}}}, kds_keyring_client:validate_init(Id, DecryptedMasterKeyShare, root_url(C)) diff --git a/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl index b7478e4..6876af8 100644 --- a/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl @@ -78,8 +78,9 @@ init_check_meta(C) -> #{keys := #{}, current_key_id := 0}, kds_keyring_client:get_keyring_meta(root_url(C)) ), + _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 2000, 200), _ = kds_ct_keyring:init(C), - _ = kds_ct_utils:await_status(unlocked, root_url(C), 500, 100), + _ = kds_ct_utils:await_status(unlocked, root_url(C), 2000, 200), _ = ?assertMatch( #{keys := #{0 := #{retired := false}}, current_key_id := 0}, kds_keyring_client:get_keyring_meta(root_url(C)) From dc2acb66ef8da783fa2e37dd625b45c752c8a44e Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 18:21:23 +0300 Subject: [PATCH 06/11] Adjusts timeouts to fix init tests --- apps/kds/test/kds_keyring_api_tests_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/kds/test/kds_keyring_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_api_tests_SUITE.erl index 1471a03..c8f461b 100644 --- a/apps/kds/test/kds_keyring_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_api_tests_SUITE.erl @@ -159,7 +159,7 @@ init(C) -> } } }, - kds_ct_utils:await_initialization_phase(validation, root_url(C), 5000, 200) + kds_ct_utils:await_status(not_initialized, root_url(C), 5000, 200) ), ok = validate_init(DecryptedMasterKeyShares, C), _ = ?assertMatch( From d5be2ad20af15e23d7de4362ab510b8b1a8934b6 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 18:29:55 +0300 Subject: [PATCH 07/11] Another timeouts --- apps/kds/test/kds_ct_utils.erl | 2 +- apps/kds/test/kds_keyring_api_tests_SUITE.erl | 4 ++-- apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl | 4 ++-- apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl | 3 ++- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/kds/test/kds_ct_utils.erl b/apps/kds/test/kds_ct_utils.erl index 93fac9d..495e878 100644 --- a/apps/kds/test/kds_ct_utils.erl +++ b/apps/kds/test/kds_ct_utils.erl @@ -96,7 +96,7 @@ start_clear(Config) -> request_timeout => 60000 }}, {shutdown_timeout, 0}, - {keyring_rotation_lifetime, 1000}, + {keyring_rotation_lifetime, 3000}, {keyring_unlock_lifetime, 1000}, {keyring_rekeying_lifetime, 3000}, {keyring_initialize_lifetime, 3000}, diff --git a/apps/kds/test/kds_keyring_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_api_tests_SUITE.erl index c8f461b..03425db 100644 --- a/apps/kds/test/kds_keyring_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_api_tests_SUITE.erl @@ -159,7 +159,7 @@ init(C) -> } } }, - kds_ct_utils:await_status(not_initialized, root_url(C), 5000, 200) + kds_ct_utils:await_initialization_phase(validation, root_url(C), 5000, 200) ), ok = validate_init(DecryptedMasterKeyShares, C), _ = ?assertMatch( @@ -192,7 +192,7 @@ init_with_timeout(C) -> init_with_cancel(C) -> {Id, DecryptedMasterKeyShare} = partial_init(C), ok = kds_keyring_client:cancel_init(root_url(C)), - _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 2000, 200), + _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 3000, 200), _ = ?assertEqual( {error, {invalid_activity, {initialization, uninitialized}}}, kds_keyring_client:validate_init(Id, DecryptedMasterKeyShare, root_url(C)) diff --git a/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl index 6876af8..42e71ac 100644 --- a/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl @@ -78,9 +78,9 @@ init_check_meta(C) -> #{keys := #{}, current_key_id := 0}, kds_keyring_client:get_keyring_meta(root_url(C)) ), - _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 2000, 200), + _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 3000, 200), _ = kds_ct_keyring:init(C), - _ = kds_ct_utils:await_status(unlocked, root_url(C), 2000, 200), + _ = kds_ct_utils:await_status(unlocked, root_url(C), 3000, 200), _ = ?assertMatch( #{keys := #{0 := #{retired := false}}, current_key_id := 0}, kds_keyring_client:get_keyring_meta(root_url(C)) diff --git a/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl index 8f3d864..2be2b49 100644 --- a/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl @@ -65,8 +65,9 @@ init_check_keyring(C) -> {error, {invalid_status, not_initialized}}, get_keyring(C) ), + _ = kds_ct_utils:await_initialization_phase(uninitialized, management_root_url(C), 3000, 200), _ = kds_ct_keyring:init(C), - _ = kds_ct_utils:await_status(unlocked, management_root_url(C), 500, 100), + _ = kds_ct_utils:await_status(unlocked, management_root_url(C), 3000, 200), _ = ?assertMatch( #{ meta := #{current_key_id := 0, version := 1, keys := #{0 := #{retired := false}}}, From 04e571461939eeb8b960e0034f7b8cdc2bacf0b0 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 18:38:44 +0300 Subject: [PATCH 08/11] Rearranges key preparation part of init test --- apps/kds/test/kds_ct_utils.erl | 2 +- apps/kds/test/kds_keyring_api_tests_SUITE.erl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/kds/test/kds_ct_utils.erl b/apps/kds/test/kds_ct_utils.erl index 495e878..93fac9d 100644 --- a/apps/kds/test/kds_ct_utils.erl +++ b/apps/kds/test/kds_ct_utils.erl @@ -96,7 +96,7 @@ start_clear(Config) -> request_timeout => 60000 }}, {shutdown_timeout, 0}, - {keyring_rotation_lifetime, 3000}, + {keyring_rotation_lifetime, 1000}, {keyring_unlock_lifetime, 1000}, {keyring_rekeying_lifetime, 3000}, {keyring_initialize_lifetime, 3000}, diff --git a/apps/kds/test/kds_keyring_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_api_tests_SUITE.erl index 03425db..8d3d251 100644 --- a/apps/kds/test/kds_keyring_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_api_tests_SUITE.erl @@ -139,11 +139,11 @@ end_per_group(_, C) -> -spec init(config()) -> _. init(C) -> Threshold = 2, - EncryptedMasterKeyShares = kds_keyring_client:start_init(Threshold, root_url(C)), Shareholders = kds_shareholder:get_all(), - _ = ?assertEqual(length(EncryptedMasterKeyShares), length(Shareholders)), EncPrivateKeys = enc_private_keys(C), SigPrivateKeys = sig_private_keys(C), + EncryptedMasterKeyShares = kds_keyring_client:start_init(Threshold, root_url(C)), + _ = ?assertEqual(length(EncryptedMasterKeyShares), length(Shareholders)), DecryptedMasterKeyShares = kds_ct_keyring:decrypt_and_sign_masterkeys( EncryptedMasterKeyShares, EncPrivateKeys, @@ -159,7 +159,7 @@ init(C) -> } } }, - kds_ct_utils:await_initialization_phase(validation, root_url(C), 5000, 200) + kds_ct_utils:await_initialization_phase(validation, root_url(C), 1000, 100) ), ok = validate_init(DecryptedMasterKeyShares, C), _ = ?assertMatch( From 8fef70c2da747d06aa8ee9cdaadc23d6729b09c2 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 18:43:54 +0300 Subject: [PATCH 09/11] Dumps krm state on timeout --- apps/kds/test/kds_ct_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/kds/test/kds_ct_utils.erl b/apps/kds/test/kds_ct_utils.erl index 93fac9d..591cc84 100644 --- a/apps/kds/test/kds_ct_utils.erl +++ b/apps/kds/test/kds_ct_utils.erl @@ -265,7 +265,7 @@ await_initialization_phase(ExpectedPhase, RootUrl, Timeout, WaitTime) -> -spec await_matching_state( pos_integer(), woody:url(), fun((kds_keyring_manager:status()) -> ok | {wait, pos_integer()}) ) -> - kds_keyring_manager:status() | {error, timeout}. + kds_keyring_manager:status() | {error, timeout, kds_keyring_manager:status()}. await_matching_state(Timeout, RootUrl, F) -> TimeoutTime = erlang:monotonic_time(millisecond) + Timeout, {ok, State} = await_matching_state_(TimeoutTime, RootUrl, F), @@ -283,7 +283,7 @@ await_matching_state_(TimeoutTime, RootUrl, F) -> await_matching_state_(TimeoutTime, RootUrl, F) end; _ -> - {error, timeout} + {error, timeout, State} end. %% From 38741021dd0ed0addf49fd957b4a65b0bae52fd6 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 18:48:51 +0300 Subject: [PATCH 10/11] Dbg init test --- apps/kds/test/kds_keyring_api_tests_SUITE.erl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/kds/test/kds_keyring_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_api_tests_SUITE.erl index 8d3d251..fbb973f 100644 --- a/apps/kds/test/kds_keyring_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_api_tests_SUITE.erl @@ -143,6 +143,19 @@ init(C) -> EncPrivateKeys = enc_private_keys(C), SigPrivateKeys = sig_private_keys(C), EncryptedMasterKeyShares = kds_keyring_client:start_init(Threshold, root_url(C)), + %% DEBUG + _ = ?assertMatch( + #{ + status := not_initialized, + activities := #{ + initialization := #{ + phase := validation, + validation_shares := #{} + } + } + }, + kds_ct_utils:await_initialization_phase(validation, root_url(C), 1000, 100) + ), _ = ?assertEqual(length(EncryptedMasterKeyShares), length(Shareholders)), DecryptedMasterKeyShares = kds_ct_keyring:decrypt_and_sign_masterkeys( EncryptedMasterKeyShares, From bdef878eb53f99fe68ef8cc5972209facfab916a Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 26 Oct 2023 18:59:48 +0300 Subject: [PATCH 11/11] Increases initialization timeout --- apps/kds/test/kds_ct_utils.erl | 2 +- apps/kds/test/kds_keyring_api_tests_SUITE.erl | 23 +++++-------------- .../test/kds_keyring_meta_api_tests_SUITE.erl | 3 ++- .../kds_keyring_storage_api_tests_SUITE.erl | 6 +++-- 4 files changed, 13 insertions(+), 21 deletions(-) diff --git a/apps/kds/test/kds_ct_utils.erl b/apps/kds/test/kds_ct_utils.erl index 591cc84..4c3291b 100644 --- a/apps/kds/test/kds_ct_utils.erl +++ b/apps/kds/test/kds_ct_utils.erl @@ -99,7 +99,7 @@ start_clear(Config) -> {keyring_rotation_lifetime, 1000}, {keyring_unlock_lifetime, 1000}, {keyring_rekeying_lifetime, 3000}, - {keyring_initialize_lifetime, 3000}, + {keyring_initialize_lifetime, 10000}, {shareholders, #{ <<"1">> => #{ owner => <<"ndiezel">>, diff --git a/apps/kds/test/kds_keyring_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_api_tests_SUITE.erl index fbb973f..e5591cd 100644 --- a/apps/kds/test/kds_keyring_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_api_tests_SUITE.erl @@ -139,29 +139,17 @@ end_per_group(_, C) -> -spec init(config()) -> _. init(C) -> Threshold = 2, + EncryptedMasterKeyShares = kds_keyring_client:start_init(Threshold, root_url(C)), Shareholders = kds_shareholder:get_all(), + _ = ?assertEqual(length(EncryptedMasterKeyShares), length(Shareholders)), EncPrivateKeys = enc_private_keys(C), SigPrivateKeys = sig_private_keys(C), - EncryptedMasterKeyShares = kds_keyring_client:start_init(Threshold, root_url(C)), - %% DEBUG - _ = ?assertMatch( - #{ - status := not_initialized, - activities := #{ - initialization := #{ - phase := validation, - validation_shares := #{} - } - } - }, - kds_ct_utils:await_initialization_phase(validation, root_url(C), 1000, 100) - ), - _ = ?assertEqual(length(EncryptedMasterKeyShares), length(Shareholders)), DecryptedMasterKeyShares = kds_ct_keyring:decrypt_and_sign_masterkeys( EncryptedMasterKeyShares, EncPrivateKeys, SigPrivateKeys ), + Timeout = genlib_app:env(kds, keyring_initialize_lifetime), _ = ?assertMatch( #{ status := not_initialized, @@ -172,7 +160,7 @@ init(C) -> } } }, - kds_ct_utils:await_initialization_phase(validation, root_url(C), 1000, 100) + kds_ct_utils:await_initialization_phase(validation, root_url(C), Timeout, 200) ), ok = validate_init(DecryptedMasterKeyShares, C), _ = ?assertMatch( @@ -205,7 +193,8 @@ init_with_timeout(C) -> init_with_cancel(C) -> {Id, DecryptedMasterKeyShare} = partial_init(C), ok = kds_keyring_client:cancel_init(root_url(C)), - _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 3000, 200), + Timeout = genlib_app:env(kds, keyring_initialize_lifetime), + _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), Timeout, 200), _ = ?assertEqual( {error, {invalid_activity, {initialization, uninitialized}}}, kds_keyring_client:validate_init(Id, DecryptedMasterKeyShare, root_url(C)) diff --git a/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl index 42e71ac..ede10b0 100644 --- a/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_meta_api_tests_SUITE.erl @@ -78,7 +78,8 @@ init_check_meta(C) -> #{keys := #{}, current_key_id := 0}, kds_keyring_client:get_keyring_meta(root_url(C)) ), - _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), 3000, 200), + Timeout = genlib_app:env(kds, keyring_initialize_lifetime), + _ = kds_ct_utils:await_initialization_phase(uninitialized, root_url(C), Timeout, 200), _ = kds_ct_keyring:init(C), _ = kds_ct_utils:await_status(unlocked, root_url(C), 3000, 200), _ = ?assertMatch( diff --git a/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl b/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl index 2be2b49..60c2dae 100644 --- a/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl +++ b/apps/kds/test/kds_keyring_storage_api_tests_SUITE.erl @@ -65,9 +65,11 @@ init_check_keyring(C) -> {error, {invalid_status, not_initialized}}, get_keyring(C) ), - _ = kds_ct_utils:await_initialization_phase(uninitialized, management_root_url(C), 3000, 200), + Timeout = genlib_app:env(kds, keyring_initialize_lifetime), + _ = kds_ct_utils:await_initialization_phase(uninitialized, management_root_url(C), Timeout, 200), _ = kds_ct_keyring:init(C), - _ = kds_ct_utils:await_status(unlocked, management_root_url(C), 3000, 200), + UnlockTimeout = genlib_app:env(kds, keyring_unlock_lifetime), + _ = kds_ct_utils:await_status(unlocked, management_root_url(C), UnlockTimeout, 200), _ = ?assertMatch( #{ meta := #{current_key_id := 0, version := 1, keys := #{0 := #{retired := false}}},