From fa6d98c961da267b57f47b1f741026147f627544 Mon Sep 17 00:00:00 2001 From: Kozlov Yakov Date: Tue, 6 Nov 2018 20:45:40 +0300 Subject: [PATCH 001/310] Allow binary payload in lambda invoke request --- src/erlcloud_lambda.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index da450db7c..914a82f3b 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -357,13 +357,13 @@ invoke(FunctionName, Payload) when is_list(Payload)-> invoke(FunctionName, Payload, default_config()). -spec invoke(FunctionName :: binary(), - Payload :: list(), + Payload :: list() | binary(), Config :: aws_config() | binary()) -> return_val(). invoke(FunctionName, Payload, ConfigOrQualifier) when is_list(Payload)-> invoke(FunctionName, Payload, [], ConfigOrQualifier). -spec invoke(FunctionName :: binary(), - Payload :: list(), + Payload :: list() | binary(), Options :: list(), Config :: aws_config() | binary()) -> return_val(). invoke(FunctionName, Payload, Options, Config = #aws_config{}) -> @@ -372,7 +372,7 @@ invoke(FunctionName, Payload, Options, Qualifier) when is_binary(Qualifier) -> invoke(FunctionName, Payload, Options, Qualifier, default_config()). -spec invoke(FunctionName :: binary(), - Payload :: list(), + Payload :: list() | binary(), Options :: list(), Qualifier :: binary()| undefined, Config :: aws_config()) -> return_val(). @@ -761,6 +761,8 @@ decode_body(<<>>) -> decode_body(BinData) -> jsx:decode(BinData). +encode_body(Bin) when is_binary(Bin) -> + Bin; encode_body(undefined) -> <<>>; encode_body([]) -> From 6cec7e4442c524ce65f0d03c61e6df883cc1e92b Mon Sep 17 00:00:00 2001 From: Yakov Kozlov Date: Wed, 7 Nov 2018 10:03:24 +0300 Subject: [PATCH 002/310] Add erlcloud_lambda raw_response_body option --- src/erlcloud_lambda.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index 914a82f3b..56b926f4b 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -743,22 +743,25 @@ lambda_request_no_update(Config, Method, Path, Options, Body, QParam) -> Value -> Value end, ShowRespHeaders = proplists:get_value(show_headers, Options, false), + RawBody = proplists:get_value(raw_response_body, Options, false), Hdrs = proplists:delete(show_headers, Options), Headers = headers(Method, Path, Hdrs, Config, encode_body(Body), QParam), case erlcloud_aws:do_aws_request_form_raw( Method, Config#aws_config.lambda_scheme, Config#aws_config.lambda_host, Config#aws_config.lambda_port, Path, Form, Headers, Config, ShowRespHeaders) of {ok, RespHeaders, RespBody} -> - {ok, RespHeaders, decode_body(RespBody)}; + {ok, RespHeaders, decode_body(RespBody, RawBody)}; {ok, RespBody} -> - {ok, decode_body(RespBody)}; + {ok, decode_body(RespBody, RawBody)}; E -> E end. -decode_body(<<>>) -> +decode_body(Body, true) -> + Body; +decode_body(<<>>, _RawBody) -> []; -decode_body(BinData) -> +decode_body(BinData, _RawBody) -> jsx:decode(BinData). encode_body(Bin) when is_binary(Bin) -> From 2737176c4697e45f84a23877064cea2033ed1975 Mon Sep 17 00:00:00 2001 From: Kozlov Yakov Date: Wed, 7 Nov 2018 17:43:35 +0300 Subject: [PATCH 003/310] Fix raw_response_body option --- src/erlcloud_lambda.erl | 3 ++- test/erlcloud_lambda_tests.erl | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index 56b926f4b..0618e1e3f 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -744,7 +744,8 @@ lambda_request_no_update(Config, Method, Path, Options, Body, QParam) -> end, ShowRespHeaders = proplists:get_value(show_headers, Options, false), RawBody = proplists:get_value(raw_response_body, Options, false), - Hdrs = proplists:delete(show_headers, Options), + Hdrs0 = proplists:delete(show_headers, Options), + Hdrs = proplists:delete(raw_response_body, Hdrs0), Headers = headers(Method, Path, Hdrs, Config, encode_body(Body), QParam), case erlcloud_aws:do_aws_request_form_raw( Method, Config#aws_config.lambda_scheme, Config#aws_config.lambda_host, diff --git a/test/erlcloud_lambda_tests.erl b/test/erlcloud_lambda_tests.erl index 7e8a78835..e8a9e7244 100644 --- a/test/erlcloud_lambda_tests.erl +++ b/test/erlcloud_lambda_tests.erl @@ -637,6 +637,11 @@ api_tests(_) -> Expected = {ok, [], [{<<"message">>, <<"Hello World!">>}]}, ?assertEqual(Expected, Result) end, + fun() -> + Result = erlcloud_lambda:invoke(<<"name">>, [], [raw_response_body], #aws_config{}), + Expected = {ok, <<"{\"message\":\"Hello World!\"}">>}, + ?assertEqual(Expected, Result) + end, fun() -> Result = erlcloud_lambda:list_aliases(<<"name">>), Expected = {ok, [{<<"Aliases">>, From 0fb0847eeb6a8da44adaa12a77858a5f976150d7 Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Thu, 8 Nov 2018 19:08:21 -0600 Subject: [PATCH 004/310] Upgrade lhttpc to 1.6.2 Incorporate erlcloud/lhttpc#22 and erlcloud/lhttpc#23 --- rebar.config | 2 +- rebar.config.script | 2 +- rebar.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index 5837d3a0c..2eb1ac9e7 100644 --- a/rebar.config +++ b/rebar.config @@ -18,7 +18,7 @@ {deps, [ {jsx, "2.8.0"}, - {lhttpc, "1.6.1"}, + {lhttpc, "1.6.2"}, {eini, "1.2.5"}, {base16, "1.0.0"} ]}. diff --git a/rebar.config.script b/rebar.config.script index 3be6f90f8..b686876dd 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -9,7 +9,7 @@ case erlang:function_exported(rebar3, main, 1) of {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "2.8.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.5"}}}, - {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.1"}}}, + {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, {base16, ".*", {git, "https://github.com/goj/base16.git", {tag, "1.0.0"}}}]} | lists:keydelete(deps, 1, CONFIG)] end. diff --git a/rebar.lock b/rebar.lock index 15d035382..fd8b1b919 100644 --- a/rebar.lock +++ b/rebar.lock @@ -2,11 +2,11 @@ [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.5">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.8.0">>},0}, - {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.6.1">>},0}]}. + {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.6.2">>},0}]}. [ {pkg_hash,[ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, {<<"eini">>, <<"98E4988474FAAF821E3511579090AF989096ADBDB5F7BFAA03AA0A9AC80296B2">>}, {<<"jsx">>, <<"749BEC6D205C694AE1786D62CEA6CC45A390437E24835FD16D12D74F07097727">>}, - {<<"lhttpc">>, <<"8592681D9E86E36E33E8B0383709F973FB7737487E013EE2624A665C41A6B8EF">>}]} + {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} ]. From 7f6e5ca71e4688f07a156927328cd1cd5c0a99f4 Mon Sep 17 00:00:00 2001 From: Grigory Starinkin Date: Thu, 15 Nov 2018 08:41:10 +0000 Subject: [PATCH 005/310] update meck dependency to be able compile using erlang 21 --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 2eb1ac9e7..c0db6801d 100644 --- a/rebar.config +++ b/rebar.config @@ -32,6 +32,6 @@ {profiles, [ - {test, [{deps, [{meck, "0.8.4"}]}]} + {test, [{deps, [{meck, "0.8.12"}]}]} ,{warnings, [{erl_opts, [warnings_as_errors]}]} ]}. From 5d8085b4c175021bbf7a6d1bb4d1f2bff0dbae50 Mon Sep 17 00:00:00 2001 From: Grigory Starinkin Date: Thu, 15 Nov 2018 18:05:07 +0000 Subject: [PATCH 006/310] bump meck to 0.8.12 for rebar2 or older --- rebar.config.script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.script b/rebar.config.script index b686876dd..10e1730c5 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -5,7 +5,7 @@ case erlang:function_exported(rebar3, main, 1) of false -> % rebar 2.x or older %% Use git-based deps %% profiles - [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.4"}}}, + [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.12"}}}, {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "2.8.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.5"}}}, From 7b6d16f3a1bece865de5661f667f16d2b143dec9 Mon Sep 17 00:00:00 2001 From: Vyacheslav Vorobyov Date: Wed, 21 Nov 2018 20:03:17 +0300 Subject: [PATCH 007/310] Add ELB describe_tags. (#555) * Add ELB describe_tags. * Follow reviewers comments. * Add 'describe_tags_all'. * Purge paged feature for 'describe tags' due to they not even present at AWS API. * Purge more. * Revert rebar3 seek. But add fallback to rebar3 which is in a repo. * Remove unused util function and 'define' directive. * Remove exports. --- Makefile | 2 +- src/erlcloud_elb.erl | 57 +++++++++++++++++++++++++---- src/erlcloud_util.erl | 9 +++-- src/erlcloud_xml.erl | 2 +- test/erlcloud_elb_tests.erl | 73 +++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 14 deletions(-) create mode 100644 test/erlcloud_elb_tests.erl diff --git a/Makefile b/Makefile index b6c556855..9399206a4 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ # determine which Rebar we want to be running REBAR2=$(shell which rebar || echo ./rebar) -REBAR3=$(shell which rebar3) +REBAR3=$(shell which rebar3 || echo ./rebar3) ifeq ($(FORCE_REBAR2),true) REBAR=$(REBAR2) REBAR_VSN=2 diff --git a/src/erlcloud_elb.erl b/src/erlcloud_elb.erl index 9cf93025d..5c8f6ebc8 100644 --- a/src/erlcloud_elb.erl +++ b/src/erlcloud_elb.erl @@ -12,22 +12,25 @@ describe_load_balancer/1, describe_load_balancer/2, - describe_load_balancers/0, describe_load_balancers/1, + describe_load_balancers/0, describe_load_balancers/1, describe_load_balancers/2, describe_load_balancers/3, describe_load_balancers/4, describe_load_balancers_all/0, describe_load_balancers_all/1, describe_load_balancers_all/2, configure_health_check/2, configure_health_check/3, - + create_load_balancer_policy/3, create_load_balancer_policy/4, create_load_balancer_policy/5, delete_load_balancer_policy/2, delete_load_balancer_policy/3, - - describe_load_balancer_policies/0, describe_load_balancer_policies/1, + + describe_load_balancer_policies/0, describe_load_balancer_policies/1, describe_load_balancer_policies/2, describe_load_balancer_policies/3, - - describe_load_balancer_policy_types/0, describe_load_balancer_policy_types/1, + + describe_load_balancer_policy_types/0, describe_load_balancer_policy_types/1, describe_load_balancer_policy_types/2, - describe_load_balancer_attributes/1, describe_load_balancer_attributes/2]). + describe_load_balancer_attributes/1, describe_load_balancer_attributes/2, + + describe_tags/2, describe_tags/1 + ]). -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). @@ -40,11 +43,16 @@ -define(DESCRIBE_ELBS_PATH, "/DescribeLoadBalancersResponse/DescribeLoadBalancersResult/LoadBalancerDescriptions/member"). -define(DESCRIBE_ELBS_NEXT_TOKEN, - "/DescribeLoadBalancersResponse/DescribeLoadBalancersResult/NextMarker"). + "/DescribeLoadBalancersResponse/NextMarker"). -define(DESCRIBE_ELB_POLICIES_PATH, "/DescribeLoadBalancerPoliciesResponse/DescribeLoadBalancerPoliciesResult/PolicyDescriptions/member"). -define(DESCRIBE_ELB_POLICY_TYPE_PATH, "/DescribeLoadBalancerPolicyTypesResponse/DescribeLoadBalancerPolicyTypesResult/PolicyTypeDescriptions/member"). +-define(DESCRIBE_ELBS_TAGS_PATH, + "/DescribeTagsResponse/DescribeTagsResult/TagDescriptions/member"). + +-type result() :: {ok, term()} | {error, metadata_not_available | container_credentials_unavailable | erlcloud_aws:httpc_result_error()}. + -import(erlcloud_xml, [get_text/2, get_integer/2, get_list/2]). @@ -211,6 +219,24 @@ describe_load_balancers(Names, Params, Config) -> {error, Reason} end. +-spec describe_tags(ElbNames) -> result() when ElbNames :: list(string()). +describe_tags(Names) -> + describe_tags(Names, default_config()). + +-spec describe_tags(ElbNames, AwsConfig) -> result() + when + ElbNames :: list(string()), + AwsConfig :: aws_config(). +describe_tags(Names, Config) -> + P = member_params("LoadBalancerNames.member.", Names), + case elb_query(Config, "DescribeTags", P) of + {ok, Doc} -> + Elbs = xmerl_xpath:string(?DESCRIBE_ELBS_TAGS_PATH, Doc), + {ok, lists:map(fun extract_elb_tags/1, Elbs)}; + {error, Reason} -> + {error, Reason} + end. + -spec describe_load_balancers_all() -> {ok, [term()]} | {error, term()}. describe_load_balancers_all() -> @@ -514,6 +540,8 @@ describe_all(Fun, AwsConfig, Marker, Acc) -> case Fun(Marker, AwsConfig) of {ok, Res} -> {ok, lists:append(Acc, Res)}; + {ok, Res, NewMarker} -> + describe_all(Fun, AwsConfig, NewMarker, lists:append(Acc, Res)); {{paged, NewMarker}, Res} -> describe_all(Fun, AwsConfig, NewMarker, lists:append(Acc, Res)); {error, Reason} -> @@ -545,3 +573,16 @@ elb_request(Config, Action, Params) -> {error, Reason} -> erlang:error({aws_error, Reason}) end. + + +extract_elb_tags(Item) -> + [ + {load_balancer_name, get_text("LoadBalancerName", Item)}, + {tags, [extract_tag(L) || L <- xmerl_xpath:string("Tags/member", Item)]} + ]. + +extract_tag(Item) -> + [ + {value, get_text("Value", Item)}, + {key, get_text("Key", Item)} + ]. diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index 09626d039..c2d3eae7a 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -1,8 +1,8 @@ -module(erlcloud_util). --export([sha_mac/2, sha256_mac/2, md5/1, sha256/1,rand_uniform/1, - is_dns_compliant_name/1, - query_all/4, query_all/5, query_all_token/4, make_response/2, - get_items/2, to_string/1, encode_list/2, next_token/2]). +-export([sha_mac/2, sha256_mac/2, md5/1, sha256/1, rand_uniform/1, + is_dns_compliant_name/1, + query_all/4, query_all/5, query_all_token/4, make_response/2, + get_items/2, to_string/1, encode_list/2, next_token/2]). -define(MAX_ITEMS, 1000). @@ -128,3 +128,4 @@ next_token(Path, XML) -> ok end. + diff --git a/src/erlcloud_xml.erl b/src/erlcloud_xml.erl index 49379ad74..631dc4493 100644 --- a/src/erlcloud_xml.erl +++ b/src/erlcloud_xml.erl @@ -79,7 +79,7 @@ get_float(XPath, Node) -> get_text(#xmlText{value=Value}) -> Value; get_text(#xmlElement{content=Content}) -> - lists:flatten([get_text(Node) || Node <- Content]). + lists:flatmap(fun get_text/1, Content). get_text(XPath, Doc) -> get_text(XPath, Doc, ""). get_text({XPath, AttrName}, Doc, Default) -> diff --git a/test/erlcloud_elb_tests.erl b/test/erlcloud_elb_tests.erl new file mode 100644 index 000000000..335dfbb9b --- /dev/null +++ b/test/erlcloud_elb_tests.erl @@ -0,0 +1,73 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +-module(erlcloud_elb_tests). +-include_lib("eunit/include/eunit.hrl"). + + +%%%=================================================================== +%%% Test entry points +%%%=================================================================== + +start() -> + meck:new(erlcloud_aws, [passthrough]), + ok. + + +stop(_) -> + meck:unload(erlcloud_aws). + + +elb_tags_test_() -> + {foreach, + fun start/0, + fun stop/1, + [ + {"Request describe_tags.", + fun() -> + meck:expect(erlcloud_aws, aws_request_xml4, fun(_, _, _, _, _, _) -> + {ok, describe_tags_response_xmerl()} end), + LoadBalancerName = "vvorobyov-classic", + Resp = erlcloud_elb:describe_tags( + [LoadBalancerName], + erlcloud_aws:default_config()), + ?assertMatch( + {ok, [ + [ + {load_balancer_name, LoadBalancerName}, + {tags, [[{value, _}, {key, _}], [{value, _}, {key, _}]]} + ] + ]}, + Resp + ) + end} + ] + }. + + +%%%=================================================================== +%%% Helpers +%%%=================================================================== + +describe_tags_response_xmerl() -> + XML = " + + + + vvorobyov-classic + + + tag-value-2 + tag-key-2 + + + tag-value-2 + tag-key-1 + + + + + + + 75bbeca1-e357-11e8-b2f4-735be4940a86 + +", + element(1, xmerl_scan:string(XML)). From b5e74484557b8fc9048409221d6a820f4ad89c2b Mon Sep 17 00:00:00 2001 From: getong <3949379+getong@users.noreply.github.com> Date: Sat, 1 Dec 2018 01:00:04 +0800 Subject: [PATCH 008/310] update jsx to 2.9.0 --- rebar.config | 2 +- rebar.config.script | 2 +- rebar.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index c0db6801d..ee3970f31 100644 --- a/rebar.config +++ b/rebar.config @@ -17,7 +17,7 @@ warn_unused_vars]}. {deps, [ - {jsx, "2.8.0"}, + {jsx, "2.9.0"}, {lhttpc, "1.6.2"}, {eini, "1.2.5"}, {base16, "1.0.0"} diff --git a/rebar.config.script b/rebar.config.script index 10e1730c5..27c7683d6 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -6,7 +6,7 @@ case erlang:function_exported(rebar3, main, 1) of %% Use git-based deps %% profiles [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.12"}}}, - {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "2.8.0"}}}, + {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "2.9.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.5"}}}, {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, diff --git a/rebar.lock b/rebar.lock index fd8b1b919..fb4f83c7b 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,12 +1,12 @@ {"1.1.0", [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.5">>},0}, - {<<"jsx">>,{pkg,<<"jsx">>,<<"2.8.0">>},0}, + {<<"jsx">>,{pkg,<<"jsx">>,<<"2.9.0">>},0}, {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.6.2">>},0}]}. [ {pkg_hash,[ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, {<<"eini">>, <<"98E4988474FAAF821E3511579090AF989096ADBDB5F7BFAA03AA0A9AC80296B2">>}, - {<<"jsx">>, <<"749BEC6D205C694AE1786D62CEA6CC45A390437E24835FD16D12D74F07097727">>}, + {<<"jsx">>, <<"D2F6E5F069C00266CAD52FB15D87C428579EA4D7D73A33669E12679E203329DD">>}, {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} ]. From 4cf06a0947129f906dbb21569a44f8889e7bd71e Mon Sep 17 00:00:00 2001 From: getong <3949379+getong@users.noreply.github.com> Date: Sat, 1 Dec 2018 01:40:38 +0800 Subject: [PATCH 009/310] test erlang/otp 21 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4bf7ed6b6..a51f67989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ otp_release: - 18.3 - 19.3 - 20.3 + - 21.1.3 env: global: - MAIN_OTP=19.3 From 42e6f306b2bfe155be4a99951f81894a01941330 Mon Sep 17 00:00:00 2001 From: Evgeny Bob Date: Fri, 30 Nov 2018 23:13:30 -0600 Subject: [PATCH 010/310] Update .travis.yml --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index a51f67989..c8bca2286 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,12 @@ sudo: false language: erlang otp_release: - - 17.5 - - 18.3 - 19.3 - 20.3 - 21.1.3 env: global: - - MAIN_OTP=19.3 + - MAIN_OTP=20.3 matrix: - FORCE_REBAR2=true - FORCE_REBAR2=false PATH=$PATH:$PWD From c04c8b3316f6ad84dac2ae993258adb39b3c36a2 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Mon, 3 Dec 2018 15:38:52 +0800 Subject: [PATCH 011/310] add list_all_tables and wait_for_table --- src/erlcloud_ddb2.erl | 57 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index c8f6d75a4..cc9ecf731 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -105,6 +105,7 @@ list_backups/0, list_backups/1, list_backups/2, list_global_tables/0, list_global_tables/1, list_global_tables/2, list_tables/0, list_tables/1, list_tables/2, + list_all_tables/0, list_all_tables/1, list_tags_of_resource/1, list_tags_of_resource/2, list_tags_of_resource/3, put_item/2, put_item/3, put_item/4, %% Note that query is a Erlang reserved word, so we use q instead @@ -118,7 +119,8 @@ update_item/3, update_item/4, update_item/5, update_global_table/2, update_global_table/3, update_global_table/4, update_table/2, update_table/3, update_table/4, update_table/5, - update_time_to_live/2, update_time_to_live/3, update_time_to_live/4 + update_time_to_live/2, update_time_to_live/3, update_time_to_live/4, + wait_for_table/1, wait_for_table/2, wait_for_table/3 ]). -export_type( @@ -2577,6 +2579,27 @@ list_tables(Opts, Config) -> out(Return, fun(Json, UOpts) -> undynamize_record(list_tables_record(), Json, UOpts) end, DdbOpts, #ddb2_list_tables.table_names, {ok, []}). + +list_all_tables() -> + list_all_tables(default_config()). + +list_all_tables(AWSCfg) -> + do_list_all_tables(undefined, AWSCfg, []). + +do_list_all_tables(LastTable, AWSCfg, Result) -> + Options = [{exclusive_start_table_name, LastTable}, {out, record}], + case erlcloud_ddb2:list_tables(Options, AWSCfg) of + {ok, #ddb2_list_tables{table_names = TableNames, last_evaluated_table_name = LastTableName}} -> + NewResult = TableNames ++ Result, + case LastTableName of + undefined -> {ok, NewResult}; + _ -> do_list_all_tables(LastTableName, AWSCfg, NewResult) + end; + {error, _} = Error -> + Error + end. + + %%%------------------------------------------------------------------------------ %%% ListTagsOfResource %%%------------------------------------------------------------------------------ @@ -3540,6 +3563,38 @@ update_time_to_live(Table, Opts, Config) when is_list(Opts) -> update_time_to_live(Table, AttributeName, Enabled) -> update_time_to_live(Table, [{attribute_name, AttributeName}, {enabled, Enabled}]). + +%%------------------------------------------------------------------------------ +%% @doc +%% wait until table_status==active +%% +%% ===Example=== +%% +%% ``` +%% erlcloud_ddb2:wait_for_table(<<"TableName">>, 3000, Config) +%% ''' +%% @end +%%------------------------------------------------------------------------------ +wait_for_table(Table, Interval, AWSCfg) when is_binary(Table) -> + case erlcloud_ddb2:describe_table(Table, [{out, record}], AWSCfg) of + {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = active}}} -> + ok; + {ok, _} -> + timer:sleep(Interval), + wait_for_table(Table, Interval, AWSCfg); + {error, Reason} -> + error(Reason) + end; +wait_for_table(Table, Interval, AWSCfg) when is_list(Table) -> + wait_for_table(list_to_binary(Table), Interval, AWSCfg). + +wait_for_table(Table, AWSCfg) -> + wait_for_table(list_to_binary(Table), 3000, AWSCfg). + +wait_for_table(Table) -> + wait_for_table(list_to_binary(Table), default_config()). + + to_binary(X) when is_binary(X) -> X; to_binary(X) when is_list(X) -> From 907cf3c89a5a5eb9275e0d80844dba8c076989f4 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Mon, 3 Dec 2018 17:29:03 +0800 Subject: [PATCH 012/310] add list_all_tables and wait_for_table --- src/erlcloud_ddb2.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index cc9ecf731..07ac31a00 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -2583,6 +2583,7 @@ list_tables(Opts, Config) -> list_all_tables() -> list_all_tables(default_config()). +-spec list_all_tables(aws_config()) -> {ok, [table_name()]} | {error, any()}. list_all_tables(AWSCfg) -> do_list_all_tables(undefined, AWSCfg, []). @@ -3575,6 +3576,7 @@ update_time_to_live(Table, AttributeName, Enabled) -> %% ''' %% @end %%------------------------------------------------------------------------------ +-spec wait_for_table(table_name(), non_neg_integer(), aws_config()) -> ok | {error, any()}. wait_for_table(Table, Interval, AWSCfg) when is_binary(Table) -> case erlcloud_ddb2:describe_table(Table, [{out, record}], AWSCfg) of {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = active}}} -> @@ -3583,16 +3585,16 @@ wait_for_table(Table, Interval, AWSCfg) when is_binary(Table) -> timer:sleep(Interval), wait_for_table(Table, Interval, AWSCfg); {error, Reason} -> - error(Reason) + {error, Reason} end; wait_for_table(Table, Interval, AWSCfg) when is_list(Table) -> wait_for_table(list_to_binary(Table), Interval, AWSCfg). wait_for_table(Table, AWSCfg) -> - wait_for_table(list_to_binary(Table), 3000, AWSCfg). + wait_for_table(Table, 3000, AWSCfg). wait_for_table(Table) -> - wait_for_table(list_to_binary(Table), default_config()). + wait_for_table(Table, default_config()). to_binary(X) when is_binary(X) -> From 52675a828b326011951b733e9a996ef63948ae0e Mon Sep 17 00:00:00 2001 From: pengzheng Date: Tue, 4 Dec 2018 02:31:48 +0800 Subject: [PATCH 013/310] add list_all_tables and wait_for_table --- src/erlcloud_ddb_util.erl | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index 6759b1ebf..5d6f22012 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -23,8 +23,10 @@ delete_hash_key/3, delete_hash_key/4, delete_hash_key/5, get_all/2, get_all/3, get_all/4, put_all/2, put_all/3, put_all/4, + list_all_tables/0, list_all_tables/1, q_all/2, q_all/3, q_all/4, scan_all/1, scan_all/2, scan_all/3, + wait_for_table/1, wait_for_table/2, wait_for_table/3, write_all/2, write_all/3, write_all/4 ]). @@ -230,6 +232,28 @@ batch_get_retry(RequestItems, DdbOpts, Config, Acc) -> batch_get_retry(Unprocessed, DdbOpts, Config, Items ++ Acc) end. +%%%------------------------------------------------------------------------------ +%%% list_all_tables +%%%------------------------------------------------------------------------------ + +list_all_tables() -> + list_all_tables(default_config()). + +-spec list_all_tables(aws_config()) -> {ok, [table_name()]} | {error, any()}. +list_all_tables(Config) -> + do_list_all_tables(undefined, Config, []). + +do_list_all_tables(LastTable, Config, Result) -> + Options = [{exclusive_start_table_name, LastTable}, {out, record}], + case erlcloud_ddb2:list_tables(Options, Config) of + {ok, #ddb2_list_tables{table_names = TableNames, last_evaluated_table_name = undefined}} -> + {ok, Result ++ TableNames}; + {ok, #ddb2_list_tables{table_names = TableNames, last_evaluated_table_name = LastTableName}} -> + do_list_all_tables(LastTableName, Config, Result ++ TableNames); + {error, _} = Error -> + Error + end. + %%%------------------------------------------------------------------------------ %%% put_all %%%------------------------------------------------------------------------------ @@ -477,6 +501,48 @@ batch_write_retry(RequestItems, Config) -> batch_write_retry(Unprocessed, Config) end. +%%------------------------------------------------------------------------------ +%% @doc +%% wait until table_status==active. +%% RetryTimes = 0 means infinity. +%% +%% ===Example=== +%% +%% ` +%% erlcloud_ddb2:wait_for_table(<<"TableName">>, 3000, 40, Config) +%% ' +%% @end +%%------------------------------------------------------------------------------ + +-spec wait_for_table(table_name(), pos_integer(), non_neg_integer(), aws_config()) -> ok | {error, timeout | any()}. +wait_for_table(Table, Interval, RetryTimes, AWSCfg) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> + case erlcloud_ddb2:describe_table(Table, [{out, record}], AWSCfg) of + {ok, #ddb2_describe_table{table = _#ddb2_table_description{table_status = active}}} -> + ok; + {ok, _} -> + case RetryTimes of + 0 -> + timer:sleep(Interval), + wait_for_table(Table, Interval, RetryTimes, AWSCfg); + 1 -> + {error, timeout}; + _ -> + timer:sleep(Interval), + wait_for_table(Table, Interval, RetryTimes - 1, AWSCfg) + end; + {error, Reason} -> + {error, Reason} + end; +wait_for_table(Table, Interval, RetryTimes, AWSCfg) when is_list(Table) -> + wait_for_table(list_to_binary(Table), Interval, RetryTimes, AWSCfg). + +wait_for_table(Table, AWSCfg) -> + wait_for_table(Table, 3000, 0, AWSCfg). + +wait_for_table(Table) -> + wait_for_table(Table, default_config()). + + write_all_result([ok | T]) -> write_all_result(T); write_all_result([{error, Reason} | _]) -> From 462f81515312cce0bab2988c727eec67b7bb6f27 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Tue, 4 Dec 2018 02:32:23 +0800 Subject: [PATCH 014/310] backup --- include/erlcloud_ddb2.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index f55e5f52f..04239884c 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -226,7 +226,7 @@ }). -record(ddb2_time_to_live_specification, - {attribute_name :: undefined | erlcloud_ddb2:attr_name(), + {attribute_name :: undefined | erlcloud_ddb2:attr_name(), enabled :: undefined | boolean() }). From 040134520fa6850c3bf5ab289d08947a5ca863b8 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Tue, 4 Dec 2018 02:32:54 +0800 Subject: [PATCH 015/310] add list_all_tables and wait_for_table --- src/erlcloud_ddb2.erl | 59 +------------------------------------------ 1 file changed, 1 insertion(+), 58 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 07ac31a00..c8f6d75a4 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -105,7 +105,6 @@ list_backups/0, list_backups/1, list_backups/2, list_global_tables/0, list_global_tables/1, list_global_tables/2, list_tables/0, list_tables/1, list_tables/2, - list_all_tables/0, list_all_tables/1, list_tags_of_resource/1, list_tags_of_resource/2, list_tags_of_resource/3, put_item/2, put_item/3, put_item/4, %% Note that query is a Erlang reserved word, so we use q instead @@ -119,8 +118,7 @@ update_item/3, update_item/4, update_item/5, update_global_table/2, update_global_table/3, update_global_table/4, update_table/2, update_table/3, update_table/4, update_table/5, - update_time_to_live/2, update_time_to_live/3, update_time_to_live/4, - wait_for_table/1, wait_for_table/2, wait_for_table/3 + update_time_to_live/2, update_time_to_live/3, update_time_to_live/4 ]). -export_type( @@ -2579,28 +2577,6 @@ list_tables(Opts, Config) -> out(Return, fun(Json, UOpts) -> undynamize_record(list_tables_record(), Json, UOpts) end, DdbOpts, #ddb2_list_tables.table_names, {ok, []}). - -list_all_tables() -> - list_all_tables(default_config()). - --spec list_all_tables(aws_config()) -> {ok, [table_name()]} | {error, any()}. -list_all_tables(AWSCfg) -> - do_list_all_tables(undefined, AWSCfg, []). - -do_list_all_tables(LastTable, AWSCfg, Result) -> - Options = [{exclusive_start_table_name, LastTable}, {out, record}], - case erlcloud_ddb2:list_tables(Options, AWSCfg) of - {ok, #ddb2_list_tables{table_names = TableNames, last_evaluated_table_name = LastTableName}} -> - NewResult = TableNames ++ Result, - case LastTableName of - undefined -> {ok, NewResult}; - _ -> do_list_all_tables(LastTableName, AWSCfg, NewResult) - end; - {error, _} = Error -> - Error - end. - - %%%------------------------------------------------------------------------------ %%% ListTagsOfResource %%%------------------------------------------------------------------------------ @@ -3564,39 +3540,6 @@ update_time_to_live(Table, Opts, Config) when is_list(Opts) -> update_time_to_live(Table, AttributeName, Enabled) -> update_time_to_live(Table, [{attribute_name, AttributeName}, {enabled, Enabled}]). - -%%------------------------------------------------------------------------------ -%% @doc -%% wait until table_status==active -%% -%% ===Example=== -%% -%% ``` -%% erlcloud_ddb2:wait_for_table(<<"TableName">>, 3000, Config) -%% ''' -%% @end -%%------------------------------------------------------------------------------ --spec wait_for_table(table_name(), non_neg_integer(), aws_config()) -> ok | {error, any()}. -wait_for_table(Table, Interval, AWSCfg) when is_binary(Table) -> - case erlcloud_ddb2:describe_table(Table, [{out, record}], AWSCfg) of - {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = active}}} -> - ok; - {ok, _} -> - timer:sleep(Interval), - wait_for_table(Table, Interval, AWSCfg); - {error, Reason} -> - {error, Reason} - end; -wait_for_table(Table, Interval, AWSCfg) when is_list(Table) -> - wait_for_table(list_to_binary(Table), Interval, AWSCfg). - -wait_for_table(Table, AWSCfg) -> - wait_for_table(Table, 3000, AWSCfg). - -wait_for_table(Table) -> - wait_for_table(Table, default_config()). - - to_binary(X) when is_binary(X) -> X; to_binary(X) when is_list(X) -> From 83af68fac4fe626a76cd9c5b15a402f4be69af5e Mon Sep 17 00:00:00 2001 From: pengzheng Date: Tue, 4 Dec 2018 02:37:33 +0800 Subject: [PATCH 016/310] backup --- test/erlcloud_ddb2_tests.erl | 122 +++++++++++++++++------------------ 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index 9195d3e3c..cddac432d 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -874,7 +874,7 @@ create_backup_input_tests(_) -> Tests = [?_ddb_test( {"CreateBackup example request", - ?_f(erlcloud_ddb2:create_backup(<<"Forum_Backup">>,<<"Forum">>)), " + ?_f(erlcloud_ddb2:create_backup(<<"Forum_Backup">>, <<"Forum">>)), " { \"BackupName\": \"Forum_Backup\", \"TableName\": \"Forum\" @@ -915,7 +915,7 @@ create_backup_output_tests(_) -> }) ], - output_tests(?_f(erlcloud_ddb2:create_backup(<<"Forum_Backup">>,<<"Forum">>)), Tests). + output_tests(?_f(erlcloud_ddb2:create_backup(<<"Forum_Backup">>, <<"Forum">>)), Tests). %% CreateGlobalTable input test: create_global_table_input_tests(_) -> @@ -1826,10 +1826,10 @@ delete_item_input_tests(_) -> Tests = [?_ddb_test( {"DeleteItem example request", - ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{return_values, all_old}, + [{return_values, all_old}, {expected, {<<"Replies">>, null}}])), " { \"TableName\": \"Thread\", @@ -1852,9 +1852,9 @@ delete_item_input_tests(_) -> ?_ddb_test( {"DeleteItem example request with ConditionExpression", ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{return_values, all_old}, + [{return_values, all_old}, {condition_expression, <<"attribute_not_exists(Replies)">>}])), " { \"TableName\": \"Thread\", @@ -1872,9 +1872,9 @@ delete_item_input_tests(_) -> }), ?_ddb_test( {"DeleteItem return metrics", - ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, - {<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, - [{return_consumed_capacity, total}, + ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, + {<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{return_consumed_capacity, total}, {return_item_collection_metrics, size}])), " { \"TableName\": \"Thread\", @@ -2737,9 +2737,9 @@ get_item_input_tests(_) -> [?_ddb_test( {"GetItem example request, with fully specified keys", ?_f(erlcloud_ddb2:get_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, + [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, consistent_read, {return_consumed_capacity, total}, %% Make sure options at beginning of list override later options @@ -2748,10 +2748,10 @@ get_item_input_tests(_) -> Example1Response}), ?_ddb_test( {"GetItem example request, with inferred key types", - ?_f(erlcloud_ddb2:get_item(<<"Thread">>, - [{<<"ForumName">>, "Amazon DynamoDB"}, + ?_f(erlcloud_ddb2:get_item(<<"Thread">>, + [{<<"ForumName">>, "Amazon DynamoDB"}, {<<"Subject">>, <<"How do I update multiple items?">>}], - [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, + [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, {consistent_read, true}, {return_consumed_capacity, total}] )), @@ -2759,9 +2759,9 @@ get_item_input_tests(_) -> ?_ddb_test( {"GetItem example request, with ProjectionExpression", ?_f(erlcloud_ddb2:get_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{projection_expression, <<"LastPostDateTime, Message, Tags">>}, + [{projection_expression, <<"LastPostDateTime, Message, Tags">>}, consistent_read, {return_consumed_capacity, total}] )), " @@ -2924,7 +2924,7 @@ list_backups_input_tests(_) -> Tests = [?_ddb_test( {"ListBackups example request", - ?_f(erlcloud_ddb2:list_backups([{limit, 4},{exclusive_start_backup_arn, <<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523516528459-dfd5667f">>},{table_name, <<"Forum">>},{time_range_lower_bound,1522926603.688},{time_range_upper_bound, 1523022454.098}])), " + ?_f(erlcloud_ddb2:list_backups([{limit, 4}, {exclusive_start_backup_arn, <<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523516528459-dfd5667f">>}, {table_name, <<"Forum">>}, {time_range_lower_bound, 1522926603.688}, {time_range_upper_bound, 1523022454.098}])), " { \"Limit\": 4, \"TableName\": \"Forum\", @@ -3001,7 +3001,7 @@ list_global_tables_input_tests(_) -> }), ?_ddb_test( {"ListGlobalTables empty request", - ?_f(erlcloud_ddb2:list_global_tables()), + ?_f(erlcloud_ddb2:list_global_tables()), "{}" }) @@ -3091,7 +3091,7 @@ list_tables_input_tests(_) -> }), ?_ddb_test( {"ListTables empty request", - ?_f(erlcloud_ddb2:list_tables()), + ?_f(erlcloud_ddb2:list_tables()), "{}" }) @@ -3181,14 +3181,14 @@ put_item_input_tests(_) -> Tests = [?_ddb_test( {"PutItem example request", - ?_f(erlcloud_ddb2:put_item(<<"Thread">>, - [{<<"LastPostedBy">>, <<"fred@example.com">>}, + ?_f(erlcloud_ddb2:put_item(<<"Thread">>, + [{<<"LastPostedBy">>, <<"fred@example.com">>}, {<<"ForumName">>, <<"Amazon DynamoDB">>}, {<<"LastPostDateTime">>, <<"201303190422">>}, {<<"Tags">>, {ss, [<<"Update">>, <<"Multiple Items">>, <<"HelpMe">>]}}, {<<"Subject">>, <<"How do I update multiple items?">>}, {<<"Message">>, <<"I want to update multiple items in a single API call. What's the best way to do that?">>}], - [{expected, [{<<"ForumName">>, null}, {<<"Subject">>, null}]}])), " + [{expected, [{<<"ForumName">>, null}, {<<"Subject">>, null}]}])), " { \"TableName\": \"Thread\", \"Item\": { @@ -3223,11 +3223,11 @@ put_item_input_tests(_) -> }), ?_ddb_test( {"PutItem float inputs", - ?_f(erlcloud_ddb2:put_item(<<"Thread">>, - [{<<"typed float">>, {n, 1.2}}, + ?_f(erlcloud_ddb2:put_item(<<"Thread">>, + [{<<"typed float">>, {n, 1.2}}, {<<"untyped float">>, 3.456}, {<<"mixed set">>, {ns, [7.8, 9.0, 10]}}], - [])), " + [])), " { \"TableName\": \"Thread\", \"Item\": { @@ -3246,13 +3246,13 @@ put_item_input_tests(_) -> ?_ddb_test( {"PutItem example request with ConditionExpression and ExpressionAttributeValues", ?_f(erlcloud_ddb2:put_item(<<"Thread">>, - [{<<"LastPostedBy">>, <<"fred@example.com">>}, + [{<<"LastPostedBy">>, <<"fred@example.com">>}, {<<"ForumName">>, <<"Amazon DynamoDB">>}, {<<"LastPostDateTime">>, <<"201303190422">>}, {<<"Tags">>, {ss, [<<"Update">>, <<"Multiple Items">>, <<"HelpMe">>]}}, {<<"Subject">>, <<"How do I update multiple items?">>}, {<<"Message">>, <<"I want to update multiple items in a single API call. What's the best way to do that?">>}], - [{condition_expression, <<"ForumName <> :f and Subject <> :s">>}, + [{condition_expression, <<"ForumName <> :f and Subject <> :s">>}, {expression_attribute_values, [ {<<":f">>, <<"Amazon DynamoDB">>}, {<<":s">>, <<"How do I update multiple items?">>}]}])), " @@ -3292,7 +3292,7 @@ put_item_input_tests(_) -> ?_ddb_test( {"PutItem request with complex item", ?_f(erlcloud_ddb2:put_item(<<"Table">>, - [{<<"bool_true">>, {bool, true}}, + [{<<"bool_true">>, {bool, true}}, {<<"bool_false">>, {bool, false}}, {<<"null_value">>, {null, true}}, {<<"list_value">>, {l, ["string", {ss, ["string1", "string2"]}]}}, @@ -3441,9 +3441,9 @@ q_input_tests(_) -> [?_ddb_test( {"Query example 1 request", ?_f(erlcloud_ddb2:q(<<"Thread">>, - [{<<"LastPostDateTime">>, {{s, <<"20130101">>}, {s, <<"20130115">>}}, between}, + [{<<"LastPostDateTime">>, {{s, <<"20130101">>}, {s, <<"20130115">>}}, between}, {<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}], - [{index_name, <<"LastPostIndex">>}, + [{index_name, <<"LastPostIndex">>}, {select, all_attributes}, {limit, 3}, {consistent_read, true}])), " @@ -3479,8 +3479,8 @@ q_input_tests(_) -> ?_ddb_test( {"Query example 2 request", ?_f(erlcloud_ddb2:q(<<"Thread">>, - {<<"ForumName">>, <<"Amazon DynamoDB">>, eq}, - [{select, count}, + {<<"ForumName">>, <<"Amazon DynamoDB">>, eq}, + [{select, count}, {consistent_read, true}])), " { \"TableName\": \"Thread\", @@ -3501,8 +3501,8 @@ q_input_tests(_) -> ?_ddb_test( {"Query exclusive start key", ?_f(erlcloud_ddb2:q(<<"Thread">>, - [{<<"ForumName">>, <<"Amazon DynamoDB">>, eq}], - [{select, count}, + [{<<"ForumName">>, <<"Amazon DynamoDB">>, eq}], + [{select, count}, {index_name, <<"LastPostIndex">>}, {exclusive_start_key, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"Exclusive key can have 3 parts">>}}, @@ -3537,8 +3537,8 @@ q_input_tests(_) -> ?_ddb_test( {"Query example request with KeyConditionExpression, ProjectionExpression and ExpressionAttributeValues", ?_f(erlcloud_ddb2:q(<<"Reply">>, - <<"Id = :v1 AND PostedBy BETWEEN :v2a AND :v2b">>, - [{index_name, <<"PostedBy-Index">>}, + <<"Id = :v1 AND PostedBy BETWEEN :v2a AND :v2b">>, + [{index_name, <<"PostedBy-Index">>}, {limit, 3}, {consistent_read, true}, {projection_expression, <<"Id, PostedBy, ReplyDateTime">>}, @@ -3681,7 +3681,7 @@ restore_table_from_backup_input_tests(_) -> Tests = [?_ddb_test( {"RestoreTableFromBackup example request", - ?_f(erlcloud_ddb2:restore_table_from_backup(<<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb">>,<<"Thread">>)), " + ?_f(erlcloud_ddb2:restore_table_from_backup(<<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb">>, <<"Thread">>)), " { \"BackupArn\": \"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb\", \"TargetTableName\": \"Thread\" @@ -3924,7 +3924,7 @@ restore_table_from_backup_output_tests(_) -> }}) ], - output_tests(?_f(erlcloud_ddb2:restore_table_from_backup(<<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb">>,<<"Thread">>)), Tests). + output_tests(?_f(erlcloud_ddb2:restore_table_from_backup(<<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb">>, <<"Thread">>)), Tests). %% RestoreTableToPointInTime test based on the API examples: %% https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_RestoreTableToPointInTime.html @@ -4196,8 +4196,8 @@ scan_input_tests(_) -> }), ?_ddb_test( {"Scan example 2 request", - ?_f(erlcloud_ddb2:scan(<<"Reply">>, - [{scan_filter, [{<<"PostedBy">>, <<"joe@example.com">>, eq}]}, + ?_f(erlcloud_ddb2:scan(<<"Reply">>, + [{scan_filter, [{<<"PostedBy">>, <<"joe@example.com">>, eq}]}, {return_consumed_capacity, total}])), " { \"TableName\": \"Reply\", @@ -4217,7 +4217,7 @@ scan_input_tests(_) -> ?_ddb_test( {"Scan consistent read", ?_f(erlcloud_ddb2:scan(<<"Reply">>, - [{consistent_read, true}])), " + [{consistent_read, true}])), " { \"TableName\": \"Reply\", \"ConsistentRead\": true @@ -4225,8 +4225,8 @@ scan_input_tests(_) -> }), ?_ddb_test( {"Scan exclusive start key", - ?_f(erlcloud_ddb2:scan(<<"Reply">>, - [{exclusive_start_key, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + ?_f(erlcloud_ddb2:scan(<<"Reply">>, + [{exclusive_start_key, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"LastPostDateTime">>, {n, 20130102054211}}]}])), " { \"TableName\": \"Reply\", @@ -4286,7 +4286,7 @@ scan_input_tests(_) -> ?_ddb_test( {"Scan example with FilterExpression", ?_f(erlcloud_ddb2:scan(<<"Reply">>, - [{filter_expression, <<"PostedBy = :val">>}, + [{filter_expression, <<"PostedBy = :val">>}, {expression_attribute_values, [ {<<":val">>, {s, <<"joe@example.com">>}}]}, {return_consumed_capacity, total}])), " @@ -4649,7 +4649,7 @@ update_continuous_backups_input_tests(_) -> Tests = [?_ddb_test( {"UpdateContinuousBackups example request", - ?_f(erlcloud_ddb2:update_continuous_backups(<<"Thread">>,true)), " + ?_f(erlcloud_ddb2:update_continuous_backups(<<"Thread">>, true)), " { \"PointInTimeRecoverySpecification\": { \"PointInTimeRecoveryEnabled\": true @@ -4693,7 +4693,7 @@ update_continuous_backups_output_tests(_) -> }) ], - output_tests(?_f(erlcloud_ddb2:update_continuous_backups(<<"Thread">>,true)), Tests). + output_tests(?_f(erlcloud_ddb2:update_continuous_backups(<<"Thread">>, true)), Tests). %% UpdateItem test based on the API examples: %% http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html @@ -4701,11 +4701,11 @@ update_item_input_tests(_) -> Tests = [?_ddb_test( {"UpdateItem example request", - ?_f(erlcloud_ddb2:update_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + ?_f(erlcloud_ddb2:update_item(<<"Thread">>, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{<<"LastPostedBy">>, {s, <<"alice@example.com">>}, put}], - [{expected, {<<"LastPostedBy">>, {s, <<"fred@example.com">>}}}, + [{<<"LastPostedBy">>, {s, <<"alice@example.com">>}, put}], + [{expected, {<<"LastPostedBy">>, {s, <<"fred@example.com">>}}}, {return_values, all_new}])), " { \"TableName\": \"Thread\", @@ -4737,10 +4737,10 @@ update_item_input_tests(_) -> ?_ddb_test( {"UpdateItem example request 2", ?_f(erlcloud_ddb2:update_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"A question about updates">>}}], - [{<<"Replies">>, {n, 1}, add}], - [{return_values, none}])), " + [{<<"Replies">>, {n, 1}, add}], + [{return_values, none}])), " { \"TableName\": \"Thread\", \"Key\": { @@ -4765,10 +4765,10 @@ update_item_input_tests(_) -> ?_ddb_test( {"UpdateItem example request with UpdateExpression and ConditionExpression", ?_f(erlcloud_ddb2:update_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"Maximum number of items?">>}}], - <<"set LastPostedBy = :val1">>, - [{condition_expression, <<"LastPostedBy = :val2">>}, + <<"set LastPostedBy = :val1">>, + [{condition_expression, <<"LastPostedBy = :val2">>}, {expression_attribute_values, [ {<<":val1">>, "alice@example.com"}, {<<":val2">>, "fred@example.com"}]}, @@ -4795,9 +4795,9 @@ update_item_input_tests(_) -> ?_ddb_test( {"UpdateItem no attribute updates", ?_f(erlcloud_ddb2:update_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"A question about updates">>}}], - [])), " + [])), " { \"TableName\": \"Thread\", \"Key\": { @@ -5411,5 +5411,5 @@ update_time_to_live_output_tests(_) -> {ok, #ddb2_time_to_live_specification{ attribute_name = <<"ExpirationTime">>, enabled = true}}})], - output_tests(?_f(erlcloud_ddb2:update_time_to_live(<<"SessionData">>, - [{attribute_name, <<"ExpirationTime">>}, {enabled, true}])), Tests). + output_tests(?_f(erlcloud_ddb2:update_time_to_live(<<"SessionData">>, + [{attribute_name, <<"ExpirationTime">>}, {enabled, true}])), Tests). From ded9c8870fa8bb26d95325aca6dd0e43fb7ae6fc Mon Sep 17 00:00:00 2001 From: Yakov Kozlov Date: Tue, 4 Dec 2018 00:46:56 +0300 Subject: [PATCH 017/310] Increase erlcloud_sts max session duration to 12h --- include/erlcloud_aws.hrl | 2 +- src/erlcloud_aws.erl | 4 ++-- src/erlcloud_sts.erl | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index b1fff2bda..61663f567 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -4,7 +4,7 @@ -record(aws_assume_role,{ role_arn :: string() | undefined, session_name = "erlcloud" :: string(), - duration_secs = 900 :: 900..3600, + duration_secs = 900 :: 900..43200, external_id :: string() | undefined }). diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 9f1732a9a..eeb5f38fd 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -67,7 +67,7 @@ -record(profile_options, { session_name :: string(), - session_secs :: 900..3600, + session_secs :: 900..43200, external_id :: string() }). @@ -1163,7 +1163,7 @@ profile( Name ) -> -type profile_option() :: {role_session_name, string()} - | {role_session_secs, 900..3600}. + | {role_session_secs, 900..43200}. %%%--------------------------------------------------------------------------- -spec profile( Name :: atom(), Options :: [profile_option()] ) -> diff --git a/src/erlcloud_sts.erl b/src/erlcloud_sts.erl index 2f5a4dfdd..1882b050a 100644 --- a/src/erlcloud_sts.erl +++ b/src/erlcloud_sts.erl @@ -22,11 +22,11 @@ assume_role(AwsConfig, RoleArn, RoleSessionName, DurationSeconds) -> % See http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html --spec assume_role(#aws_config{}, string(), string(), 900..3600, undefined | string()) -> {#aws_config{}, proplist()} | no_return(). +-spec assume_role(#aws_config{}, string(), string(), 900..43200, undefined | string()) -> {#aws_config{}, proplist()} | no_return(). assume_role(AwsConfig, RoleArn, RoleSessionName, DurationSeconds, ExternalId) when length(RoleArn) >= 20, length(RoleSessionName) >= 2, length(RoleSessionName) =< 64, - DurationSeconds >= 900, DurationSeconds =< 3600 -> + DurationSeconds >= 900, DurationSeconds =< 43200 -> Params = [ From 4e0a2788ea2d4757b8c9160250592ef22a810c98 Mon Sep 17 00:00:00 2001 From: motobob Date: Mon, 3 Dec 2018 17:12:00 -0600 Subject: [PATCH 018/310] Update README.md --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 333d62a2e..439f1dcc6 100644 --- a/README.md +++ b/README.md @@ -66,10 +66,11 @@ Below is the library roadmap update along with regular features and fixes. ### Supported Erlang versions At the moment we support the following OTP releases: - - 17.5 - - 18.1 - - 19.1 - - 20.0 + - 19.3 + - 20.3 + - 21.1 + +it might still work on 17+ (primariliy due to Erlang maps) but we do not guarantee that. ## Getting started ## You need to clone the repository and download rebar/rebar3 (if it's not already available in your path). From 28c519dfd8733f6d614e2d4a1565721a5804ce6e Mon Sep 17 00:00:00 2001 From: tomasz Date: Tue, 4 Dec 2018 13:47:05 +0100 Subject: [PATCH 019/310] Handle Number data without explicit type in SQS attributes --- src/erlcloud_sqs.erl | 2 ++ test/erlcloud_sqs_tests.erl | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index f4a0cfc6e..5a089ff85 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -414,6 +414,8 @@ decode_message_attribute(Attributes) -> end, [F(Attr) || Attr <- Attributes]. +decode_message_attribute_value(["Number"], Value) -> + list_to_integer(Value); decode_message_attribute_value(["Number", "int"], Value) -> list_to_integer(Value); decode_message_attribute_value(["Number", "int", CustomType], Value) -> diff --git a/test/erlcloud_sqs_tests.erl b/test/erlcloud_sqs_tests.erl index edfa5f8ac..a9e4c8608 100644 --- a/test/erlcloud_sqs_tests.erl +++ b/test/erlcloud_sqs_tests.erl @@ -305,6 +305,13 @@ receive_messages_with_message_attributes(_) -> 42 + + number + + Number + 56 + + binary @@ -342,6 +349,7 @@ receive_messages_with_message_attributes(_) -> {"content-type", "application/json"}, {"float", 3.1415926}, {"integer", 42}, + {"number", 56}, {"binary", <<"Binary string">>}, {"uuid", {"uuid", <<"db3bf1fc-0cac-4cf8-8d2c-5c307ad4ac3a">>}} ]} From 031090d7d278ab068fd5205cb3268592406b6857 Mon Sep 17 00:00:00 2001 From: tomasz Date: Tue, 4 Dec 2018 15:25:15 +0100 Subject: [PATCH 020/310] Refactor erlcloud_sqs:decode_message_attribute_value and add test for invalid attribute --- src/erlcloud_sqs.erl | 13 +++++++--- test/erlcloud_sqs_tests.erl | 51 ++++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index 5a089ff85..0c6194209 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -414,7 +414,7 @@ decode_message_attribute(Attributes) -> end, [F(Attr) || Attr <- Attributes]. -decode_message_attribute_value(["Number"], Value) -> +decode_message_attribute_value("Number", Value) -> list_to_integer(Value); decode_message_attribute_value(["Number", "int"], Value) -> list_to_integer(Value); @@ -424,16 +424,21 @@ decode_message_attribute_value(["Number", "float"], Value) -> list_to_float(Value); decode_message_attribute_value(["Number", "float", CustomType], Value) -> {CustomType, list_to_float(Value)}; -decode_message_attribute_value(["String"], Value) -> +decode_message_attribute_value("String", Value) -> Value; decode_message_attribute_value(["String", CustomType], Value) -> {CustomType, Value}; -decode_message_attribute_value(["Binary"], Value) -> +decode_message_attribute_value("Binary", Value) -> list_to_binary(Value); decode_message_attribute_value(["Binary", CustomType], Value) -> {CustomType, list_to_binary(Value)}; decode_message_attribute_value(DataType, Value) -> - decode_message_attribute_value(string:tokens(DataType, "."), Value). + case string:tokens(DataType, ".") of + [_Other] -> % check if datatype is something not handled above by Number/String/Binary + erlang:error(decode_message_attribute_value_error, [DataType, Value]); + Parsed -> + decode_message_attribute_value(Parsed, Value) + end. decode_msg_attributes(Attrs) -> [{decode_msg_attribute_name(Name), diff --git a/test/erlcloud_sqs_tests.erl b/test/erlcloud_sqs_tests.erl index a9e4c8608..5e71a9814 100644 --- a/test/erlcloud_sqs_tests.erl +++ b/test/erlcloud_sqs_tests.erl @@ -22,7 +22,8 @@ erlcloud_api_test_() -> fun send_message_with_message_attributes/1, fun receive_messages_with_message_attributes/1, fun send_message_batch/1, - fun send_message_batch_with_message_attributes/1 + fun send_message_batch_with_message_attributes/1, + fun receive_messages_with_unknown_attributes/1 ]}. start() -> @@ -306,11 +307,11 @@ receive_messages_with_message_attributes(_) -> - number - - Number - 56 - + number + + Number + 56 + binary @@ -360,6 +361,44 @@ receive_messages_with_message_attributes(_) -> MessageResponse, Expected})], output_tests(?_f(erlcloud_sqs:receive_message("Queue", all, 1, 30, none, all, erlcloud_aws:default_config())), Tests). + receive_messages_with_unknown_attributes(_) -> + MessageResponse = " + + + + 5fea7756-0ea4-451a-a703-a558b933e274 + MbZj6wDWli+JvwwJaBV+3dcjk2YW2vA3+STFFljTM8tJJg6HRG6PYSasuWXPJB+CwLj1FjgXUv1uSj1gUPAWV66FU/WeR4mq2OKpEGYWbnLmpRCJVAyeMjeU5ZBdtcQ+QEauMZc8ZRv37sIW2iJKq3M9MFx1YvV11A2x/KSbkJ0= + fafb00f5732ab283681e124bf8747ed1 + This is a test message + + unknown + + Unknown + invalid + + + + + + + b6633655-283d-45b4-aee4-4e84e0ae6afa + + + ", + + F = fun() -> + ?assertException(error,decode_message_attribute_value_error, + erlcloud_sqs:receive_message("Queue", all, 1, 30, none, all, erlcloud_aws:default_config())), + decode_message_attribute_value_error + end, + + Tests = + [?_sqs_test( + {"Test receives a message with message unknown attribute.", + MessageResponse, decode_message_attribute_value_error})], + + output_tests(F, Tests). + send_message_batch(_) -> Batch = [{"batch-message-01", "Hello", [], [{message_group_id, "GroupId"}]}, {"batch-message-02", "World", [], [{message_deduplication_id, "DedupId"}]}], From 727270079c545e16b02d0ea04ab0b080f902c2e2 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Wed, 5 Dec 2018 01:26:12 +0800 Subject: [PATCH 021/310] add list_tables_all and wait_for_table_active functions --- src/erlcloud_ddb_util.erl | 49 ++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index 5d6f22012..0391a247a 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -23,11 +23,12 @@ delete_hash_key/3, delete_hash_key/4, delete_hash_key/5, get_all/2, get_all/3, get_all/4, put_all/2, put_all/3, put_all/4, - list_all_tables/0, list_all_tables/1, + list_tables_all/0, list_tables_all/1, q_all/2, q_all/3, q_all/4, scan_all/1, scan_all/2, scan_all/3, - wait_for_table/1, wait_for_table/2, wait_for_table/3, - write_all/2, write_all/3, write_all/4 + wait_for_table_active/1, wait_for_table_active/2, wait_for_table_active/4, + write_all/2, write_all/3, write_all/4, + flatreverse/1 ]). -ifdef(TEST). @@ -233,23 +234,23 @@ batch_get_retry(RequestItems, DdbOpts, Config, Acc) -> end. %%%------------------------------------------------------------------------------ -%%% list_all_tables +%%% list_tables_all %%%------------------------------------------------------------------------------ -list_all_tables() -> - list_all_tables(default_config()). +list_tables_all() -> + list_tables_all(default_config()). --spec list_all_tables(aws_config()) -> {ok, [table_name()]} | {error, any()}. -list_all_tables(Config) -> - do_list_all_tables(undefined, Config, []). +-spec list_tables_all(aws_config()) -> {ok, [table_name()]} | {error, any()}. +list_tables_all(Config) -> + do_list_tables_all(undefined, Config, []). -do_list_all_tables(LastTable, Config, Result) -> +do_list_tables_all(LastTable, Config, Result) -> Options = [{exclusive_start_table_name, LastTable}, {out, record}], case erlcloud_ddb2:list_tables(Options, Config) of {ok, #ddb2_list_tables{table_names = TableNames, last_evaluated_table_name = undefined}} -> - {ok, Result ++ TableNames}; + {ok, flatreverse([Result, TableNames])}; {ok, #ddb2_list_tables{table_names = TableNames, last_evaluated_table_name = LastTableName}} -> - do_list_all_tables(LastTableName, Config, Result ++ TableNames); + do_list_tables_all(LastTableName, Config, flatreverse([Result, TableNames])); {error, _} = Error -> Error end. @@ -509,38 +510,38 @@ batch_write_retry(RequestItems, Config) -> %% ===Example=== %% %% ` -%% erlcloud_ddb2:wait_for_table(<<"TableName">>, 3000, 40, Config) +%% erlcloud_ddb2:wait_for_table_active(<<"TableName">>, 3000, 40, Config) %% ' %% @end %%------------------------------------------------------------------------------ --spec wait_for_table(table_name(), pos_integer(), non_neg_integer(), aws_config()) -> ok | {error, timeout | any()}. -wait_for_table(Table, Interval, RetryTimes, AWSCfg) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> +-spec wait_for_table_active(table_name(), pos_integer(), non_neg_integer(), aws_config()) -> ok | {error, timeout | any()}. +wait_for_table_active(Table, Interval, RetryTimes, AWSCfg) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> case erlcloud_ddb2:describe_table(Table, [{out, record}], AWSCfg) of - {ok, #ddb2_describe_table{table = _#ddb2_table_description{table_status = active}}} -> + {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = active}}} -> ok; {ok, _} -> case RetryTimes of 0 -> timer:sleep(Interval), - wait_for_table(Table, Interval, RetryTimes, AWSCfg); + wait_for_table_active(Table, Interval, RetryTimes, AWSCfg); 1 -> {error, timeout}; _ -> timer:sleep(Interval), - wait_for_table(Table, Interval, RetryTimes - 1, AWSCfg) + wait_for_table_active(Table, Interval, RetryTimes - 1, AWSCfg) end; {error, Reason} -> {error, Reason} end; -wait_for_table(Table, Interval, RetryTimes, AWSCfg) when is_list(Table) -> - wait_for_table(list_to_binary(Table), Interval, RetryTimes, AWSCfg). +wait_for_table_active(Table, Interval, RetryTimes, AWSCfg) when is_list(Table) -> + wait_for_table_active(list_to_binary(Table), Interval, RetryTimes, AWSCfg). -wait_for_table(Table, AWSCfg) -> - wait_for_table(Table, 3000, 0, AWSCfg). +wait_for_table_active(Table, AWSCfg) -> + wait_for_table_active(Table, 3000, 0, AWSCfg). -wait_for_table(Table) -> - wait_for_table(Table, default_config()). +wait_for_table_active(Table) -> + wait_for_table_active(Table, default_config()). write_all_result([ok | T]) -> From f0b6ec8b525351b44a0f153b1a7dbac9539e35b3 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Wed, 5 Dec 2018 01:34:51 +0800 Subject: [PATCH 022/310] add list_tables_all and wait_for_table_active functions --- src/erlcloud_ddb_util.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index 0391a247a..d91ac6341 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -516,20 +516,20 @@ batch_write_retry(RequestItems, Config) -> %%------------------------------------------------------------------------------ -spec wait_for_table_active(table_name(), pos_integer(), non_neg_integer(), aws_config()) -> ok | {error, timeout | any()}. -wait_for_table_active(Table, Interval, RetryTimes, AWSCfg) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> - case erlcloud_ddb2:describe_table(Table, [{out, record}], AWSCfg) of +wait_for_table_active(Table, Interval, RetryTimes, Config) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> + case erlcloud_ddb2:describe_table(Table, [{out, record}], Config) of {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = active}}} -> ok; {ok, _} -> case RetryTimes of 0 -> timer:sleep(Interval), - wait_for_table_active(Table, Interval, RetryTimes, AWSCfg); + wait_for_table_active(Table, Interval, RetryTimes, Config); 1 -> {error, timeout}; _ -> timer:sleep(Interval), - wait_for_table_active(Table, Interval, RetryTimes - 1, AWSCfg) + wait_for_table_active(Table, Interval, RetryTimes - 1, Config) end; {error, Reason} -> {error, Reason} From a618a20990f27fc46f9b2abb6a9e67933d784657 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Fri, 7 Dec 2018 00:22:31 +0800 Subject: [PATCH 023/310] add units tests --- test/erlcloud_ddb_util_tests.erl | 598 +++++++++++++++++++++++++++++++ 1 file changed, 598 insertions(+) diff --git a/test/erlcloud_ddb_util_tests.erl b/test/erlcloud_ddb_util_tests.erl index 250737843..4711c0a48 100644 --- a/test/erlcloud_ddb_util_tests.erl +++ b/test/erlcloud_ddb_util_tests.erl @@ -30,6 +30,7 @@ operation_test_() -> [fun delete_all_tests/1, fun delete_hash_key_tests/1, fun get_all_tests/1, + fun list_tables_all_tests/1, fun put_all_tests/1, fun q_all_tests/1, fun q_all_attributes/0, @@ -37,6 +38,7 @@ operation_test_() -> fun scan_all_tests/1, fun scan_all_attributes/0, fun scan_all_count/0, + fun wait_for_table_active_tests/1, fun write_all_tests/1 ]}. @@ -442,6 +444,26 @@ get_all_tests(_) -> ], multi_call_tests(Tests). +list_tables_all_tests(_) -> + Tests = + [?_ddb_test( + {"list_tables_all return 3 tablenames", + ?_f(erlcloud_ddb_util:list_tables_all()), + [{"{}", + "{\"TableNames\":[\"tab1\",\"tab2\",\"tab3\"]}"}], + {ok, [<<"tab1">>,<<"tab2">>,<<"tab3">>]}}), + + ?_ddb_test( + {"list_tables_all tow batches, keep order", + ?_f(erlcloud_ddb_util:list_tables_all()), + [{"{}", + "{\"LastEvaluatedTableName\": \"tab3\", \"TableNames\":[\"tab1\",\"tab2\",\"tab3\"]}"}, + {"{\"ExclusiveStartTableName\":\"tab3\"}", + "{\"TableNames\":[\"tab5\",\"tab4\"]}"}], + {ok, [<<"tab1">>,<<"tab2">>,<<"tab3">>,<<"tab5">>,<<"tab4">>]}}) + ], + multi_call_tests(Tests). + put_all_tests(_) -> Tests = [?_ddb_test( @@ -804,6 +826,582 @@ scan_all_count() -> erlcloud_ddb_util:scan_all(<<"tbl">>, [])), meck:unload(EDDB). +wait_for_table_active_tests(_) -> + Tests = + [?_ddb_test( + {"wait_for_table_active table is active", + ?_f(erlcloud_ddb_util:wait_for_table_active(<<"Thread">>)), + [{"{\"TableName\":\"Thread\"}", + " +{ + \"Table\": { + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"Subject\", + \"AttributeType\": \"S\" + } + ], + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"SubjectIndex\", + \"IndexSizeBytes\": 2048, + \"IndexStatus\": \"CREATING\", + \"ItemCount\": 47, + \"KeySchema\": [ + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"NonKeyAttributes\" : [ + \"Author\" + ], + \"ProjectionType\": \"INCLUDE\" + }, + \"ProvisionedThroughput\": { + \"LastDecreaseDateTime\": 0, + \"LastIncreaseDateTime\": 1, + \"NumberOfDecreasesToday\": 2, + \"ReadCapacityUnits\": 3, + \"WriteCapacityUnits\": 4 + } + } + ], + \"CreationDateTime\": 1.363729002358E9, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"RANGE\" + } + ], + \"LocalSecondaryIndexes\": [ + { + \"IndexName\": \"LastPostIndex\", + \"IndexSizeBytes\": 0, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"ProjectionType\": \"KEYS_ONLY\" + } + } + ], + \"ProvisionedThroughput\": { + \"NumberOfDecreasesToday\": 0, + \"ReadCapacityUnits\": 5, + \"WriteCapacityUnits\": 5 + }, + \"TableName\": \"Thread\", + \"TableSizeBytes\": 0, + \"TableStatus\": \"ACTIVE\" + } +} + "}], + ok}), + + ?_ddb_test( + {"wait_for_table_active updating to active, RetryTimes = 0 (infinity)", + ?_f(erlcloud_ddb_util:wait_for_table_active(<<"Thread">>)), + [{"{\"TableName\":\"Thread\"}", + " +{ + \"Table\": { + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"Subject\", + \"AttributeType\": \"S\" + } + ], + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"SubjectIndex\", + \"IndexSizeBytes\": 2048, + \"IndexStatus\": \"CREATING\", + \"ItemCount\": 47, + \"KeySchema\": [ + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"NonKeyAttributes\" : [ + \"Author\" + ], + \"ProjectionType\": \"INCLUDE\" + }, + \"ProvisionedThroughput\": { + \"LastDecreaseDateTime\": 0, + \"LastIncreaseDateTime\": 1, + \"NumberOfDecreasesToday\": 2, + \"ReadCapacityUnits\": 3, + \"WriteCapacityUnits\": 4 + } + } + ], + \"CreationDateTime\": 1.363729002358E9, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"RANGE\" + } + ], + \"LocalSecondaryIndexes\": [ + { + \"IndexName\": \"LastPostIndex\", + \"IndexSizeBytes\": 0, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"ProjectionType\": \"KEYS_ONLY\" + } + } + ], + \"ProvisionedThroughput\": { + \"NumberOfDecreasesToday\": 0, + \"ReadCapacityUnits\": 5, + \"WriteCapacityUnits\": 5 + }, + \"TableName\": \"Thread\", + \"TableSizeBytes\": 0, + \"TableStatus\": \"UPDATING\" + } +} + "}, + {"{\"TableName\":\"Thread\"}", + " +{ + \"Table\": { + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"Subject\", + \"AttributeType\": \"S\" + } + ], + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"SubjectIndex\", + \"IndexSizeBytes\": 2048, + \"IndexStatus\": \"CREATING\", + \"ItemCount\": 47, + \"KeySchema\": [ + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"NonKeyAttributes\" : [ + \"Author\" + ], + \"ProjectionType\": \"INCLUDE\" + }, + \"ProvisionedThroughput\": { + \"LastDecreaseDateTime\": 0, + \"LastIncreaseDateTime\": 1, + \"NumberOfDecreasesToday\": 2, + \"ReadCapacityUnits\": 3, + \"WriteCapacityUnits\": 4 + } + } + ], + \"CreationDateTime\": 1.363729002358E9, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"RANGE\" + } + ], + \"LocalSecondaryIndexes\": [ + { + \"IndexName\": \"LastPostIndex\", + \"IndexSizeBytes\": 0, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"ProjectionType\": \"KEYS_ONLY\" + } + } + ], + \"ProvisionedThroughput\": { + \"NumberOfDecreasesToday\": 0, + \"ReadCapacityUnits\": 5, + \"WriteCapacityUnits\": 5 + }, + \"TableName\": \"Thread\", + \"TableSizeBytes\": 0, + \"TableStatus\": \"ACTIVE\" + } +} + "}], + ok}), + + ?_ddb_test( + {"wait_for_table_active table is deleting", + ?_f(erlcloud_ddb_util:wait_for_table_active(<<"Thread">>)), + [{"{\"TableName\":\"Thread\"}", + " +{ + \"Table\": { + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"Subject\", + \"AttributeType\": \"S\" + } + ], + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"SubjectIndex\", + \"IndexSizeBytes\": 2048, + \"IndexStatus\": \"CREATING\", + \"ItemCount\": 47, + \"KeySchema\": [ + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"NonKeyAttributes\" : [ + \"Author\" + ], + \"ProjectionType\": \"INCLUDE\" + }, + \"ProvisionedThroughput\": { + \"LastDecreaseDateTime\": 0, + \"LastIncreaseDateTime\": 1, + \"NumberOfDecreasesToday\": 2, + \"ReadCapacityUnits\": 3, + \"WriteCapacityUnits\": 4 + } + } + ], + \"CreationDateTime\": 1.363729002358E9, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"RANGE\" + } + ], + \"LocalSecondaryIndexes\": [ + { + \"IndexName\": \"LastPostIndex\", + \"IndexSizeBytes\": 0, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"ProjectionType\": \"KEYS_ONLY\" + } + } + ], + \"ProvisionedThroughput\": { + \"NumberOfDecreasesToday\": 0, + \"ReadCapacityUnits\": 5, + \"WriteCapacityUnits\": 5 + }, + \"TableName\": \"Thread\", + \"TableSizeBytes\": 0, + \"TableStatus\": \"DELETING\" + } +} + "}], + {error, deleting}}), + + ?_ddb_test( + {"wait_for_table_active timeout", + ?_f(erlcloud_ddb_util:wait_for_table_active(<<"Thread">>, 10, 2)), + [{"{\"TableName\":\"Thread\"}", + " +{ + \"Table\": { + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"Subject\", + \"AttributeType\": \"S\" + } + ], + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"SubjectIndex\", + \"IndexSizeBytes\": 2048, + \"IndexStatus\": \"CREATING\", + \"ItemCount\": 47, + \"KeySchema\": [ + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"NonKeyAttributes\" : [ + \"Author\" + ], + \"ProjectionType\": \"INCLUDE\" + }, + \"ProvisionedThroughput\": { + \"LastDecreaseDateTime\": 0, + \"LastIncreaseDateTime\": 1, + \"NumberOfDecreasesToday\": 2, + \"ReadCapacityUnits\": 3, + \"WriteCapacityUnits\": 4 + } + } + ], + \"CreationDateTime\": 1.363729002358E9, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"RANGE\" + } + ], + \"LocalSecondaryIndexes\": [ + { + \"IndexName\": \"LastPostIndex\", + \"IndexSizeBytes\": 0, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"ProjectionType\": \"KEYS_ONLY\" + } + } + ], + \"ProvisionedThroughput\": { + \"NumberOfDecreasesToday\": 0, + \"ReadCapacityUnits\": 5, + \"WriteCapacityUnits\": 5 + }, + \"TableName\": \"Thread\", + \"TableSizeBytes\": 0, + \"TableStatus\": \"UPDATING\" + } +} + "}, + {"{\"TableName\":\"Thread\"}", + " +{ + \"Table\": { + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"Subject\", + \"AttributeType\": \"S\" + } + ], + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"SubjectIndex\", + \"IndexSizeBytes\": 2048, + \"IndexStatus\": \"CREATING\", + \"ItemCount\": 47, + \"KeySchema\": [ + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"NonKeyAttributes\" : [ + \"Author\" + ], + \"ProjectionType\": \"INCLUDE\" + }, + \"ProvisionedThroughput\": { + \"LastDecreaseDateTime\": 0, + \"LastIncreaseDateTime\": 1, + \"NumberOfDecreasesToday\": 2, + \"ReadCapacityUnits\": 3, + \"WriteCapacityUnits\": 4 + } + } + ], + \"CreationDateTime\": 1.363729002358E9, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"RANGE\" + } + ], + \"LocalSecondaryIndexes\": [ + { + \"IndexName\": \"LastPostIndex\", + \"IndexSizeBytes\": 0, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"ProjectionType\": \"KEYS_ONLY\" + } + } + ], + \"ProvisionedThroughput\": { + \"NumberOfDecreasesToday\": 0, + \"ReadCapacityUnits\": 5, + \"WriteCapacityUnits\": 5 + }, + \"TableName\": \"Thread\", + \"TableSizeBytes\": 0, + \"TableStatus\": \"UPDATING\" + } +} + "}], + {error, timeout}}) + ], + multi_call_tests(Tests). + %% Currently don't have tests for the parallel write (more than 25 items). write_all_tests(_) -> Tests = From 40debff327548152cc379936ec1687703df89c99 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Fri, 7 Dec 2018 00:23:41 +0800 Subject: [PATCH 024/310] add list_tables_all and wait_for_table_active functions --- src/erlcloud_ddb_util.erl | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index d91ac6341..800bb8bf4 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -26,7 +26,7 @@ list_tables_all/0, list_tables_all/1, q_all/2, q_all/3, q_all/4, scan_all/1, scan_all/2, scan_all/3, - wait_for_table_active/1, wait_for_table_active/2, wait_for_table_active/4, + wait_for_table_active/1, wait_for_table_active/2, wait_for_table_active/3, wait_for_table_active/4, write_all/2, write_all/3, write_all/4, flatreverse/1 ]). @@ -248,9 +248,9 @@ do_list_tables_all(LastTable, Config, Result) -> Options = [{exclusive_start_table_name, LastTable}, {out, record}], case erlcloud_ddb2:list_tables(Options, Config) of {ok, #ddb2_list_tables{table_names = TableNames, last_evaluated_table_name = undefined}} -> - {ok, flatreverse([Result, TableNames])}; + {ok, flatreverse([TableNames, Result])}; {ok, #ddb2_list_tables{table_names = TableNames, last_evaluated_table_name = LastTableName}} -> - do_list_tables_all(LastTableName, Config, flatreverse([Result, TableNames])); + do_list_tables_all(LastTableName, Config, flatreverse([TableNames, Result])); {error, _} = Error -> Error end. @@ -515,11 +515,14 @@ batch_write_retry(RequestItems, Config) -> %% @end %%------------------------------------------------------------------------------ --spec wait_for_table_active(table_name(), pos_integer(), non_neg_integer(), aws_config()) -> ok | {error, timeout | any()}. +-spec wait_for_table_active(table_name(), pos_integer(), non_neg_integer(), aws_config()) -> + ok | {error, deleting, timeout | any()}. wait_for_table_active(Table, Interval, RetryTimes, Config) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> case erlcloud_ddb2:describe_table(Table, [{out, record}], Config) of {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = active}}} -> ok; + {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = deleting}}} -> + {error, deleting}; {ok, _} -> case RetryTimes of 0 -> @@ -533,9 +536,10 @@ wait_for_table_active(Table, Interval, RetryTimes, Config) when is_binary(Table) end; {error, Reason} -> {error, Reason} - end; -wait_for_table_active(Table, Interval, RetryTimes, AWSCfg) when is_list(Table) -> - wait_for_table_active(list_to_binary(Table), Interval, RetryTimes, AWSCfg). + end. + +wait_for_table_active(Table, Interval, RetryTimes) -> + wait_for_table_active(Table, Interval, RetryTimes, default_config()). wait_for_table_active(Table, AWSCfg) -> wait_for_table_active(Table, 3000, 0, AWSCfg). From 0bd74ec3584230223b66aba37c44b0259e79c71d Mon Sep 17 00:00:00 2001 From: pengzheng Date: Fri, 7 Dec 2018 00:32:56 +0800 Subject: [PATCH 025/310] add list_tables_all and wait_for_table_active functions --- src/erlcloud_ddb_util.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index 800bb8bf4..55bd2fb25 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -516,7 +516,7 @@ batch_write_retry(RequestItems, Config) -> %%------------------------------------------------------------------------------ -spec wait_for_table_active(table_name(), pos_integer(), non_neg_integer(), aws_config()) -> - ok | {error, deleting, timeout | any()}. + ok | {error, deleting | timeout | any()}. wait_for_table_active(Table, Interval, RetryTimes, Config) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> case erlcloud_ddb2:describe_table(Table, [{out, record}], Config) of {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = active}}} -> From 288bfd57891308d80687ead5702a735ccce1f1d0 Mon Sep 17 00:00:00 2001 From: pengzheng Date: Fri, 7 Dec 2018 00:40:48 +0800 Subject: [PATCH 026/310] add list_tables_all and wait_for_table_active functions --- test/erlcloud_ddb2_tests.erl | 122 +++++++++++++++++------------------ 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index cddac432d..9195d3e3c 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -874,7 +874,7 @@ create_backup_input_tests(_) -> Tests = [?_ddb_test( {"CreateBackup example request", - ?_f(erlcloud_ddb2:create_backup(<<"Forum_Backup">>, <<"Forum">>)), " + ?_f(erlcloud_ddb2:create_backup(<<"Forum_Backup">>,<<"Forum">>)), " { \"BackupName\": \"Forum_Backup\", \"TableName\": \"Forum\" @@ -915,7 +915,7 @@ create_backup_output_tests(_) -> }) ], - output_tests(?_f(erlcloud_ddb2:create_backup(<<"Forum_Backup">>, <<"Forum">>)), Tests). + output_tests(?_f(erlcloud_ddb2:create_backup(<<"Forum_Backup">>,<<"Forum">>)), Tests). %% CreateGlobalTable input test: create_global_table_input_tests(_) -> @@ -1826,10 +1826,10 @@ delete_item_input_tests(_) -> Tests = [?_ddb_test( {"DeleteItem example request", - ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{return_values, all_old}, + [{return_values, all_old}, {expected, {<<"Replies">>, null}}])), " { \"TableName\": \"Thread\", @@ -1852,9 +1852,9 @@ delete_item_input_tests(_) -> ?_ddb_test( {"DeleteItem example request with ConditionExpression", ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{return_values, all_old}, + [{return_values, all_old}, {condition_expression, <<"attribute_not_exists(Replies)">>}])), " { \"TableName\": \"Thread\", @@ -1872,9 +1872,9 @@ delete_item_input_tests(_) -> }), ?_ddb_test( {"DeleteItem return metrics", - ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, - {<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, - [{return_consumed_capacity, total}, + ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, + {<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{return_consumed_capacity, total}, {return_item_collection_metrics, size}])), " { \"TableName\": \"Thread\", @@ -2737,9 +2737,9 @@ get_item_input_tests(_) -> [?_ddb_test( {"GetItem example request, with fully specified keys", ?_f(erlcloud_ddb2:get_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, + [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, consistent_read, {return_consumed_capacity, total}, %% Make sure options at beginning of list override later options @@ -2748,10 +2748,10 @@ get_item_input_tests(_) -> Example1Response}), ?_ddb_test( {"GetItem example request, with inferred key types", - ?_f(erlcloud_ddb2:get_item(<<"Thread">>, - [{<<"ForumName">>, "Amazon DynamoDB"}, + ?_f(erlcloud_ddb2:get_item(<<"Thread">>, + [{<<"ForumName">>, "Amazon DynamoDB"}, {<<"Subject">>, <<"How do I update multiple items?">>}], - [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, + [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, {consistent_read, true}, {return_consumed_capacity, total}] )), @@ -2759,9 +2759,9 @@ get_item_input_tests(_) -> ?_ddb_test( {"GetItem example request, with ProjectionExpression", ?_f(erlcloud_ddb2:get_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{projection_expression, <<"LastPostDateTime, Message, Tags">>}, + [{projection_expression, <<"LastPostDateTime, Message, Tags">>}, consistent_read, {return_consumed_capacity, total}] )), " @@ -2924,7 +2924,7 @@ list_backups_input_tests(_) -> Tests = [?_ddb_test( {"ListBackups example request", - ?_f(erlcloud_ddb2:list_backups([{limit, 4}, {exclusive_start_backup_arn, <<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523516528459-dfd5667f">>}, {table_name, <<"Forum">>}, {time_range_lower_bound, 1522926603.688}, {time_range_upper_bound, 1523022454.098}])), " + ?_f(erlcloud_ddb2:list_backups([{limit, 4},{exclusive_start_backup_arn, <<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523516528459-dfd5667f">>},{table_name, <<"Forum">>},{time_range_lower_bound,1522926603.688},{time_range_upper_bound, 1523022454.098}])), " { \"Limit\": 4, \"TableName\": \"Forum\", @@ -3001,7 +3001,7 @@ list_global_tables_input_tests(_) -> }), ?_ddb_test( {"ListGlobalTables empty request", - ?_f(erlcloud_ddb2:list_global_tables()), + ?_f(erlcloud_ddb2:list_global_tables()), "{}" }) @@ -3091,7 +3091,7 @@ list_tables_input_tests(_) -> }), ?_ddb_test( {"ListTables empty request", - ?_f(erlcloud_ddb2:list_tables()), + ?_f(erlcloud_ddb2:list_tables()), "{}" }) @@ -3181,14 +3181,14 @@ put_item_input_tests(_) -> Tests = [?_ddb_test( {"PutItem example request", - ?_f(erlcloud_ddb2:put_item(<<"Thread">>, - [{<<"LastPostedBy">>, <<"fred@example.com">>}, + ?_f(erlcloud_ddb2:put_item(<<"Thread">>, + [{<<"LastPostedBy">>, <<"fred@example.com">>}, {<<"ForumName">>, <<"Amazon DynamoDB">>}, {<<"LastPostDateTime">>, <<"201303190422">>}, {<<"Tags">>, {ss, [<<"Update">>, <<"Multiple Items">>, <<"HelpMe">>]}}, {<<"Subject">>, <<"How do I update multiple items?">>}, {<<"Message">>, <<"I want to update multiple items in a single API call. What's the best way to do that?">>}], - [{expected, [{<<"ForumName">>, null}, {<<"Subject">>, null}]}])), " + [{expected, [{<<"ForumName">>, null}, {<<"Subject">>, null}]}])), " { \"TableName\": \"Thread\", \"Item\": { @@ -3223,11 +3223,11 @@ put_item_input_tests(_) -> }), ?_ddb_test( {"PutItem float inputs", - ?_f(erlcloud_ddb2:put_item(<<"Thread">>, - [{<<"typed float">>, {n, 1.2}}, + ?_f(erlcloud_ddb2:put_item(<<"Thread">>, + [{<<"typed float">>, {n, 1.2}}, {<<"untyped float">>, 3.456}, {<<"mixed set">>, {ns, [7.8, 9.0, 10]}}], - [])), " + [])), " { \"TableName\": \"Thread\", \"Item\": { @@ -3246,13 +3246,13 @@ put_item_input_tests(_) -> ?_ddb_test( {"PutItem example request with ConditionExpression and ExpressionAttributeValues", ?_f(erlcloud_ddb2:put_item(<<"Thread">>, - [{<<"LastPostedBy">>, <<"fred@example.com">>}, + [{<<"LastPostedBy">>, <<"fred@example.com">>}, {<<"ForumName">>, <<"Amazon DynamoDB">>}, {<<"LastPostDateTime">>, <<"201303190422">>}, {<<"Tags">>, {ss, [<<"Update">>, <<"Multiple Items">>, <<"HelpMe">>]}}, {<<"Subject">>, <<"How do I update multiple items?">>}, {<<"Message">>, <<"I want to update multiple items in a single API call. What's the best way to do that?">>}], - [{condition_expression, <<"ForumName <> :f and Subject <> :s">>}, + [{condition_expression, <<"ForumName <> :f and Subject <> :s">>}, {expression_attribute_values, [ {<<":f">>, <<"Amazon DynamoDB">>}, {<<":s">>, <<"How do I update multiple items?">>}]}])), " @@ -3292,7 +3292,7 @@ put_item_input_tests(_) -> ?_ddb_test( {"PutItem request with complex item", ?_f(erlcloud_ddb2:put_item(<<"Table">>, - [{<<"bool_true">>, {bool, true}}, + [{<<"bool_true">>, {bool, true}}, {<<"bool_false">>, {bool, false}}, {<<"null_value">>, {null, true}}, {<<"list_value">>, {l, ["string", {ss, ["string1", "string2"]}]}}, @@ -3441,9 +3441,9 @@ q_input_tests(_) -> [?_ddb_test( {"Query example 1 request", ?_f(erlcloud_ddb2:q(<<"Thread">>, - [{<<"LastPostDateTime">>, {{s, <<"20130101">>}, {s, <<"20130115">>}}, between}, + [{<<"LastPostDateTime">>, {{s, <<"20130101">>}, {s, <<"20130115">>}}, between}, {<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}], - [{index_name, <<"LastPostIndex">>}, + [{index_name, <<"LastPostIndex">>}, {select, all_attributes}, {limit, 3}, {consistent_read, true}])), " @@ -3479,8 +3479,8 @@ q_input_tests(_) -> ?_ddb_test( {"Query example 2 request", ?_f(erlcloud_ddb2:q(<<"Thread">>, - {<<"ForumName">>, <<"Amazon DynamoDB">>, eq}, - [{select, count}, + {<<"ForumName">>, <<"Amazon DynamoDB">>, eq}, + [{select, count}, {consistent_read, true}])), " { \"TableName\": \"Thread\", @@ -3501,8 +3501,8 @@ q_input_tests(_) -> ?_ddb_test( {"Query exclusive start key", ?_f(erlcloud_ddb2:q(<<"Thread">>, - [{<<"ForumName">>, <<"Amazon DynamoDB">>, eq}], - [{select, count}, + [{<<"ForumName">>, <<"Amazon DynamoDB">>, eq}], + [{select, count}, {index_name, <<"LastPostIndex">>}, {exclusive_start_key, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"Exclusive key can have 3 parts">>}}, @@ -3537,8 +3537,8 @@ q_input_tests(_) -> ?_ddb_test( {"Query example request with KeyConditionExpression, ProjectionExpression and ExpressionAttributeValues", ?_f(erlcloud_ddb2:q(<<"Reply">>, - <<"Id = :v1 AND PostedBy BETWEEN :v2a AND :v2b">>, - [{index_name, <<"PostedBy-Index">>}, + <<"Id = :v1 AND PostedBy BETWEEN :v2a AND :v2b">>, + [{index_name, <<"PostedBy-Index">>}, {limit, 3}, {consistent_read, true}, {projection_expression, <<"Id, PostedBy, ReplyDateTime">>}, @@ -3681,7 +3681,7 @@ restore_table_from_backup_input_tests(_) -> Tests = [?_ddb_test( {"RestoreTableFromBackup example request", - ?_f(erlcloud_ddb2:restore_table_from_backup(<<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb">>, <<"Thread">>)), " + ?_f(erlcloud_ddb2:restore_table_from_backup(<<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb">>,<<"Thread">>)), " { \"BackupArn\": \"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb\", \"TargetTableName\": \"Thread\" @@ -3924,7 +3924,7 @@ restore_table_from_backup_output_tests(_) -> }}) ], - output_tests(?_f(erlcloud_ddb2:restore_table_from_backup(<<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb">>, <<"Thread">>)), Tests). + output_tests(?_f(erlcloud_ddb2:restore_table_from_backup(<<"arn:aws:dynamodb:us-east-1:387047610112:table/Forum/backup/01523517555423-69b67bcb">>,<<"Thread">>)), Tests). %% RestoreTableToPointInTime test based on the API examples: %% https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_RestoreTableToPointInTime.html @@ -4196,8 +4196,8 @@ scan_input_tests(_) -> }), ?_ddb_test( {"Scan example 2 request", - ?_f(erlcloud_ddb2:scan(<<"Reply">>, - [{scan_filter, [{<<"PostedBy">>, <<"joe@example.com">>, eq}]}, + ?_f(erlcloud_ddb2:scan(<<"Reply">>, + [{scan_filter, [{<<"PostedBy">>, <<"joe@example.com">>, eq}]}, {return_consumed_capacity, total}])), " { \"TableName\": \"Reply\", @@ -4217,7 +4217,7 @@ scan_input_tests(_) -> ?_ddb_test( {"Scan consistent read", ?_f(erlcloud_ddb2:scan(<<"Reply">>, - [{consistent_read, true}])), " + [{consistent_read, true}])), " { \"TableName\": \"Reply\", \"ConsistentRead\": true @@ -4225,8 +4225,8 @@ scan_input_tests(_) -> }), ?_ddb_test( {"Scan exclusive start key", - ?_f(erlcloud_ddb2:scan(<<"Reply">>, - [{exclusive_start_key, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + ?_f(erlcloud_ddb2:scan(<<"Reply">>, + [{exclusive_start_key, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"LastPostDateTime">>, {n, 20130102054211}}]}])), " { \"TableName\": \"Reply\", @@ -4286,7 +4286,7 @@ scan_input_tests(_) -> ?_ddb_test( {"Scan example with FilterExpression", ?_f(erlcloud_ddb2:scan(<<"Reply">>, - [{filter_expression, <<"PostedBy = :val">>}, + [{filter_expression, <<"PostedBy = :val">>}, {expression_attribute_values, [ {<<":val">>, {s, <<"joe@example.com">>}}]}, {return_consumed_capacity, total}])), " @@ -4649,7 +4649,7 @@ update_continuous_backups_input_tests(_) -> Tests = [?_ddb_test( {"UpdateContinuousBackups example request", - ?_f(erlcloud_ddb2:update_continuous_backups(<<"Thread">>, true)), " + ?_f(erlcloud_ddb2:update_continuous_backups(<<"Thread">>,true)), " { \"PointInTimeRecoverySpecification\": { \"PointInTimeRecoveryEnabled\": true @@ -4693,7 +4693,7 @@ update_continuous_backups_output_tests(_) -> }) ], - output_tests(?_f(erlcloud_ddb2:update_continuous_backups(<<"Thread">>, true)), Tests). + output_tests(?_f(erlcloud_ddb2:update_continuous_backups(<<"Thread">>,true)), Tests). %% UpdateItem test based on the API examples: %% http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html @@ -4701,11 +4701,11 @@ update_item_input_tests(_) -> Tests = [?_ddb_test( {"UpdateItem example request", - ?_f(erlcloud_ddb2:update_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + ?_f(erlcloud_ddb2:update_item(<<"Thread">>, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], - [{<<"LastPostedBy">>, {s, <<"alice@example.com">>}, put}], - [{expected, {<<"LastPostedBy">>, {s, <<"fred@example.com">>}}}, + [{<<"LastPostedBy">>, {s, <<"alice@example.com">>}, put}], + [{expected, {<<"LastPostedBy">>, {s, <<"fred@example.com">>}}}, {return_values, all_new}])), " { \"TableName\": \"Thread\", @@ -4737,10 +4737,10 @@ update_item_input_tests(_) -> ?_ddb_test( {"UpdateItem example request 2", ?_f(erlcloud_ddb2:update_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"A question about updates">>}}], - [{<<"Replies">>, {n, 1}, add}], - [{return_values, none}])), " + [{<<"Replies">>, {n, 1}, add}], + [{return_values, none}])), " { \"TableName\": \"Thread\", \"Key\": { @@ -4765,10 +4765,10 @@ update_item_input_tests(_) -> ?_ddb_test( {"UpdateItem example request with UpdateExpression and ConditionExpression", ?_f(erlcloud_ddb2:update_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"Maximum number of items?">>}}], - <<"set LastPostedBy = :val1">>, - [{condition_expression, <<"LastPostedBy = :val2">>}, + <<"set LastPostedBy = :val1">>, + [{condition_expression, <<"LastPostedBy = :val2">>}, {expression_attribute_values, [ {<<":val1">>, "alice@example.com"}, {<<":val2">>, "fred@example.com"}]}, @@ -4795,9 +4795,9 @@ update_item_input_tests(_) -> ?_ddb_test( {"UpdateItem no attribute updates", ?_f(erlcloud_ddb2:update_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"A question about updates">>}}], - [])), " + [])), " { \"TableName\": \"Thread\", \"Key\": { @@ -5411,5 +5411,5 @@ update_time_to_live_output_tests(_) -> {ok, #ddb2_time_to_live_specification{ attribute_name = <<"ExpirationTime">>, enabled = true}}})], - output_tests(?_f(erlcloud_ddb2:update_time_to_live(<<"SessionData">>, - [{attribute_name, <<"ExpirationTime">>}, {enabled, true}])), Tests). + output_tests(?_f(erlcloud_ddb2:update_time_to_live(<<"SessionData">>, + [{attribute_name, <<"ExpirationTime">>}, {enabled, true}])), Tests). From bc6480518d8f3f446b4d0321a63d5582c74a44eb Mon Sep 17 00:00:00 2001 From: pengzheng Date: Fri, 7 Dec 2018 00:50:14 +0800 Subject: [PATCH 027/310] add list_tables_all and wait_for_table_active functions --- src/erlcloud_ddb_util.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index 55bd2fb25..e82111629 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -27,8 +27,7 @@ q_all/2, q_all/3, q_all/4, scan_all/1, scan_all/2, scan_all/3, wait_for_table_active/1, wait_for_table_active/2, wait_for_table_active/3, wait_for_table_active/4, - write_all/2, write_all/3, write_all/4, - flatreverse/1 + write_all/2, write_all/3, write_all/4 ]). -ifdef(TEST). From d084e35ee2cba413d5bad5df1d6ee63dae480d6f Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Fri, 7 Dec 2018 13:09:39 +0100 Subject: [PATCH 028/310] fix some long lines --- src/erlcloud_s3.erl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index bad2fa41a..893ea85b2 100755 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -315,9 +315,11 @@ extract_delete_objects_batch_err_contents(Nodes) -> to_flat_format([{key,Key},{code,Code},{message,Message}]) -> {Key,Code,Message}. -% returns paths list from AWS S3 root directory, used as input to delete_objects_batch +% returns paths list from AWS S3 root directory, used as input to +% delete_objects_batch % example : -% 25> rp(erlcloud_s3:explore_dirstructure("xmppfiledev", ["sailfish/deleteme"], [])). +% 25> rp(erlcloud_s3:explore_dirstructure("xmppfiledev", +% ["sailfish/deleteme"], [])). % ["sailfish/deleteme/deep/deep1/deep4/ZZZ_1.txt", % "sailfish/deleteme/deep/deep1/deep4/ZZZ_0.txt", % "sailfish/deleteme/deep/deep1/ZZZ_0.txt", @@ -328,28 +330,35 @@ to_flat_format([{key,Key},{code,Code},{message,Message}]) -> explore_dirstructure(Bucketname, Branches, Accum) -> explore_dirstructure(Bucketname, Branches, Accum, default_config()). --spec explore_dirstructure(string(), list(), list(), aws_config()) -> list() | no_return(). +-spec explore_dirstructure(string(), list(), list(), aws_config()) -> + list() | no_return(). explore_dirstructure(_, [], Result, _Config) -> lists:append(Result); explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config) when is_record(Config, aws_config) -> ProcessContent = fun(Data)-> Content = proplists:get_value(contents, Data), - lists:foldl(fun(I,Acc)-> R = proplists:get_value(key, I), [R|Acc] end, [], Content) + lists:foldl(fun(I,Acc)-> R = proplists:get_value(key, I), + [R|Acc] end, [], Content) end, - Data = list_objects(Bucketname, [{prefix, Branch}, {delimiter, "/"}], Config), + Data = list_objects(Bucketname, [{prefix, Branch}, + {delimiter, "/"}], Config), case proplists:get_value(common_prefixes, Data) of [] -> % it has reached end of the branch Files = ProcessContent(Data), explore_dirstructure(Bucketname, Tail, [Files|Accum], Config); Sub -> Files = ProcessContent(Data), - List = lists:foldl(fun(I,Acc)-> R = proplists:get_value(prefix, I), [R|Acc] end, [], Sub), + List = lists:foldl(fun(I,Acc)-> R = proplists:get_value(prefix, I), + [R|Acc] end, [], Sub), Result = explore_dirstructure(Bucketname, List, [], Config), - explore_dirstructure(Bucketname, Tail, [Result, Files|Accum], Config) + explore_dirstructure(Bucketname, Tail, + [Result, Files|Accum], Config) end. + + -spec delete_object(string(), string()) -> proplist() | no_return(). delete_object(BucketName, Key) -> From 3f1b30c1cffeeb98f0bfd3fbc9f01094e800177d Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Fri, 7 Dec 2018 13:11:09 +0100 Subject: [PATCH 029/310] correct comment --- src/erlcloud_s3.erl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 893ea85b2..9dad25f10 100755 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -315,17 +315,17 @@ extract_delete_objects_batch_err_contents(Nodes) -> to_flat_format([{key,Key},{code,Code},{message,Message}]) -> {Key,Code,Message}. -% returns paths list from AWS S3 root directory, used as input to -% delete_objects_batch -% example : -% 25> rp(erlcloud_s3:explore_dirstructure("xmppfiledev", -% ["sailfish/deleteme"], [])). -% ["sailfish/deleteme/deep/deep1/deep4/ZZZ_1.txt", -% "sailfish/deleteme/deep/deep1/deep4/ZZZ_0.txt", -% "sailfish/deleteme/deep/deep1/ZZZ_0.txt", -% "sailfish/deleteme/deep/ZZZ_0.txt"] -% ok -% +%% returns paths list from AWS S3 root directory, used as input to +%% delete_objects_batch +%% example : +%% 25> rp(erlcloud_s3:explore_dirstructure("xmppfiledev", +%% ["sailfish/deleteme"], [])). +%% ["sailfish/deleteme/deep/deep1/deep4/ZZZ_1.txt", +%% "sailfish/deleteme/deep/deep1/deep4/ZZZ_0.txt", +%% "sailfish/deleteme/deep/deep1/ZZZ_0.txt", +%% "sailfish/deleteme/deep/ZZZ_0.txt"] +%% ok +%% -spec explore_dirstructure(string(), list(), list()) -> list() | no_return(). explore_dirstructure(Bucketname, Branches, Accum) -> explore_dirstructure(Bucketname, Branches, Accum, default_config()). From ba23f314bfdef4f491347168de398cb10c139091 Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Fri, 7 Dec 2018 13:36:49 +0100 Subject: [PATCH 030/310] keep exploring on truncated true --- src/erlcloud_s3.erl | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 9dad25f10..1d185d86e 100755 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -332,9 +332,13 @@ explore_dirstructure(Bucketname, Branches, Accum) -> -spec explore_dirstructure(string(), list(), list(), aws_config()) -> list() | no_return(). -explore_dirstructure(_, [], Result, _Config) -> - lists:append(Result); -explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config) +explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config) -> + explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config, []). + + +explore_dirstructure(_, [], Result, _Config, _Marker) -> + lists:append(Result); +explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config, Marker) when is_record(Config, aws_config) -> ProcessContent = fun(Data)-> Content = proplists:get_value(contents, Data), @@ -343,21 +347,24 @@ explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config) end, Data = list_objects(Bucketname, [{prefix, Branch}, - {delimiter, "/"}], Config), - case proplists:get_value(common_prefixes, Data) of - [] -> % it has reached end of the branch - Files = ProcessContent(Data), - explore_dirstructure(Bucketname, Tail, [Files|Accum], Config); - Sub -> - Files = ProcessContent(Data), - List = lists:foldl(fun(I,Acc)-> R = proplists:get_value(prefix, I), - [R|Acc] end, [], Sub), - Result = explore_dirstructure(Bucketname, List, [], Config), - explore_dirstructure(Bucketname, Tail, - [Result, Files|Accum], Config) - end. - - + {delimiter, "/"}, + {marker, Marker}], Config), + Files = ProcessContent(Data), + Sub = proplists:get_value(common_prefixes, Data), + SubDirs = lists:foldl(fun(I,Acc)-> R = proplists:get_value(prefix, I), + [R|Acc] end, [], Sub), + SubFiles = explore_dirstructure(Bucketname, SubDirs, [], Config, []), + TruncFiles = + case proplists:get_value(is_truncated, Data) of + false -> + []; + true -> + NextMarker = proplists:get_value(next_marker, Data), + explore_dirstructure(Bucketname, [Branch], [], Config, + NextMarker) + end, + explore_dirstructure(Bucketname, Tail, + [SubFiles, TruncFiles, Files|Accum], Config, []). -spec delete_object(string(), string()) -> proplist() | no_return(). From c1edaeb80f727d0f1d3627eab1f83eecf820b44d Mon Sep 17 00:00:00 2001 From: pengzheng Date: Sat, 8 Dec 2018 13:57:30 +0800 Subject: [PATCH 031/310] add list_tables_all and wait_for_table_active functions --- src/erlcloud_ddb_util.erl | 13 ++++++------- test/erlcloud_ddb_util_tests.erl | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index e82111629..415e87eec 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -504,7 +504,6 @@ batch_write_retry(RequestItems, Config) -> %%------------------------------------------------------------------------------ %% @doc %% wait until table_status==active. -%% RetryTimes = 0 means infinity. %% %% ===Example=== %% @@ -514,8 +513,8 @@ batch_write_retry(RequestItems, Config) -> %% @end %%------------------------------------------------------------------------------ --spec wait_for_table_active(table_name(), pos_integer(), non_neg_integer(), aws_config()) -> - ok | {error, deleting | timeout | any()}. +-spec wait_for_table_active(table_name(), pos_integer() | infinity, non_neg_integer(), aws_config()) -> + ok | {error, deleting | retry_threshold_exceeded | any()}. wait_for_table_active(Table, Interval, RetryTimes, Config) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> case erlcloud_ddb2:describe_table(Table, [{out, record}], Config) of {ok, #ddb2_describe_table{table = #ddb2_table_description{table_status = active}}} -> @@ -524,11 +523,11 @@ wait_for_table_active(Table, Interval, RetryTimes, Config) when is_binary(Table) {error, deleting}; {ok, _} -> case RetryTimes of - 0 -> + infinity -> timer:sleep(Interval), - wait_for_table_active(Table, Interval, RetryTimes, Config); + wait_for_table_active(Table, infinity, RetryTimes, Config); 1 -> - {error, timeout}; + {error, retry_threshold_exceeded}; _ -> timer:sleep(Interval), wait_for_table_active(Table, Interval, RetryTimes - 1, Config) @@ -541,7 +540,7 @@ wait_for_table_active(Table, Interval, RetryTimes) -> wait_for_table_active(Table, Interval, RetryTimes, default_config()). wait_for_table_active(Table, AWSCfg) -> - wait_for_table_active(Table, 3000, 0, AWSCfg). + wait_for_table_active(Table, 3000, 100, AWSCfg). wait_for_table_active(Table) -> wait_for_table_active(Table, default_config()). diff --git a/test/erlcloud_ddb_util_tests.erl b/test/erlcloud_ddb_util_tests.erl index 4711c0a48..2b4829a5c 100644 --- a/test/erlcloud_ddb_util_tests.erl +++ b/test/erlcloud_ddb_util_tests.erl @@ -926,7 +926,7 @@ wait_for_table_active_tests(_) -> ok}), ?_ddb_test( - {"wait_for_table_active updating to active, RetryTimes = 0 (infinity)", + {"wait_for_table_active updating to active, RetryTimes = infinity", ?_f(erlcloud_ddb_util:wait_for_table_active(<<"Thread">>)), [{"{\"TableName\":\"Thread\"}", " @@ -1212,7 +1212,7 @@ wait_for_table_active_tests(_) -> {error, deleting}}), ?_ddb_test( - {"wait_for_table_active timeout", + {"wait_for_table_active retry_threshold_exceeded", ?_f(erlcloud_ddb_util:wait_for_table_active(<<"Thread">>, 10, 2)), [{"{\"TableName\":\"Thread\"}", " @@ -1398,7 +1398,7 @@ wait_for_table_active_tests(_) -> } } "}], - {error, timeout}}) + {error, retry_threshold_exceeded}}) ], multi_call_tests(Tests). From 7ebee394390a45563ad0af12476612188a41d613 Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Mon, 10 Dec 2018 20:14:33 -0600 Subject: [PATCH 032/310] erlcloud_ddb2: add support for billing mode * Add [billing_mode option](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_CreateTable.html#DDB-CreateTable-request-BillingMode) to create/update_table * Make provisioned throughput for tables and global secondary indices (at create/update) optional * Add [billing_mode_summary](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BillingModeSummary.html) record to table description --- include/erlcloud_ddb2.hrl | 5 ++ src/erlcloud_ddb2.erl | 107 ++++++++++++++++------- test/erlcloud_ddb2_tests.erl | 163 +++++++++++++++++++++++++++++++++++ 3 files changed, 242 insertions(+), 33 deletions(-) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index f55e5f52f..aefc9b6a9 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -43,6 +43,10 @@ replication_group :: undefined | [#ddb2_replica{}] }). +-record(ddb2_billing_mode_summary, + {billing_mode :: undefined | erlcloud_ddb2:billing_mode(), + last_update_to_pay_per_request_date_time :: undefined | date_time()}). + -record(ddb2_provisioned_throughput_description, {last_decrease_date_time :: undefined | date_time(), last_increase_date_time :: undefined | date_time(), @@ -81,6 +85,7 @@ -record(ddb2_table_description, {attribute_definitions :: undefined | erlcloud_ddb2:attr_defs(), + billing_mode_summary :: undefined | #ddb2_billing_mode_summary{}, creation_date_time :: undefined | number(), global_secondary_indexes :: undefined | [#ddb2_global_secondary_index_description{}], item_count :: undefined | integer(), diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index c8f6d75a4..5b7f00473 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -91,7 +91,7 @@ batch_write_item/1, batch_write_item/2, batch_write_item/3, create_backup/2, create_backup/3, create_backup/4, create_global_table/2, create_global_table/3, create_global_table/4, - create_table/5, create_table/6, create_table/7, + create_table/4, create_table/5, create_table/6, create_table/7, delete_backup/1, delete_backup/2, delete_backup/3, delete_item/2, delete_item/3, delete_item/4, delete_table/1, delete_table/2, delete_table/3, @@ -140,6 +140,7 @@ batch_write_item_request_item/0, batch_write_item_return/0, boolean_opt/1, + billing_mode/0, comparison_op/0, condition/0, conditional_op/0, @@ -323,12 +324,15 @@ default_config() -> erlcloud_aws:default_config(). -type read_units() :: pos_integer(). -type write_units() :: pos_integer(). +-type billing_mode() :: provisioned | pay_per_request. + -type index_name() :: binary(). -type projection() :: keys_only | {include, [attr_name()]} | all. --type global_secondary_index_def() :: {index_name(), key_schema(), projection(), read_units(), write_units()}. +-type global_secondary_index_def() :: {index_name(), key_schema(), projection()} | + {index_name(), key_schema(), projection(), read_units(), write_units()}. -type sse_description_status() :: enabling | enabled | disabling | disabled. -type sse_description() :: {status, sse_description_status()}. @@ -490,6 +494,10 @@ dynamize_provisioned_throughput({ReadUnits, WriteUnits}) -> {<<"WriteCapacityUnits">>, WriteUnits}]. -spec dynamize_global_secondary_index(global_secondary_index_def()) -> jsx:json_term(). +dynamize_global_secondary_index({IndexName, KeySchema, Projection}) -> + [{<<"IndexName">>, IndexName}, + {<<"KeySchema">>, dynamize_key_schema(KeySchema)}, + {<<"Projection">>, dynamize_projection(Projection)}]; dynamize_global_secondary_index({IndexName, KeySchema, Projection, ReadUnits, WriteUnits}) -> [{<<"IndexName">>, IndexName}, {<<"KeySchema">>, dynamize_key_schema(KeySchema)}, @@ -1078,6 +1086,20 @@ out(Result, Undynamize, Opts, Index, Default) -> %%% Shared Records %%%------------------------------------------------------------------------------ +-spec undynamize_billing_mode(binary(), undynamize_opts()) -> billing_mode(). +undynamize_billing_mode(<<"PROVISIONED">>, _) -> provisioned; +undynamize_billing_mode(<<"PAY_PER_REQUEST">>, _) -> pay_per_request. + +-spec billing_mode_summary_record() -> record_desc(). +billing_mode_summary_record() -> + {#ddb2_billing_mode_summary{}, + [{<<"BillingMode">>, #ddb2_billing_mode_summary.billing_mode, fun undynamize_billing_mode/2}, + {<<"LastUpdateToPayPerRequestDateTime">>, + #ddb2_billing_mode_summary.last_update_to_pay_per_request_date_time, fun id/2}]}. + +undynamize_billing_mode_summary(V, Opts) -> + undynamize_record(billing_mode_summary_record(), V, Opts). + undynamize_consumed_capacity_units(V, _Opts) -> {_, CapacityUnits} = lists:keyfind(<<"CapacityUnits">>, 1, V), CapacityUnits. @@ -1205,6 +1227,7 @@ restore_summary_record() -> table_description_record() -> {#ddb2_table_description{}, [{<<"AttributeDefinitions">>, #ddb2_table_description.attribute_definitions, fun undynamize_attr_defs/2}, + {<<"BillingModeSummary">>, #ddb2_table_description.billing_mode_summary, fun undynamize_billing_mode_summary/2}, {<<"CreationDateTime">>, #ddb2_table_description.creation_date_time, fun id/2}, {<<"GlobalSecondaryIndexes">>, #ddb2_table_description.global_secondary_indexes, fun(V, Opts) -> [undynamize_record(global_secondary_index_description_record(), I, Opts) || I <- V] end}, @@ -1623,6 +1646,10 @@ create_global_table(GlobalTableName, ReplicationGroup, Opts, Config) -> -type local_secondary_indexes() :: maybe_list(local_secondary_index_def()). -type global_secondary_indexes() :: maybe_list(global_secondary_index_def()). +-spec dynamize_billing_mode(billing_mode()) -> binary(). +dynamize_billing_mode(provisioned) -> <<"PROVISIONED">>; +dynamize_billing_mode(pay_per_request) -> <<"PAY_PER_REQUEST">>. + -spec dynamize_local_secondary_index(hash_key_name(), local_secondary_index_def()) -> jsx:json_term(). dynamize_local_secondary_index(HashKey, {IndexName, RangeKey, Projection}) -> [{<<"IndexName">>, IndexName}, @@ -1641,7 +1668,8 @@ dynamize_global_secondary_indexes(Value) -> dynamize_sse_specification({enabled, Enabled}) when is_boolean(Enabled) -> [{<<"Enabled">>, Enabled}]. --type create_table_opt() :: {local_secondary_indexes, local_secondary_indexes()} | +-type create_table_opt() :: {billing_mode, billing_mode()} | + {local_secondary_indexes, local_secondary_indexes()} | {global_secondary_indexes, global_secondary_indexes()} | {sse_specification, sse_specification()} | {stream_specification, stream_specification()}. @@ -1649,10 +1677,12 @@ dynamize_sse_specification({enabled, Enabled}) when is_boolean(Enabled) -> -spec create_table_opts(key_schema()) -> opt_table(). create_table_opts(KeySchema) -> - [{local_secondary_indexes, <<"LocalSecondaryIndexes">>, + [{billing_mode, <<"BillingMode">>, fun dynamize_billing_mode/1}, + {local_secondary_indexes, <<"LocalSecondaryIndexes">>, fun(V) -> dynamize_local_secondary_indexes(KeySchema, V) end}, {global_secondary_indexes, <<"GlobalSecondaryIndexes">>, fun dynamize_global_secondary_indexes/1}, + {provisioned_throughput, <<"ProvisionedThroughput">>, fun dynamize_provisioned_throughput/1}, {sse_specification, <<"SSESpecification">>, fun dynamize_sse_specification/1}, {stream_specification, <<"StreamSpecification">>, fun dynamize_stream_specification/1}]. @@ -1665,19 +1695,13 @@ create_table_record() -> -type create_table_return() :: ddb_return(#ddb2_create_table{}, #ddb2_table_description{}). --spec create_table(table_name(), attr_defs(), key_schema(), read_units(), write_units()) +-spec create_table(table_name(), attr_defs(), key_schema(), create_table_opts()) -> create_table_return(). -create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits) -> - create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, [], default_config()). - --spec create_table(table_name(), attr_defs(), key_schema(), read_units(), write_units(), - create_table_opts()) - -> create_table_return(). -create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, Opts) -> - create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, Opts, default_config()). +create_table(Table, AttrDefs, KeySchema, Opts) -> + create_table(Table, AttrDefs, KeySchema, Opts, default_config()). %%------------------------------------------------------------------------------ -%% @doc +%% @doc %% DynamoDB API: %% [http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_CreateTable.html] %% @@ -1686,8 +1710,8 @@ create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, Opts) -> %% Create a table with hash key "ForumName" and range key "Subject" %% with a local secondary index on "LastPostDateTime" %% and a global secondary index on "Subject" as hash key and "LastPostDateTime" -%% as range key, read and write capacity 10, projecting all fields -%% +%% as range key, read and write capacity 10, projecting all fields +%% %% ` %% {ok, Description} = %% erlcloud_ddb2:create_table( @@ -1696,33 +1720,48 @@ create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, Opts) -> %% {<<"Subject">>, s}, %% {<<"LastPostDateTime">>, s}], %% {<<"ForumName">>, <<"Subject">>}, -%% 5, -%% 5, -%% [{local_secondary_indexes, +%% [{provisioned_throughput, {5, 5}}, +%% {local_secondary_indexes, %% [{<<"LastPostIndex">>, <<"LastPostDateTime">>, keys_only}]}, %% {global_secondary_indexes, [ %% {<<"SubjectTimeIndex">>, {<<"Subject">>, <<"LastPostDateTime">>}, all, 10, 10} -%% ]} +%% ]} %% ]), %% ' %% @end %%------------------------------------------------------------------------------ --spec create_table(table_name(), attr_defs(), key_schema(), read_units(), write_units(), - create_table_opts(), aws_config()) +-spec create_table(table_name(), attr_defs(), key_schema(), read_units(), write_units()) + -> create_table_return(); + (table_name(), attr_defs(), key_schema(), create_table_opts(), aws_config()) -> create_table_return(). -create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, Opts, Config) -> +create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits) + when is_integer(ReadUnits), is_integer(WriteUnits) -> + create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, [], default_config()); +create_table(Table, AttrDefs, KeySchema, Opts, Config) -> {AwsOpts, DdbOpts} = opts(create_table_opts(KeySchema), Opts), Return = erlcloud_ddb_impl:request( - Config, - "DynamoDB_20120810.CreateTable", - [{<<"TableName">>, Table}, - {<<"AttributeDefinitions">>, dynamize_attr_defs(AttrDefs)}, - {<<"KeySchema">>, dynamize_key_schema(KeySchema)}, - {<<"ProvisionedThroughput">>, dynamize_provisioned_throughput({ReadUnits, WriteUnits})}] - ++ AwsOpts), - out(Return, fun(Json, UOpts) -> undynamize_record(create_table_record(), Json, UOpts) end, + Config, + "DynamoDB_20120810.CreateTable", + [{<<"TableName">>, Table}, + {<<"AttributeDefinitions">>, dynamize_attr_defs(AttrDefs)}, + {<<"KeySchema">>, dynamize_key_schema(KeySchema)}] + ++ AwsOpts), + out(Return, fun(Json, UOpts) -> undynamize_record(create_table_record(), Json, UOpts) end, DdbOpts, #ddb2_create_table.table_description). +-spec create_table(table_name(), attr_defs(), key_schema(), read_units(), write_units(), + create_table_opts()) + -> create_table_return(). +create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, Opts) -> + create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, Opts, default_config()). + +-spec create_table(table_name(), attr_defs(), key_schema(), read_units(), write_units(), + create_table_opts(), aws_config()) + -> create_table_return(). +create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, Opts0, Config) -> + Opts = [{provisioned_throughput, {ReadUnits, WriteUnits}} | Opts0], + create_table(Table, AttrDefs, KeySchema, Opts, Config). + %%%------------------------------------------------------------------------------ %%% DeleteBackup %%%------------------------------------------------------------------------------ @@ -3397,7 +3436,8 @@ dynamize_global_secondary_index_update(Index) -> dynamize_global_secondary_index_updates(Updates) -> dynamize_maybe_list(fun dynamize_global_secondary_index_update/1, Updates). --type update_table_opt() :: {provisioned_throughput, {read_units(), write_units()}} | +-type update_table_opt() :: {billing_mode, billing_mode()} | + {provisioned_throughput, {read_units(), write_units()}} | {attribute_definitions, attr_defs()} | {global_secondary_index_updates, global_secondary_index_updates()} | {stream_specification, stream_specification()} | @@ -3406,7 +3446,8 @@ dynamize_global_secondary_index_updates(Updates) -> -spec update_table_opts() -> opt_table(). update_table_opts() -> - [{provisioned_throughput, <<"ProvisionedThroughput">>, fun dynamize_provisioned_throughput/1}, + [{billing_mode, <<"BillingMode">>, fun dynamize_billing_mode/1}, + {provisioned_throughput, <<"ProvisionedThroughput">>, fun dynamize_provisioned_throughput/1}, {attribute_definitions, <<"AttributeDefinitions">>, fun dynamize_attr_defs/1}, {global_secondary_index_updates, <<"GlobalSecondaryIndexUpdates">>, fun dynamize_global_secondary_index_updates/1}, diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index 9195d3e3c..af57911e4 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -1225,6 +1225,84 @@ create_table_input_tests(_) -> \"ReadCapacityUnits\": 1, \"WriteCapacityUnits\": 1 } +}" + }), + ?_ddb_test( + {"CreateTable with billing_mode = provisioned", + ?_f(erlcloud_ddb2:create_table( + <<"Thread">>, + {<<"ForumName">>, s}, + <<"ForumName">>, + [{billing_mode, provisioned}, + {provisioned_throughput, {1,1}}] + )), " +{ + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + } + ], + \"BillingMode\": \"PROVISIONED\", + \"TableName\": \"Thread\", + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + } + ], + \"ProvisionedThroughput\": { + \"ReadCapacityUnits\": 1, + \"WriteCapacityUnits\": 1 + } +}" + }), + ?_ddb_test( + {"CreateTable with billing_mode = pay_per_request", + ?_f(erlcloud_ddb2:create_table( + <<"Thread">>, + {<<"ForumName">>, s}, + <<"ForumName">>, + [{billing_mode, pay_per_request}, + {global_secondary_indexes, + [{<<"SubjectIndex">>, {<<"Subject">>, <<"LastPostDateTime">>}, {include, [<<"Author">>]}}]}] + )), " +{ + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + } + ], + \"BillingMode\": \"PAY_PER_REQUEST\", + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"SubjectIndex\", + \"KeySchema\": [ + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"NonKeyAttributes\": [ + \"Author\" + ], + \"ProjectionType\": \"INCLUDE\" + } + } + ], + \"TableName\": \"Thread\", + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + } + ] }" }) ], @@ -1532,6 +1610,10 @@ create_table_output_tests(_) -> \"AttributeType\": \"S\" } ], + \"BillingModeSummary\": { + \"BillingMode\": \"PROVISIONED\", + \"LastUpdateToPayPerRequestDateTime\": 1.36372808007E9 + }, \"CreationDateTime\": 1.36372808007E9, \"GlobalSecondaryIndexes\": [ { @@ -1606,6 +1688,10 @@ create_table_output_tests(_) -> {attribute_definitions = [{<<"ForumName">>, s}, {<<"LastPostDateTime">>, s}, {<<"Subject">>, s}], + billing_mode_summary = + #ddb2_billing_mode_summary{ + billing_mode = provisioned, + last_update_to_pay_per_request_date_time = 1363728080.07}, creation_date_time = 1363728080.07, item_count = 0, key_schema = {<<"ForumName">>, <<"Subject">>}, @@ -1659,6 +1745,10 @@ create_table_output_tests(_) -> \"AttributeType\": \"S\" } ], + \"BillingModeSummary\": { + \"BillingMode\": \"PROVISIONED\", + \"LastUpdateToPayPerRequestDateTime\": 1.36372808007E9 + }, \"GlobalSecondaryIndexes\": [ { \"IndexName\": \"SubjectIndex\", @@ -1737,6 +1827,10 @@ create_table_output_tests(_) -> {attribute_definitions = [{<<"ForumName">>, s}, {<<"LastPostDateTime">>, s}, {<<"Subject">>, s}], + billing_mode_summary = + #ddb2_billing_mode_summary{ + billing_mode = provisioned, + last_update_to_pay_per_request_date_time = 1363728080.07}, creation_date_time = 1363728080.07, item_count = 0, key_schema = {<<"ForumName">>, <<"Subject">>}, @@ -1782,6 +1876,10 @@ create_table_output_tests(_) -> \"AttributeType\": \"S\" } ], + \"BillingModeSummary\": { + \"BillingMode\": \"PROVISIONED\", + \"LastUpdateToPayPerRequestDateTime\": 1.36372808007E9 + }, \"CreationDateTime\": 1.36372808007E9, \"ItemCount\": 0, \"KeySchema\": [ @@ -1802,6 +1900,10 @@ create_table_output_tests(_) -> }", {ok, #ddb2_table_description {attribute_definitions = [{<<"ForumName">>, s}], + billing_mode_summary = + #ddb2_billing_mode_summary{ + billing_mode = provisioned, + last_update_to_pay_per_request_date_time = 1363728080.07}, creation_date_time = 1363728080.07, item_count = 0, key_schema = <<"ForumName">>, @@ -1815,6 +1917,58 @@ create_table_output_tests(_) -> write_capacity_units = 1}, table_name = <<"Thread">>, table_size_bytes = 0, + table_status = creating}}}), + ?_ddb_test( + {"CreateTable response with billing_mode = pay_per_request", " +{ + \"TableDescription\": { + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + } + ], + \"BillingModeSummary\": { + \"BillingMode\": \"PAY_PER_REQUEST\", + \"LastUpdateToPayPerRequestDateTime\": 1.36372808007E9 + }, + \"CreationDateTime\": 1.36372808007E9, + \"ItemCount\": 0, + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + } + ], + \"ProvisionedThroughput\": { + \"NumberOfDecreasesToday\": 0, + \"ReadCapacityUnits\": 0, + \"WriteCapacityUnits\": 0 + }, + \"TableName\": \"Thread\", + \"TableSizeBytes\": 0, + \"TableStatus\": \"CREATING\" + } +}", + {ok, #ddb2_table_description + {attribute_definitions = [{<<"ForumName">>, s}], + billing_mode_summary = + #ddb2_billing_mode_summary{ + billing_mode = pay_per_request, + last_update_to_pay_per_request_date_time = 1363728080.07}, + creation_date_time = 1363728080.07, + item_count = 0, + key_schema = <<"ForumName">>, + local_secondary_indexes = undefined, + provisioned_throughput = + #ddb2_provisioned_throughput_description{ + last_decrease_date_time = undefined, + last_increase_date_time = undefined, + number_of_decreases_today = 0, + read_capacity_units = 0, + write_capacity_units = 0}, + table_name = <<"Thread">>, + table_size_bytes = 0, table_status = creating}}}) ], @@ -5160,6 +5314,15 @@ update_table_input_tests(_) -> } } ] +}" + }), + ?_ddb_test( + {"UpdateTable example request billing_mode = pay_per_request", + ?_f(erlcloud_ddb2:update_table(<<"Thread">>, + [{billing_mode, pay_per_request}])), " +{ + \"TableName\": \"Thread\", + \"BillingMode\": \"PAY_PER_REQUEST\" }" }) ], From 5fd56ce24bf7fb07bd91f4c385f437c0a9f6e69f Mon Sep 17 00:00:00 2001 From: Grigory Starinkin Date: Wed, 12 Dec 2018 07:35:41 +0000 Subject: [PATCH 033/310] allow library users to override configs on demand --- src/erlcloud_aws.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index eeb5f38fd..76f69c320 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -11,7 +11,7 @@ aws_request_form_raw/8, do_aws_request_form_raw/9, param_list/2, default_config/0, auto_config/0, auto_config/1, - default_config_region/2, + default_config_region/2, default_config_override/1, update_config/1,clear_config/1, clear_expired_configs/0, service_config/3, service_host/2, configure/1, format_timestamp/1, From 9ab14ab4aedc386813325174f7858e1f5f304abc Mon Sep 17 00:00:00 2001 From: Roland Karlsson Date: Fri, 14 Dec 2018 11:36:13 +0100 Subject: [PATCH 034/310] Fix erlcloud_s3:put_bucket_lifecycle spec bug --- src/erlcloud_s3.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 1d185d86e..0c067339a 100755 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -465,7 +465,7 @@ get_bucket_lifecycle(BucketName, Config) Error end. --spec put_bucket_lifecycle(string(), binary()) -> ok | {error, Reason::term()} | no_return(). +-spec put_bucket_lifecycle(string(), list() | binary()) -> ok | {error, Reason::term()} | no_return(). put_bucket_lifecycle(BucketName, Policy) -> put_bucket_lifecycle(BucketName, Policy, default_config()). From 8b8c2229cbc5fb3ff50d910a654d840ec22916e2 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Wed, 26 Dec 2018 15:13:10 +0000 Subject: [PATCH 035/310] GetEventSelectors API implemented for CloudTrails --- src/erlcloud_cloudtrail.erl | 10 ++++++++ test/erlcloud_cloudtrail_tests.erl | 40 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/src/erlcloud_cloudtrail.erl b/src/erlcloud_cloudtrail.erl index a1a2194f9..6f4327b7a 100644 --- a/src/erlcloud_cloudtrail.erl +++ b/src/erlcloud_cloudtrail.erl @@ -13,6 +13,7 @@ delete_trail/1, delete_trail/2, describe_trails/0, describe_trails/1, describe_trails/2, describe_trails/3, get_trail_status/1, get_trail_status/2, + get_event_selectors/1, get_event_selectors/2, start_logging/1, start_logging/2, stop_logging/1, stop_logging/2, update_trail/4, update_trail/5, update_trail/6, @@ -126,6 +127,15 @@ get_trail_status(Trail, Config) -> Json = [{<<"Name">>, list_to_binary(Trail)}], ct_request("GetTrailStatus", Json, Config). +-spec get_event_selectors([string()]) -> ct_return(). +get_event_selectors(Trail) -> + get_event_selectors(Trail, default_config()). + +-spec get_event_selectors([string()], aws_config()) -> ct_return(). +get_event_selectors(Trail, Config) -> + Json = [{<<"TrailName">>, list_to_binary(Trail)}], + ct_request("GetEventSelectors", Json, Config). + -spec start_logging([string()] ) -> ct_return(). start_logging(Trail) -> start_logging(Trail, default_config()). diff --git a/test/erlcloud_cloudtrail_tests.erl b/test/erlcloud_cloudtrail_tests.erl index 55e27b565..bf5284377 100644 --- a/test/erlcloud_cloudtrail_tests.erl +++ b/test/erlcloud_cloudtrail_tests.erl @@ -33,6 +33,8 @@ operation_test_() -> fun delete_trail_output_tests/1, fun describe_trails_input_tests/1, fun describe_trails_output_tests/1, + fun get_event_selectors_input_tests/1, + fun get_event_selectors_output_tests/1, fun get_trail_status_input_tests/1, fun get_trail_status_output_tests/1, fun start_logging_input_tests/1, @@ -305,6 +307,44 @@ describe_trails_output_tests(_) -> ], output_tests(?_f(erlcloud_cloudtrail:describe_trails(["test"], erlcloud_aws:default_config())), Tests). +%% GetEventSelectors test based on the API examples: +%% https://docs.aws.amazon.com/awscloudtrail/latest/APIReference/API_GetEventSelectors.html +get_event_selectors_input_tests(_) -> + Tests = + [?_cloudtrail_test( + {"GetEventSelectors example request", + ?_f(erlcloud_cloudtrail:get_event_selectors("test", erlcloud_aws:default_config())), " +{ + + \"TrailName\": \"test\" +}" + }) + ], + Response = "{}", + + input_tests(Response, Tests). + +get_event_selectors_output_tests(_) -> + Response = <<"{\"EventSelectors\": [ + { + \"DataResources\": [ + { + \"Type\": \"AWS::S3::Object\", + \"Values\": [ \"arn:aws:s3:::test-bucket/\" ] + } ], + \"IncludeManagementEvents\": true, + \"ReadWriteType\": \"All\" + } ], + \"TrailARN\": \"awn:aws:cloudtrail:us-west-1:1234567890:trail/test-trail\" + }">>, + + Tests = + [?_cloudtrail_test( + {"GetEventSelectors example response", Response, + {ok, jsx:decode(Response)}}) + ], + output_tests(?_f(erlcloud_cloudtrail:get_event_selectors("test", erlcloud_aws:default_config())), Tests). + %% GetTrailStatus test based on the API examples: %% http://docs.aws.amazon.com/awscloudtrail/latest/APIReference/API_GetTrailStatus.html get_trail_status_input_tests(_) -> From 165885bffea8b0b63a51f180f11f2b7a36d65896 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Mon, 14 Jan 2019 15:32:39 -0600 Subject: [PATCH 036/310] Enable passing a function for put_records encoding --- src/erlcloud_kinesis.erl | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index d7e6ebcf3..5f1bb77ec 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -15,7 +15,7 @@ get_shard_iterator/3, get_shard_iterator/4, get_shard_iterator/5, get_records/1, get_records/2, get_records/3, get_records/4, put_record/3, put_record/4, put_record/5, put_record/6, put_record/7, - put_records/2, put_records/3, + put_records/2, put_records/3, put_records/4, merge_shards/3, merge_shards/4, split_shards/3, split_shards/4, add_tags_to_stream/2, add_tags_to_stream/3, @@ -608,28 +608,42 @@ put_record(StreamName, PartitionKey, Data, ExplicitHashKey, Ordering, Options, C put_records(StreamName, Items) -> put_records(StreamName, Items, default_config()). --spec put_records(binary(), put_records_items(), Config) -> erlcloud_kinesis_impl:json_return() when - Config :: aws_config(). +-spec put_records(binary(), put_records_items(), function() | aws_config()) -> erlcloud_kinesis_impl:json_return(). +%% @doc This function takes a fun which is responsible for encoding each of the +%% data items in the Items list. +put_records(StreamName, Items, EncodingFun) when is_function(EncodingFun) -> + Operation = put_records_operation(), + Json = prepare_put_records_data(StreamName, Items, EncodingFun), + erlcloud_kinesis_impl:request(default_config(), Operation, Json); put_records(StreamName, Items, Config) -> Operation = put_records_operation(), - Json = prepare_put_records_data(StreamName, Items), + Json = prepare_put_records_data(StreamName, Items, fun default_put_encoding/1), erlcloud_kinesis_impl:request(Config, Operation, Json). +-spec put_records(binary(), put_records_items(), + function(), Config :: aws_config()) -> erlcloud_kinesis_impl:json_return(). +put_records(StreamName, Items, EncodingFun, Config) -> + Operation = put_records_operation(), + Json = prepare_put_records_data(StreamName, Items, EncodingFun), + erlcloud_kinesis_impl:request(Config, Operation, Json). + +default_put_encoding(D) -> base64:encode(D). + put_records_operation() -> "Kinesis_20131202.PutRecords". -prepare_put_records_data(StreamName, Items) -> - Records = [prepare_put_records_item(X) || X <- Items], +prepare_put_records_data(StreamName, Items, Fun) -> + Records = [prepare_put_records_item(X, Fun) || X <- Items], [{<<"StreamName">>, StreamName}, {<<"Records">>, Records}]. -prepare_put_records_item({Data, PartitionKey}) -> +prepare_put_records_item({Data, PartitionKey}, Fun) -> [{<<"PartitionKey">>, PartitionKey}, - {<<"Data">>, base64:encode(Data)}]; -prepare_put_records_item({Data, ExplicitHashKey, PartitionKey}) -> + {<<"Data">>, Fun(Data)}]; +prepare_put_records_item({Data, ExplicitHashKey, PartitionKey}, Fun) -> [{<<"PartitionKey">>, PartitionKey}, {<<"ExplicitHashKey">>, ExplicitHashKey}, - {<<"Data">>, base64:encode(Data)}]. + {<<"Data">>, Fun(Data)}]. %%------------------------------------------------------------------------------ %% @doc From ffb62425f5ddf203e88239edacf717b5fb1e1448 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Tue, 15 Jan 2019 13:28:08 -0600 Subject: [PATCH 037/310] Respond to review comments --- src/erlcloud_kinesis.erl | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index 5f1bb77ec..c6c83199b 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -609,21 +609,17 @@ put_records(StreamName, Items) -> put_records(StreamName, Items, default_config()). -spec put_records(binary(), put_records_items(), function() | aws_config()) -> erlcloud_kinesis_impl:json_return(). -%% @doc This function takes a fun which is responsible for encoding each of the -%% data items in the Items list. -put_records(StreamName, Items, EncodingFun) when is_function(EncodingFun) -> - Operation = put_records_operation(), - Json = prepare_put_records_data(StreamName, Items, EncodingFun), - erlcloud_kinesis_impl:request(default_config(), Operation, Json); + +put_records(StreamName, Items, EncodingFun) when is_function(EncodingFun, 1) -> + put_records(StreamName, Items, EncodingFun, default_config()); put_records(StreamName, Items, Config) -> - Operation = put_records_operation(), - Json = prepare_put_records_data(StreamName, Items, fun default_put_encoding/1), - erlcloud_kinesis_impl:request(Config, Operation, Json). + put_records(StreamName, Items, fun default_put_encoding/1, Config). -spec put_records(binary(), put_records_items(), function(), Config :: aws_config()) -> erlcloud_kinesis_impl:json_return(). -put_records(StreamName, Items, EncodingFun, Config) -> + +put_records(StreamName, Items, EncodingFun, Config) when is_function(EncodingFun, 1) -> Operation = put_records_operation(), Json = prepare_put_records_data(StreamName, Items, EncodingFun), erlcloud_kinesis_impl:request(Config, Operation, Json). From 948943012b7e2af83889cf81e93faef61d7dc001 Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Tue, 19 Feb 2019 09:41:16 -0600 Subject: [PATCH 038/310] erlcloud_ddb2: fix issue with serializing GSI creation when billing_mode = PPR When billing_mode = pay_per_request, updating the global secondary indexes requires a 3-tuple of `{IndexName, KeySpecification, Projection}`, but this has the same signature as a provisioned throughput update for an index, `{IndexName, RU, WU}`. To address this, we need to adjust the serialization to only internpret GSI updates with RU/WU when they are passed as integers. Otherwise the update should get passed into the new GSI creation serialization path. --- src/erlcloud_ddb2.erl | 3 ++- test/erlcloud_ddb2_tests.erl | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 5b7f00473..0d5d90d28 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -3420,7 +3420,8 @@ update_global_table(GlobalTableName, ReplicaUpdates, Opts, Config) -> -type global_secondary_index_updates() :: maybe_list(global_secondary_index_update()). -spec dynamize_global_secondary_index_update(global_secondary_index_update()) -> jsx:json_term(). -dynamize_global_secondary_index_update({IndexName, ReadUnits, WriteUnits}) -> +dynamize_global_secondary_index_update({IndexName, ReadUnits, WriteUnits}) + when is_integer(ReadUnits), is_integer(WriteUnits) -> [{<<"Update">>, [ {<<"IndexName">>, IndexName}, {<<"ProvisionedThroughput">>, dynamize_provisioned_throughput({ReadUnits, WriteUnits})} diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index af57911e4..a22352aeb 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -5314,6 +5314,38 @@ update_table_input_tests(_) -> } } ] +}" + }), + ?_ddb_test( + {"UpdateTable example request with Create GSI (pay per request)", + ?_f(erlcloud_ddb2:update_table(<<"Thread">>, + [{attribute_definitions, [{<<"HashKey1">>, s}]}, + {global_secondary_index_updates, [ + {<<"Index1">>, <<"HashKey1">>, all}]}])), " +{ + \"TableName\": \"Thread\", + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"HashKey1\", + \"AttributeType\": \"S\" + } + ], + \"GlobalSecondaryIndexUpdates\": [ + { + \"Create\": { + \"IndexName\": \"Index1\", + \"KeySchema\": [ + { + \"AttributeName\": \"HashKey1\", + \"KeyType\": \"HASH\" + } + ], + \"Projection\": { + \"ProjectionType\": \"ALL\" + } + } + } + ] }" }), ?_ddb_test( From d4a4c263feec6f2381158e61301552e4bc751b94 Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Thu, 28 Feb 2019 12:24:25 +0000 Subject: [PATCH 039/310] initial guardduty support --- include/erlcloud_aws.hrl | 3 + src/erlcloud_aws.erl | 3 + src/erlcloud_guardduty.erl | 144 +++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 src/erlcloud_guardduty.erl diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 61663f567..0bf8dc2bd 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -107,6 +107,9 @@ mms_scheme="https://"::string(), mms_host="metering.marketplace.us-east-1.amazonaws.com"::string(), mms_port=443::non_neg_integer(), + guardduty_scheme="https://"::string(), + guardduty_host="guardduty.us-east-1.amazonaws.com"::string(), + guardduty_port=443::non_neg_integer(), cur_scheme="https://"::string(), cur_host="cur.us-east-1.amazonaws.com"::string(), cur_port=443::non_neg_integer(), diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 76f69c320..73a114ad3 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -760,6 +760,9 @@ service_config(<<"cloudwatch_logs">>, Region, Config)-> Host = service_host(<<"logs">>, Region), Config#aws_config{cloudwatch_logs_host = Host}; service_config( <<"waf">>, _Region, Config ) -> Config; +service_config( <<"guardduty">> = Service, Region, Config ) -> + Host = service_host( Service, Region ), + Config#aws_config{guardduty_host = Host}; service_config( <<"cur">>, Region, Config ) -> Host = service_host(<<"cur">>, Region), Config#aws_config{cur_host = Host}. diff --git a/src/erlcloud_guardduty.erl b/src/erlcloud_guardduty.erl new file mode 100644 index 000000000..ec8510b42 --- /dev/null +++ b/src/erlcloud_guardduty.erl @@ -0,0 +1,144 @@ +-module(erlcloud_guardduty). + +-include_lib("erlcloud/include/erlcloud.hrl"). +-include_lib("erlcloud/include/erlcloud_aws.hrl"). + +%%------------------------------------------------------------------------------ +%% Library initialization. +%%------------------------------------------------------------------------------ +-export([configure/2, configure/3, new/2, new/3]). + +%%------------------------------------------------------------------------------ +%% GuardDuty API Functions +%%------------------------------------------------------------------------------ +-export([ + get_detector/1, get_detector/2, + list_detectors/0, list_detectors/1, list_detectors/2, list_detectors/3 +]). + + +-type gd_return() :: {ok, proplist()} | {error, term()}. + +-spec new(AccessKeyID ::string(), SecretAccessKey :: string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey) -> + #aws_config{access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey}. + +-spec new(AccessKeyID :: string(), SecretAccessKey :: string(), Host :: string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey, Host) -> + #aws_config{access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey, + guardduty_host=Host}. + +-spec configure(AccessKeyID :: string(), SecretAccessKey :: string()) -> ok. +configure(AccessKeyID, SecretAccessKey) -> + put(aws_config, new(AccessKeyID, SecretAccessKey)), + ok. + +-spec configure(AccessKeyID :: string(), SecretAccessKey :: string(), Host :: string()) -> ok. +configure(AccessKeyID, SecretAccessKey, Host) -> + put(aws_config, new(AccessKeyID, SecretAccessKey, Host)), + ok. + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +%%------------------------------------------------------------------------------ +%% @doc +%% GuardDuty API: +%% [https://docs.aws.amazon.com/guardduty/latest/ug/get-detector.html] +%% +%%------------------------------------------------------------------------------ + +-spec get_detector(DetectorId :: binary()) -> gd_return(). +get_detector(DetectorId) -> + get_detector(DetectorId, default_config()). + +-spec get_detector(DetectorId :: binary(), + Config :: aws_config()) -> gd_return(). +get_detector(DetectorId, Config) -> + Path = base_path() ++ "detector/" ++ binary_to_list(DetectorId), + guardduty_request(Config, get, Path, undefined). + + +%%------------------------------------------------------------------------------ +%% @doc +%% GuardDuty API: +%% [https://docs.aws.amazon.com/guardduty/latest/ug/list-detectors.html] +%% +%%------------------------------------------------------------------------------ + +-spec list_detectors() -> gd_return(). +list_detectors() -> list_detectors(default_config()). + +-spec list_detectors(Config :: aws_config()) -> gd_return(). +list_detectors(Config) -> + list_detectors(undefined, undefined, Config). + +-spec list_detectors(Marker :: binary(), + MaxItems :: integer()) -> gd_return(). +list_detectors(Marker, MaxItems) -> + list_detectors(Marker, MaxItems, default_config()). + +-spec list_detectors(Marker :: binary(), + MaxItems :: integer(), + Config :: aws_config()) -> gd_return(). +list_detectors(Marker, MaxItems, Config) -> + Path = base_path() ++ "detector", + QParams = filter_undef([{"Marker", Marker}, + {"MaxItems", MaxItems}]), + guardduty_request(Config, get, Path, undefined, QParams). + + +%%%------------------------------------------------------------------------------ +%%% Internal Functions +%%%------------------------------------------------------------------------------ + +filter_undef(List) -> + lists:filter(fun({_Name, Value}) -> Value =/= undefined end, List). + +base_path() -> + "/". + +guardduty_request(Config, Method, Path, Body) -> + guardduty_request(Config, Method, Path, Body, []). + +guardduty_request(Config, Method, Path, Body, QParam) -> + case erlcloud_aws:update_config(Config) of + {ok, Config1} -> + guardduty_request_no_update(Config1, Method, Path, Body, QParam); + {error, Reason} -> + {error, Reason} + end. + +guardduty_request_no_update(Config, Method, Path, Body, QParam) -> + Form = case encode_body(Body) of + <<>> -> erlcloud_http:make_query_string(QParam); + Value -> Value + end, + Headers = headers(Method, Path, Config, encode_body(Body), QParam), + case erlcloud_aws:aws_request_form_raw( + Method, Config#aws_config.guardduty_scheme, Config#aws_config.guardduty_host, + Config#aws_config.guardduty_port, Path, Form, Headers, Config) of + {ok, Data} -> + {ok, jsx:decode(Data)}; + E -> + E + end. + +encode_body(undefined) -> + <<>>; +encode_body([]) -> + <<"{}">>; +encode_body(Body) -> + jsx:encode(Body). + +headers(Method, Uri, Config, Body, QParam) -> + Headers = [{"host", Config#aws_config.guardduty_host}, + {"content-type", "application/json"}], + Region = erlcloud_aws:aws_region_from_host(Config#aws_config.guardduty_host), + erlcloud_aws:sign_v4(Method, Uri, Config, + Headers, Body, Region, "guardduty", QParam). + +default_config() -> erlcloud_aws:default_config(). From a648ffee14e608b29e0139b966c59f7bd97252a3 Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Thu, 28 Feb 2019 12:42:57 +0000 Subject: [PATCH 040/310] list mfa devices, simulate policies with context entries --- include/erlcloud_iam.hrl | 3 + src/erlcloud_iam.erl | 105 ++++++++++++++++++++++++++++- test/erlcloud_iam_tests.erl | 131 +++++++++++++++++++++++++++++++++++- 3 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 include/erlcloud_iam.hrl diff --git a/include/erlcloud_iam.hrl b/include/erlcloud_iam.hrl new file mode 100644 index 000000000..ad852fc1f --- /dev/null +++ b/include/erlcloud_iam.hrl @@ -0,0 +1,3 @@ +-type context_key() :: context_key_name | context_key_type | context_key_values. +-type context_entry() :: [{context_key(), string() | list()}]. +-type context_entries() :: list(context_entry()). diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index 7df62a677..cd0871743 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -4,6 +4,7 @@ -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). +-include("erlcloud_iam.hrl"). -include_lib("xmerl/include/xmerl.hrl"). %% Library initialization. @@ -57,7 +58,10 @@ generate_credential_report/0, generate_credential_report/1, get_credential_report/0, get_credential_report/1, simulate_principal_policy/2, simulate_principal_policy/3, - simulate_custom_policy/2, simulate_custom_policy/3 + simulate_custom_policy/2, simulate_custom_policy/3, simulate_custom_policy/4, + list_virtual_mfa_devices/0, list_virtual_mfa_devices/1, list_virtual_mfa_devices/2, + list_virtual_mfa_devices/3, list_virtual_mfa_devices/4, + list_virtual_mfa_devices_all/0, list_virtual_mfa_devices_all/1, list_virtual_mfa_devices_all/2 ]). -export([get_uri/2]). @@ -737,18 +741,101 @@ simulate_principal_policy(PolicySourceArn, ActionNames, #aws_config{} = Config) simulate_custom_policy(ActionNames, PolicyInputList) -> simulate_custom_policy(ActionNames, PolicyInputList, default_config()). +-spec simulate_custom_policy(list(), list(), aws_config() | context_entries()) -> {ok, proplist()} | {error, any()}. simulate_custom_policy(ActionNames, PolicyInputList, #aws_config{} = Config) when is_list(ActionNames), is_list(PolicyInputList) -> ItemPath = "/SimulateCustomPolicyResponse/SimulateCustomPolicyResult/" "EvaluationResults/member", Params = erlcloud_util:encode_list("ActionNames", ActionNames) ++ erlcloud_util:encode_list("PolicyInputList", PolicyInputList), + iam_query_all(Config, "SimulateCustomPolicy", Params, + ItemPath, data_type("EvaluationResult")); +simulate_custom_policy(ActionNames, PolicyInputList, ContextEntries) -> + simulate_custom_policy(ActionNames, PolicyInputList, ContextEntries, default_config()). + +simulate_custom_policy(ActionNames, PolicyInputList, ContextEntries, #aws_config{} = Config) + when is_list(ActionNames), is_list(PolicyInputList) -> + ItemPath = "/SimulateCustomPolicyResponse/SimulateCustomPolicyResult/" + "EvaluationResults/member", + + Params = erlcloud_util:encode_list("ActionNames", ActionNames) ++ + erlcloud_util:encode_list("PolicyInputList", PolicyInputList) ++ + encode_context_entries(ContextEntries), iam_query_all(Config, "SimulateCustomPolicy", Params, ItemPath, data_type("EvaluationResult")). +-spec list_virtual_mfa_devices() -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +list_virtual_mfa_devices() -> + list_virtual_mfa_devices(default_config()). + +-spec list_virtual_mfa_devices(string() | aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +list_virtual_mfa_devices(#aws_config{} = Config) -> + list_virtual_mfa_devices(undefined, undefined, undefined, Config); +list_virtual_mfa_devices(AssignmentStatus) -> + list_virtual_mfa_devices(AssignmentStatus, undefined, undefined, default_config()). + +-spec list_virtual_mfa_devices(string(), string() | aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +list_virtual_mfa_devices(AssignmentStatus, #aws_config{} = Config) -> + list_virtual_mfa_devices(AssignmentStatus, undefined, undefined, Config); +list_virtual_mfa_devices(AssignmentStatus, Marker) -> + list_virtual_mfa_devices(AssignmentStatus, Marker, undefined, default_config()). + +-spec list_virtual_mfa_devices(string(), string(), string()| aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +list_virtual_mfa_devices(AssignmentStatus, Marker, #aws_config{} = Config) -> + list_virtual_mfa_devices(AssignmentStatus, Marker, undefined, Config); +list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems) -> + list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems, default_config()). + +-spec list_virtual_mfa_devices(string(), string(), string(), aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems, #aws_config{} = Config) -> + Params = make_list_virtual_mfa_devices_params(AssignmentStatus, Marker, MaxItems), + ItemPath = "/ListVirtualMFADevicesResponse/ListVirtualMFADevicesResult/VirtualMFADevices/member", + iam_query(Config, "ListVirtualMFADevices", Params, ItemPath, data_type("VirtualMFADeviceMetadata")). + + +-spec list_virtual_mfa_devices_all() -> {ok, proplist()} | {error, any()}. +list_virtual_mfa_devices_all() -> + list_virtual_mfa_devices_all(default_config()). + +-spec list_virtual_mfa_devices_all(string() | aws_config()) -> {ok, proplist()} | {error, any()}. +list_virtual_mfa_devices_all(#aws_config{} = Config) -> + list_virtual_mfa_devices_all(undefined, Config); +list_virtual_mfa_devices_all(AssignmentStatus) -> + list_virtual_mfa_devices_all(AssignmentStatus, default_config()). + +-spec list_virtual_mfa_devices_all(string(), aws_config()) -> {ok, proplist()} | {error, any()}. +list_virtual_mfa_devices_all(AssignmentStatus, #aws_config{} = Config) -> + Params = make_list_virtual_mfa_devices_params(AssignmentStatus, undefined, undefined), + ItemPath = "/ListVirtualMFADevicesResponse/ListVirtualMFADevicesResult/VirtualMFADevices/member", + iam_query_all(Config, "ListVirtualMFADevices", Params, ItemPath, data_type("VirtualMFADeviceMetadata")). + + % % Utils % + +encode_context_entries(ContextEntries) -> + ParsedContextEntriesValues = [ [{"ContextKeyName", ContextKeyName}, + {"ContextKeyType", ContextKeyType}, + {"ContextKeyValues", erlcloud_util:encode_list("", ContextKeyValues)}] || + [{context_key_name, ContextKeyName}, + {context_key_type, ContextKeyType}, + {context_key_values, ContextKeyValues}] <- + ContextEntries], + EncodedContextEntries = erlcloud_aws:param_list(ParsedContextEntriesValues, "ContextEntries.member"), + lists:flatten([flatten_encoded_context_value(Key, Value) || {Key, Value} <- EncodedContextEntries]). + +flatten_encoded_context_value(Key, Value) -> + flatten_encoded_context_value(Key, Value, []). + +flatten_encoded_context_value(_, [], Acc) -> + Acc; +flatten_encoded_context_value(Key, [{SubKey, Val} | Values], Acc) -> + Acc2 = [{Key++SubKey, Val}] ++ Acc, + flatten_encoded_context_value(Key, Values, Acc2); +flatten_encoded_context_value(Key, Val, _) -> + [{Key, Val}]. + iam_query(Config, Action, Params) -> iam_query(Config, Action, Params, ?API_VERSION). @@ -833,6 +920,10 @@ extract_account_summary(Item) -> end, [], Entries). +data_type("VirtualMFADeviceMetadata") -> + [{"SerialNumber", serial_number, "String"}, + {"EnableDate", enable_date, "DateTime"}, + {"User", user, data_type("UserDetail")}]; data_type("AccountAuthorizationDetails") -> [{"UserDetailList/member", users, data_type("UserDetail")}, {"GroupDetailList/member", groups, data_type("GroupDetail")}, @@ -965,3 +1056,15 @@ data_fun("Uri") -> {?MODULE, get_uri}. get_uri(Key, Item) -> http_uri:decode(erlcloud_xml:get_text(Key, Item)). + +make_list_virtual_mfa_devices_params(undefined, undefined, undefined) -> + []; +make_list_virtual_mfa_devices_params(AssignmentStatus, Marker, MaxItems) -> + make_list_virtual_mfa_devices_param(AssignmentStatus, "AssignmentStatus") ++ + make_list_virtual_mfa_devices_param(Marker ,"Marker") ++ + make_list_virtual_mfa_devices_param(MaxItems, "MaxItems"). + +make_list_virtual_mfa_devices_param(undefined, _) -> + []; +make_list_virtual_mfa_devices_param(Param, ParamString) -> + [{ParamString, Param}]. diff --git a/test/erlcloud_iam_tests.erl b/test/erlcloud_iam_tests.erl index 066645539..cc5fc005f 100644 --- a/test/erlcloud_iam_tests.erl +++ b/test/erlcloud_iam_tests.erl @@ -99,7 +99,9 @@ iam_api_test_() -> fun simulate_custom_policy_input_test/1, fun simulate_custom_policy_output_test/1, fun simulate_principal_policy_input_test/1, - fun simulate_principal_policy_output_test/1 + fun simulate_principal_policy_output_test/1, + fun list_virtual_mfa_devices_input_test/1, + fun list_virtual_mfa_devices_output_test/1 ]}. start() -> @@ -415,6 +417,102 @@ get_account_summary_output_tests(_) -> ], output_tests(?_f(erlcloud_iam:get_account_summary()), Tests). +-define(LIST_VIRTUAL_MFA_DEVICES_RESP, + " + + false + + + arn:aws:iam::123456789012:mfa/MFAdeviceName + + + arn:aws:iam::123456789012:mfa/RootMFAdeviceName + 2011-10-20T20:49:03Z + + 123456789012 + arn:aws:iam::123456789012:root + 2009-10-13T22:00:36Z + + + + arn:aws:iam:::mfa/ExampleUserMFAdeviceName + 2011-10-31T20:45:02Z + + AIDEXAMPLE4EXAMPLEXYZ + / + ExampleUser + arn:aws:iam::111122223333:user/ExampleUser + 2011-07-01T17:23:07Z + + + + + + b61ce1b1-0401-11e1-b2f8-2dEXAMPLEbfc + + "). + +list_virtual_mfa_devices_input_test(_) -> + Tests = + [?_iam_test( + {"Test returning account registered MFA devices.", + ?_f(erlcloud_iam:list_virtual_mfa_devices()), + [ + {"Action", "ListVirtualMFADevices"} + ]}) + ], + input_tests(?LIST_VIRTUAL_MFA_DEVICES_RESP, Tests). + +list_virtual_mfa_devices_output_test(_) -> + Tests = [?_iam_test( + {"This returns the registered MFA devices", + ?LIST_VIRTUAL_MFA_DEVICES_RESP, + {ok,[ + [ + {user,[]}, + {enable_date,undefined}, + {serial_number,"arn:aws:iam::123456789012:mfa/MFAdeviceName"} + ], + [ + {user, + [ + [ + {arn,"arn:aws:iam::123456789012:root"}, + {create_date,{{2009,10,13},{22,0,36}}}, + {group_list,[]}, + {path,[]}, + {user_id,"123456789012"}, + {user_name,[]}, + {user_policy_list,[]} + ] + ] + }, + {enable_date,{{2011,10,20},{20,49,3}}}, + {serial_number,"arn:aws:iam::123456789012:mfa/RootMFAdeviceName"} + ], + [ + {user, + [ + [ + {arn,"arn:aws:iam::111122223333:user/ExampleUser"}, + {create_date,{{2011,7,1},{17,23,7}}}, + {group_list,[]}, + {path,"/"}, + {user_id,"AIDEXAMPLE4EXAMPLEXYZ"}, + {user_name,"ExampleUser"}, + {user_policy_list,[]} + ] + ] + }, + {enable_date,{{2011,10,31},{20,45,2}}}, + {serial_number,"arn:aws:iam:::mfa/ExampleUserMFAdeviceName"} + ] + ]} + }) + ], + output_tests(?_f(erlcloud_iam:list_virtual_mfa_devices()), Tests). + + -define(GET_ACCOUNT_PASSWORD_POLICY_RESP, " @@ -2696,6 +2794,9 @@ simulate_custom_policy_input_test(_) -> PolicyDoc1 = "policy_doc1", PolicyDoc2 = "policy_doc2", Action = "s3:ListBucket", + ContextEntries = [[{context_key_name,"aws:MultiFactorAuthPresent"}, + {context_key_type,"boolean"}, + {context_key_values,[true]}]], Tests = [?_iam_test( {"SimulateCustomPolicy input", @@ -2708,15 +2809,38 @@ simulate_custom_policy_input_test(_) -> {"PolicyInputList.member.1", PolicyDoc1}, {"PolicyInputList.member.2", PolicyDoc2}, {"MaxItems", "1000"} - ]}) + ]}), + ?_iam_test( + {"SimulateCustomPolicy2 input", + ?_f(erlcloud_iam:simulate_custom_policy([Action], + [PolicyDoc1], + ContextEntries)), + [{"Action","SimulateCustomPolicy"}, + {"ActionNames.member.1", http_uri:encode(Action)}, + {"PolicyInputList.member.1","policy_doc1"}, + {"ContextEntries.member.1.ContextKeyName",http_uri:encode("aws:MultiFactorAuthPresent")}, + {"ContextEntries.member.1.ContextKeyType","boolean"}, + {"ContextEntries.member.1.ContextKeyValues.member.1","true"}, + {"MaxItems","1000"}]}) ], input_tests(?SIMULATE_CUSTOM_POLICY_RESP, Tests). simulate_custom_policy_output_test(_) -> + ContextEntries = [[{context_key_name,"aws:MultiFactorAuthPresent"}, + {context_key_type,"boolean"}, + {context_key_values,[true]}]], Tests = [?_iam_test( {"SimulateCustomPolicy output", ?SIMULATE_CUSTOM_POLICY_RESP, + {ok, [[{eval_action_name, "s3:ListBucket"}, + {eval_decision, "allowed"}, + {eval_resource_name, "arn:aws:s3:::teambucket"}, + {matched_statements_list, + [[{source_policy_id, "PolicyInputList.1"}]]}]]}}), + ?_iam_test( + {"SimulateCustomPolicy2 output", + ?SIMULATE_CUSTOM_POLICY_RESP, {ok, [[{eval_action_name, "s3:ListBucket"}, {eval_decision, "allowed"}, {eval_resource_name, "arn:aws:s3:::teambucket"}, @@ -2725,7 +2849,8 @@ simulate_custom_policy_output_test(_) -> ], output_tests(?_f(erlcloud_iam:simulate_custom_policy(["s3:ListBucket"], ["policy_doc1", - "policy_doc2"])), + "policy_doc2"], + ContextEntries)), Tests). simulate_principal_policy_input_test(_) -> From 64e7583b8118aef3a80838f999ae8a8a94464a48 Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Thu, 28 Feb 2019 12:57:38 +0000 Subject: [PATCH 041/310] describe_alarms_for_metric function --- include/erlcloud_mon.hrl | 9 +- src/erlcloud_mon.erl | 183 +++++++++++++++++++++++++++++++++++- test/erlcloud_mon_tests.erl | 176 ++++++++++++++++++++++++++++++++++ 3 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 test/erlcloud_mon_tests.erl diff --git a/include/erlcloud_mon.hrl b/include/erlcloud_mon.hrl index d85192647..e8c7e7276 100644 --- a/include/erlcloud_mon.hrl +++ b/include/erlcloud_mon.hrl @@ -30,6 +30,13 @@ %%------------------------------------------------------------------------------ -type unit() :: string(). +%%------------------------------------------------------------------------------ +%% @doc The statistic for the metric, other than percentiles. +%% Valid Values: SampleCount | Average | Sum | Minimum | Maximum +%% @end +%%------------------------------------------------------------------------------ +-type statistic() :: string(). + %%------------------------------------------------------------------------------ %% @doc Dimension %% The Dimension data type further expands on the identity of a metric using a Name, Value pair. @@ -73,4 +80,4 @@ }). -type metric_datum() :: #metric_datum{}. --endif. \ No newline at end of file +-endif. diff --git a/src/erlcloud_mon.erl b/src/erlcloud_mon.erl index 2aa8f5b9c..9939613d9 100644 --- a/src/erlcloud_mon.erl +++ b/src/erlcloud_mon.erl @@ -16,6 +16,9 @@ -export([ list_metrics/5, list_metrics/4, + describe_alarms_for_metric/2, + describe_alarms_for_metric/7, + describe_alarms_for_metric/8, put_metric_data/3, put_metric_data/2, put_metric_data/6, put_metric_data/5, get_metric_statistics/4, get_metric_statistics/9, get_metric_statistics/8, @@ -30,7 +33,8 @@ -include("erlcloud_mon.hrl"). -include_lib("xmerl/include/xmerl.hrl"). --import(erlcloud_xml, [get_text/2, get_time/2]). +-import(erlcloud_xml, [get_text/2, get_time/2, get_bool/2, get_integer/2, + get_float/2, get_text/1]). -define(XMLNS_MON, "http://monitoring.amazonaws.com/doc/2010-08-01/"). -define(API_VERSION, "2010-08-01"). @@ -116,6 +120,183 @@ extract_dimension(Node) -> {value, get_text("Value", Node)} ]. +%%------------------------------------------------------------------------------ +%% @doc CloudWatch API - DescribeAlarmsForMetric +%% [https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_DescribeAlarmsForMetric.html] +%% +%% USAGE: +%% +%% erlcloud_mon:describe_alarms_for_metric("AWS/EC2", +%% "NetworkIn"). +%% [[{metric_name,"NetworkIn"}, +%% {namespace,"AWS/EC2"}, +%% {dimensions,[]}, +%% {actions_enabled,true}, +%% {alarm_actions,[{arn,"arn:aws:sns:us-east-1:123456797777:rgallego_cloudtrail_sns_topic"}]}, +%% {alarm_arn,"arn:aws:cloudwatch:us-east-1:123456797777:alarm:rgallego_unauthorized_alarm"}, +%% {alarm_configuration_updated_timestamp,{{2018,2,7}, +%% {17,38,24}}}, +%% {alarm_description,[]}, +%% {alarm_name,"rgallego_unauthorized_alarm"}, +%% {comparison_operator,"GreaterThanOrEqualToThreshold"}, +%% {evaluate_low_sample_count_percentile,[]}, +%% {evaluation_periods,1}, +%% {extended_statistic,[]}, +%% {insufficient_data_actions,[]}, +%% {ok_actions,[]}, +%% {period,300}, +%% {state_reason,"Threshold Crossed: 1 datapoint [2.0 (07/02/18 17:33:00)] was greater than or equal to the threshold (1.0)."}, +%% {state_reason_data,"{\"version\":\"1.0\",\"queryDate\":\"2018-02-07T17:38:24.953+0000\",\"startDate\":\"2018-02-07T17:33:00.000+0000\",\"statistic\":\"Sum\",\"period\":300,\"recentDatapoints\":[2.0],\"threshold\":1.0}"}, +%% {state_updated_timestamp,{{2018,2,7},{17,38,24}}}, +%% {state_value,"ALARM"}, +%% {statistic,"Sum"}, +%% {threshold,1.0}, +%% {treat_missing_data,[]}, +%% {unit,[]}]] +%% +%% @end +%%------------------------------------------------------------------------------ +-spec describe_alarms_for_metric( + Namespace ::string(), + MetricName ::string() + ) -> term(). + +describe_alarms_for_metric( + Namespace, + MetricName + ) -> + describe_alarms_for_metric(Namespace, MetricName, [], + "", undefined, "", + "", default_config()). + +%%------------------------------------------------------------------------------ +%% @doc CloudWatch API - DescribeAlarmsForMetric +%% [https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_DescribeAlarmsForMetric.html] +%% +%% USAGE: +%% +%% erlcloud_mon:describe_alarms_for_metric("AWS/EC2", +%% "NetworkIn", +%% [{"InstanceType","m1.large"}], +%% "p95", +%% 17, +%% "", +%% "Seconds"). +%% See describe_alarms_for_metric/2 +%% +%% @end +%%------------------------------------------------------------------------------ +-spec describe_alarms_for_metric( + Namespace ::string(), + MetricName ::string(), + DimensionFilter ::[{string(),string()}], + ExtendedStatistic ::string(), + Period ::pos_integer(), + Statistic ::statistic(), + Unit ::unit() + ) -> term(). + +describe_alarms_for_metric( + Namespace, + MetricName, + DimensionFilter, + ExtendedStatistic, + Period, + Statistic, + Unit + ) -> + describe_alarms_for_metric(Namespace, MetricName, DimensionFilter, + ExtendedStatistic, Period, Statistic, + Unit, default_config()). + +-spec describe_alarms_for_metric( + Namespace ::string(), + MetricName ::string(), + DimensionFilter ::[{string(),string()}], + ExtendedStatistic ::string(), + Period ::pos_integer() | undefined, + Statistic ::statistic(), + Unit ::unit(), + Config ::aws_config() + ) -> term(). + +describe_alarms_for_metric( + Namespace, + MetricName, + DimensionFilter, + ExtendedStatistic, + Period, + Statistic, + Unit, + #aws_config{} = Config + ) -> + + Params = + [{"Namespace", Namespace}, + {"MetricName", MetricName}] + ++ + [{"ExtendedStatistic", ExtendedStatistic} || ExtendedStatistic/=""] + ++ + [{"Period", Period} || Period/=undefined] + ++ + [{"Statistic", Statistic} || Statistic/=""] + ++ + [{"Unit", Unit} || Unit/=""] + ++ + lists:flatten( + [begin + {Name, Value} = lists:nth(N, DimensionFilter), + [{?FMT("Dimensions.member.~b.Name", [N]), Name}, + {?FMT("Dimensions.member.~b.Value", [N]), Value}] + end + || N<-lists:seq(1, length(DimensionFilter))] + ), + Doc = mon_query(Config, "DescribeAlarmsForMetric", Params), + Members = xmerl_xpath:string("/DescribeAlarmsForMetricResponse/DescribeAlarmsForMetricResult/MetricAlarms/member", Doc), + [extract_member_dafm(Member) || Member <- Members]. + +extract_member_dafm(Node) -> + [ + {metric_name, get_text("MetricName", Node)}, + {namespace, get_text("Namespace", Node)}, + {dimensions, + [extract_dimension_dafm(Item) || Item <- xmerl_xpath:string("Dimensions/member", Node)] + }, + {actions_enabled, get_bool("ActionsEnabled", Node)}, + {alarm_actions, + [extract_actions_dafm(Item) || Item <- xmerl_xpath:string("AlarmActions/member", Node)]}, + {alarm_arn, get_text("AlarmArn", Node)}, + {alarm_configuration_updated_timestamp, get_time("AlarmConfigurationUpdatedTimestamp", Node)}, + {alarm_description, get_text("AlarmDescription", Node)}, + {alarm_name, get_text("AlarmName", Node)}, + {comparison_operator, get_text("ComparisonOperator", Node)}, + {evaluate_low_sample_count_percentile, get_text("EvaluateLowSampleCountPercentile", Node)}, + {evaluation_periods, get_integer("EvaluationPeriods", Node)}, + {extended_statistic, get_text("ExtendedStatistic", Node)}, + {insufficient_data_actions, + [extract_actions_dafm(Item) || Item <- xmerl_xpath:string("InsufficientDataActions/member", Node)]}, + {ok_actions, + [extract_actions_dafm(Item) || Item <- xmerl_xpath:string("OKActions/member", Node)]}, + {period, get_integer("Period", Node)}, + {state_reason, get_text("StateReason", Node)}, + {state_reason_data, get_text("StateReasonData", Node)}, + {state_updated_timestamp, get_time("StateUpdatedTimestamp", Node)}, + {state_value, get_text("StateValue", Node)}, + {statistic, get_text("Statistic", Node)}, + {threshold, get_float("Threshold", Node)}, + {treat_missing_data, get_text("TreatMissingData", Node)}, + {unit, get_text("Unit", Node)} + ]. + +extract_dimension_dafm(Node) -> + [ + {name, get_text("Name", Node)}, + {value, get_text("Value", Node)} + ]. + +extract_actions_dafm(Node) -> + {arn, get_text(Node)}. + %%------------------------------------------------------------------------------ %% @doc CloudWatch API - PutMetricData %% [http://docs.amazonwebservices.com/AmazonCloudWatch/latest/APIReference/index.html?API_PutMetricData.html] diff --git a/test/erlcloud_mon_tests.erl b/test/erlcloud_mon_tests.erl new file mode 100644 index 000000000..424791157 --- /dev/null +++ b/test/erlcloud_mon_tests.erl @@ -0,0 +1,176 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +-module(erlcloud_mon_tests). + + +-include_lib("eunit/include/eunit.hrl"). +-include("erlcloud.hrl"). +-include("erlcloud_aws.hrl"). + +-define(_mon_test(T), {?LINE, T}). + +-define(_f(F), fun() -> F end). + +%%============================================================================== +%% Test entry points +%%============================================================================== + +describe_mon_test_() -> + {foreach, fun start/0, fun stop/1, [ + fun describe_alarms_for_metric_input_tests/1, + fun describe_alarms_for_metric_output_tests/1 + ]}. + +start() -> + meck:new(erlcloud_aws), + ok. + +stop(_) -> + meck:unload(erlcloud_aws). + +%%============================================================================== +%% Output Test helpers +%%============================================================================== + +output_test(Fun, {Line, {Description, Response, Result}}) -> + {Description, {Line, fun() -> + meck:expect(erlcloud_aws, aws_request_xml4, 8, + {ok, element(1, xmerl_scan:string( + binary_to_list(Response)))} + ), + Actual = Fun(), + ?assertEqual(Result, Actual) + end}}. + + +%%============================================================================== +%% Input Test helpers +%%============================================================================== + +validate_param(_Param = {Key, _Value}, Params) -> + ?assertEqual(true, proplists:is_defined(Key, Params)). + +validate_params(Params, Expected) -> + [validate_param(X, Params) + || X <- [{"Action", ""}, {"Version", ""} | Expected] + + ]. +input_expect(Response, Params) -> + + fun(get, undefined, "monitoring.amazonaws.com", undefined, "/", QParams, + "monitoring", _) -> + validate_params(QParams, Params), + Response + end. + +input_test(Response, {Line, {Description, Fun, Params}}) + when is_list(Description) -> + + InputFunction = input_expect(Response, Params), + + meck:expect(erlcloud_aws, aws_request_xml4, InputFunction), + + {Description, {Line, fun() -> + Fun() + end}}. + + +%%============================================================================== +%% Input Tests +%%============================================================================== + +describe_alarms_for_metric_input_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"" + "">>) + ))}, + + ExpectedParams = [ + {"Namespace", ""}, + {"MetricName", ""}, + {"Dimensions.member.1.Name", ""}, + {"Dimensions.member.1.Value", ""}, + {"ExtendedStatistic", ""}, + {"Period", ""}, + {"Statistic", ""}, + {"Unit", ""} + ], + + input_test(Response, ?_mon_test( + {"Test describe alarms for metric", + ?_f(erlcloud_mon:describe_alarms_for_metric( + "AWS/EC2", "NetworkIn", [{"InstanceType","m1.large"}], "p95", + 17, "Average", "Seconds", + #aws_config{})), ExpectedParams})). + + +%%============================================================================== +%% Output Tests +%%============================================================================== + +describe_alarms_for_metric_output_tests(_) -> + + Test = ?_mon_test({"Test describe alarms for metric", + <<" + + + + rgallego_unauthorized_metric + 2018-02-07T17:38:24.224Z + ALARM + 1.0 + Threshold Crossed: 1 datapoint [2.0 (07/02/18 17:33:00)] was greater than or equal to the threshold (1.0). + + + arn:aws:sns:us-east-1:123456794008:rgallego_cloudtrail_sns_topic + + 2018-02-07T17:38:24.952Z + 300 + Sum + GreaterThanOrEqualToThreshold + rgallego_unauthorized_alarm + 1 + {\"version\":\"1.0\",\"queryDate\":\"2018-02-07T17:38:24.953+0000\",\"startDate\":\"2018-02-07T17:33:00.000+0000\",\"statistic\":\"Sum\",\"period\":300,\"recentDatapoints\":[2.0],\"threshold\":1.0} + true + CISBenchmark + + arn:aws:cloudwatch:us-east-1:123456794008:alarm:rgallego_unauthorized_alarm + + + + + + 0e8470e6-1032-11e8-b27b-7db55194b86f + + ">>, + [[{metric_name,"rgallego_unauthorized_metric"}, + {namespace,"CISBenchmark"}, + {dimensions,[]}, + {actions_enabled,true}, + {alarm_actions,[{arn,"arn:aws:sns:us-east-1:123456794008:rgallego_cloudtrail_sns_topic"}]}, + {alarm_arn,"arn:aws:cloudwatch:us-east-1:123456794008:alarm:rgallego_unauthorized_alarm"}, + {alarm_configuration_updated_timestamp,{{2018,2,7}, + {17,38,24}}}, + {alarm_description,[]}, + {alarm_name,"rgallego_unauthorized_alarm"}, + {comparison_operator,"GreaterThanOrEqualToThreshold"}, + {evaluate_low_sample_count_percentile,[]}, + {evaluation_periods,1}, + {extended_statistic,[]}, + {insufficient_data_actions,[]}, + {ok_actions,[]}, + {period,300}, + {state_reason,"Threshold Crossed: 1 datapoint [2.0 (07/02/18 17:33:00)] was greater than or equal to the threshold (1.0)."}, + {state_reason_data,"{\"version\":\"1.0\",\"queryDate\":\"2018-02-07T17:38:24.953+0000\",\"startDate\":\"2018-02-07T17:33:00.000+0000\",\"statistic\":\"Sum\",\"period\":300,\"recentDatapoints\":[2.0],\"threshold\":1.0}"}, + {state_updated_timestamp,{{2018,2,7},{17,38,24}}}, + {state_value,"ALARM"}, + {statistic,"Sum"}, + {threshold,1.0}, + {treat_missing_data,[]}, + {unit,[]}]] + }), + + output_test(?_f(erlcloud_mon:describe_alarms_for_metric( + "AWS/EC2", "NetworkIn", [{"InstanceType","m1.large"}], "p95", + 17, "Average", "Seconds", #aws_config{} + )), Test). From 7497730d29e11b2035ee988088507a8c977c1aec Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Thu, 28 Feb 2019 13:06:48 +0000 Subject: [PATCH 042/310] describe_metric_filters --- src/erlcloud_cloudwatch_logs.erl | 103 +++++++++++++++++ test/erlcloud_cloudwatch_logs_tests.erl | 142 +++++++++++++++++++++++- 2 files changed, 244 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index c476c43c1..f37478482 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -25,6 +25,9 @@ -type log_stream_name() :: string() | binary() | undefined. -type log_stream_prefix() :: string() | binary() | undefined. -type limit() :: pos_integer() | undefined. +-type filter_name_prefix() :: string() | binary() | undefined. +-type metric_name() :: string() | binary() | undefined. +-type metric_namespace() :: string() | binary() | undefined. -type log_stream_order() :: log_stream_name | last_event_time | undefined. -type events() :: [#{message => binary(), timestamp => pos_integer()}]. @@ -36,6 +39,7 @@ -type log_group() :: jsx:json_term(). -type log_stream() :: jsx:json_term(). +-type metric_filters() :: jsx:json_term(). -type tag():: {binary(), binary()}. -type tags_return() :: jsx:json_term(). @@ -65,6 +69,14 @@ describe_log_streams/6, describe_log_streams/7, + describe_metric_filters/0, + describe_metric_filters/1, + describe_metric_filters/2, + describe_metric_filters/3, + describe_metric_filters/4, + describe_metric_filters/6, + describe_metric_filters/7, + put_logs_events/4, put_logs_events/5, @@ -335,6 +347,97 @@ log_events(Events) -> [maps:with([message, timestamp], X) || #{message := _, timestamp := _} = X <- Events]. +%%------------------------------------------------------------------------------ +%% @doc +%% +%% DescribeMetricFilters action +%% https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_DescribeMetricFilters.html +%% +%% @end +%%------------------------------------------------------------------------------ +-spec describe_metric_filters() -> result_paged(metric_filters()). +describe_metric_filters() -> + describe_metric_filters(default_config()). + + +-spec describe_metric_filters( + aws_config() | log_group_name() +) -> result_paged(metric_filters()). +describe_metric_filters(#aws_config{} = Config) -> + describe_metric_filters(undefined, Config); +describe_metric_filters(LogGroupName) -> + describe_metric_filters(LogGroupName, default_config()). + + +-spec describe_metric_filters( + log_group_name(), + aws_config() +) -> result_paged(metric_filters()). +describe_metric_filters(LogGroupName, Config) -> + describe_metric_filters(LogGroupName, ?DEFAULT_LIMIT, Config). + + +-spec describe_metric_filters( + log_group_name(), + limit(), + aws_config() +) -> result_paged(metric_filters()). +describe_metric_filters(LogGroupName, Limit, Config) -> + describe_metric_filters(LogGroupName, Limit, undefined, Config). + + +-spec describe_metric_filters( + log_group_name(), + limit(), + filter_name_prefix(), + aws_config() +) -> result_paged(metric_filters()). +describe_metric_filters(LogGroupName, Limit, FilterNamePrefix, Config) -> + describe_metric_filters(LogGroupName, Limit, FilterNamePrefix, undefined, + undefined, Config). + + +-spec describe_metric_filters( + log_group_name(), + limit(), + filter_name_prefix(), + metric_name(), + metric_namespace(), + aws_config() +) -> result_paged(metric_filters()). +describe_metric_filters(LogGroupName, Limit, FilterNamePrefix, MetricName, + MetricNamespace, Config) -> + describe_metric_filters(LogGroupName, Limit, FilterNamePrefix, MetricName, + MetricNamespace, undefined, Config). + + +-spec describe_metric_filters( + log_group_name(), + limit(), + filter_name_prefix(), + metric_name(), + metric_namespace(), + paging_token(), + aws_config() +) -> result_paged(metric_filters()). +describe_metric_filters(LogGroupName, Limit, FilterNamePrefix, MetricName, + MetricNamespace, PrevToken, Config) -> + case cw_request(Config, "DescribeMetricFilters", [ + {<<"logGroupName">>, LogGroupName}, + {<<"limit">>, Limit}, + {<<"filterNamePrefix">>, FilterNamePrefix}, + {<<"metricName">>, MetricName}, + {<<"metricNamespace">>, MetricNamespace}, + {<<"nextToken">>, PrevToken} + ]) of + {ok, Data} -> + MetricFilters = proplists:get_value(<<"metricFilters">>, Data, []), + NextToken = proplists:get_value(<<"nextToken">>, Data, undefined), + {ok, MetricFilters, NextToken}; + {error, Reason} -> + {error, Reason} + end. + %%------------------------------------------------------------------------------ %% @doc %% diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index efb455955..f017c5ee9 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -40,7 +40,9 @@ -define(LOG_STREAM_NAME_PREFIX, <<"welcome">>). -define(LOG_STREAM_NAME, <<"welcome">>). -define(PAGING_TOKEN, <<"arn:aws:logs:us-east-1:352773894028:log-group:/aws/apigateway/welcome:*">>). - +-define(FILTER_NAME_PREFIX, <<"aws/apigateway/welcome">>). +-define(METRIC_NAME, <<"ct_test_metric">>). +-define(METRIC_NAMESPACE, <<"CISBenchmark">>). -define(LOG_GROUP, [ {<<"arn">>, <<"arn:aws:logs:us-east-1:352773894028:log-group:/aws/apigateway/welcome:*">>}, @@ -51,6 +53,20 @@ {<<"storedBytes">>, 85} ]). +-define(METRIC_FILTER, [ + {<<"creationTime">>, 1518024063379}, + {<<"filterName">>, <<"ct_test_filter">>}, + {<<"filterPattern">>, <<"{ ($.errorCode = \"*UnauthorizedOperation\") " + "|| ($.errorCode = \"AccessDenied*\") }">>}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}, + {<<"metricTransformations">>, [ + {<<"defaultValue">>, <<"0">>}, + {<<"metricValue">>, <<"1">>}, + {<<"metricNamespace">>, ?METRIC_NAMESPACE}, + {<<"metricName">>, ?METRIC_NAME} + ]} +]). + -define(LOG_STREAM, [ {<<"arn">>, <<"arn:aws:logs:us-east-1:352773894028:log-group:/aws/apigateway/welcome:log-stream:welcome">>}, {<<"creationTime">>, 1476283527335}, @@ -76,6 +92,9 @@ erlcloud_cloudwatch_test_() -> fun describe_log_groups_input_tests/1, fun describe_log_groups_output_tests/1, + fun describe_metric_filters_input_tests/1, + fun describe_metric_filters_output_tests/1, + fun describe_log_streams_input_tests/1, fun describe_log_streams_output_tests/1, @@ -172,6 +191,118 @@ describe_log_groups_input_tests(_) -> ]). +describe_metric_filters_input_tests(_) -> + input_tests(jsx:encode([{<<"metricFilters">>, []}]), [ + ?_cloudwatch_test( + {"Tests describing metric filters with no parameters", + ?_f(erlcloud_cloudwatch_logs:describe_metric_filters()), + [{<<"Action">>, <<"DescribeMetricFilters">>}, + {<<"Version">>, ?API_VERSION}, + {<<"limit">>, ?DEFAULT_LIMIT}]} + ), + ?_cloudwatch_test( + {"Tests describing metric filters with custom AWS config provided", + ?_f(erlcloud_cloudwatch_logs:describe_metric_filters( + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"DescribeMetricFilters">>}, + {<<"Version">>, ?API_VERSION}, + {<<"limit">>, ?DEFAULT_LIMIT}]} + ), + ?_cloudwatch_test( + {"Tests describing metric filters with log group name provided", + ?_f(erlcloud_cloudwatch_logs:describe_metric_filters( + ?LOG_GROUP_NAME + )), + [{<<"Action">>, <<"DescribeMetricFilters">>}, + {<<"Version">>, ?API_VERSION}, + {<<"limit">>, ?DEFAULT_LIMIT}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests describing metric filters with custom AWS config and " + "log group name provided", + ?_f(erlcloud_cloudwatch_logs:describe_metric_filters( + ?LOG_GROUP_NAME, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"DescribeMetricFilters">>}, + {<<"Version">>, ?API_VERSION}, + {<<"limit">>, ?DEFAULT_LIMIT}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests describing metric filters with custom AWS config, " + "log group name and limit provided", + ?_f(erlcloud_cloudwatch_logs:describe_metric_filters( + ?LOG_GROUP_NAME, + ?NON_DEFAULT_LIMIT, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"DescribeMetricFilters">>}, + {<<"Version">>, ?API_VERSION}, + {<<"limit">>, ?NON_DEFAULT_LIMIT}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests describing metric filters with custom AWS config, " + "log group name, limit and filter name prefix provided", + ?_f(erlcloud_cloudwatch_logs:describe_metric_filters( + ?LOG_GROUP_NAME, + ?NON_DEFAULT_LIMIT, + ?FILTER_NAME_PREFIX, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"DescribeMetricFilters">>}, + {<<"Version">>, ?API_VERSION}, + {<<"filterNamePrefix">>, ?FILTER_NAME_PREFIX}, + {<<"limit">>, ?NON_DEFAULT_LIMIT}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests describing metric filters with custom AWS config, " + "log group name, limit, filter name prefix, metric name and " + "metric namespace provided", + ?_f(erlcloud_cloudwatch_logs:describe_metric_filters( + ?LOG_GROUP_NAME, + ?NON_DEFAULT_LIMIT, + ?FILTER_NAME_PREFIX, + ?METRIC_NAME, + ?METRIC_NAMESPACE, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"DescribeMetricFilters">>}, + {<<"Version">>, ?API_VERSION}, + {<<"limit">>, ?NON_DEFAULT_LIMIT}, + {<<"filterNamePrefix">>, ?FILTER_NAME_PREFIX}, + {<<"metricName">>, ?METRIC_NAME}, + {<<"metricNamespace">>, ?METRIC_NAMESPACE}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests describing metric filters with custom AWS config, " + "log group name, limit, filter name prefix, metric name, " + "metric namespace and pagination token provided", + ?_f(erlcloud_cloudwatch_logs:describe_metric_filters( + ?LOG_GROUP_NAME, + ?NON_DEFAULT_LIMIT, + ?FILTER_NAME_PREFIX, + ?METRIC_NAME, + ?METRIC_NAMESPACE, + ?PAGING_TOKEN, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"DescribeMetricFilters">>}, + {<<"Version">>, ?API_VERSION}, + {<<"limit">>, ?NON_DEFAULT_LIMIT}, + {<<"filterNamePrefix">>, ?FILTER_NAME_PREFIX}, + {<<"metricName">>, ?METRIC_NAME}, + {<<"metricNamespace">>, ?METRIC_NAMESPACE}, + {<<"nextToken">>, ?PAGING_TOKEN}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ) + ]). + describe_log_groups_output_tests(_) -> output_tests(?_f(erlcloud_cloudwatch_logs:describe_log_groups()), [ ?_cloudwatch_test( @@ -182,6 +313,15 @@ describe_log_groups_output_tests(_) -> ]). +describe_metric_filters_output_tests(_) -> + output_tests(?_f(erlcloud_cloudwatch_logs:describe_metric_filters()), [ + ?_cloudwatch_test( + {"Tests describing all metric filters", + jsx:encode([{<<"metricFilters">>, [?METRIC_FILTER]}]), + {ok, [?METRIC_FILTER], undefined}} + ) + ]). + describe_log_streams_input_tests(_) -> input_tests(jsx:encode([{<<"logStreams">>, []}]), [ ?_cloudwatch_test( From 87570dc58652a16343aa0aebc0b4d44b3b5d48c6 Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Thu, 28 Feb 2019 15:05:58 +0000 Subject: [PATCH 043/310] common filter_undef, removed base_path --- src/erlcloud_guardduty.erl | 11 +++-------- src/erlcloud_lambda.erl | 5 +---- src/erlcloud_util.erl | 7 +++++-- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/erlcloud_guardduty.erl b/src/erlcloud_guardduty.erl index ec8510b42..3f5d8e602 100644 --- a/src/erlcloud_guardduty.erl +++ b/src/erlcloud_guardduty.erl @@ -16,6 +16,7 @@ list_detectors/0, list_detectors/1, list_detectors/2, list_detectors/3 ]). +-import(erlcloud_util, [filter_undef/1]). -type gd_return() :: {ok, proplist()} | {error, term()}. @@ -58,7 +59,7 @@ get_detector(DetectorId) -> -spec get_detector(DetectorId :: binary(), Config :: aws_config()) -> gd_return(). get_detector(DetectorId, Config) -> - Path = base_path() ++ "detector/" ++ binary_to_list(DetectorId), + Path = "/detector/" ++ binary_to_list(DetectorId), guardduty_request(Config, get, Path, undefined). @@ -85,7 +86,7 @@ list_detectors(Marker, MaxItems) -> MaxItems :: integer(), Config :: aws_config()) -> gd_return(). list_detectors(Marker, MaxItems, Config) -> - Path = base_path() ++ "detector", + Path = "/detector", QParams = filter_undef([{"Marker", Marker}, {"MaxItems", MaxItems}]), guardduty_request(Config, get, Path, undefined, QParams). @@ -95,12 +96,6 @@ list_detectors(Marker, MaxItems, Config) -> %%% Internal Functions %%%------------------------------------------------------------------------------ -filter_undef(List) -> - lists:filter(fun({_Name, Value}) -> Value =/= undefined end, List). - -base_path() -> - "/". - guardduty_request(Config, Method, Path, Body) -> guardduty_request(Config, Method, Path, Body, []). diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index 0618e1e3f..0a76fd943 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -33,7 +33,7 @@ 'python2.7' | 'python3.6' | 'dotnetcore1.0' | 'dotnetcore2.0' | 'dotnetcore2.1' | 'nodejs4.3-edge' | 'go1.x'). -type(return_val() :: any()). - +-import(erlcloud_util, [filter_undef/1]). %%------------------------------------------------------------------------------ %% Library initialization. @@ -718,9 +718,6 @@ from_record(#erlcloud_lambda_code{s3Bucket = S3Bucket, {<<"ZipFile">>, ZipFile}], filter_undef(List). -filter_undef(List) -> - lists:filter(fun({_Name, Value}) -> Value =/= undefined end, List). - base_path() -> "/" ++ ?API_VERSION ++ "/". diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index c2d3eae7a..9d08c7f6c 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -2,7 +2,8 @@ -export([sha_mac/2, sha256_mac/2, md5/1, sha256/1, rand_uniform/1, is_dns_compliant_name/1, query_all/4, query_all/5, query_all_token/4, make_response/2, - get_items/2, to_string/1, encode_list/2, next_token/2]). + get_items/2, to_string/1, encode_list/2, next_token/2, + filter_undef/1]). -define(MAX_ITEMS, 1000). @@ -128,4 +129,6 @@ next_token(Path, XML) -> ok end. - +-spec filter_undef(proplists:proplist()) -> proplists:proplist(). +filter_undef(List) -> + lists:filter(fun({_Name, Value}) -> Value =/= undefined end, List). From 02f44675d12e45fc779a3354c08e45e971974c1a Mon Sep 17 00:00:00 2001 From: motobob Date: Fri, 1 Mar 2019 14:08:50 +0000 Subject: [PATCH 044/310] hex plugin has changed --- src/erlcloud.app.src | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/erlcloud.app.src b/src/erlcloud.app.src index 06268f059..677803889 100644 --- a/src/erlcloud.app.src +++ b/src/erlcloud.app.src @@ -3,7 +3,7 @@ {application, erlcloud, [{description, "AWS APIs library for Erlang"}, - {vsn, git}, + {vsn, "git"}, {registered, []}, {applications, [stdlib, kernel, @@ -20,7 +20,6 @@ {modules, []}, {env, []}, {licenses, ["MIT"]}, - {links, [{"Github", "https://github.com/erlcloud/erlcloud"}]}, - {maintainers, []} + {links, [{"Github", "https://github.com/erlcloud/erlcloud"}]} ] }. From fcf0cf5adb15703e18b76a2c004a015fdc3791ee Mon Sep 17 00:00:00 2001 From: motobob Date: Mon, 4 Mar 2019 09:46:26 +0000 Subject: [PATCH 045/310] update eini --- rebar.config | 2 +- rebar.config.script | 2 +- rebar.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index ee3970f31..1e3d0ef8e 100644 --- a/rebar.config +++ b/rebar.config @@ -19,7 +19,7 @@ {deps, [ {jsx, "2.9.0"}, {lhttpc, "1.6.2"}, - {eini, "1.2.5"}, + {eini, "1.2.6"}, {base16, "1.0.0"} ]}. diff --git a/rebar.config.script b/rebar.config.script index 27c7683d6..19a5167d0 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -8,7 +8,7 @@ case erlang:function_exported(rebar3, main, 1) of [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.12"}}}, {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "2.9.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, - {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.5"}}}, + {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.6"}}}, {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, {base16, ".*", {git, "https://github.com/goj/base16.git", {tag, "1.0.0"}}}]} | lists:keydelete(deps, 1, CONFIG)] diff --git a/rebar.lock b/rebar.lock index fb4f83c7b..4302de18f 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,12 +1,12 @@ {"1.1.0", [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, - {<<"eini">>,{pkg,<<"eini">>,<<"1.2.5">>},0}, + {<<"eini">>,{pkg,<<"eini">>,<<"1.2.6">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.9.0">>},0}, {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.6.2">>},0}]}. [ {pkg_hash,[ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, - {<<"eini">>, <<"98E4988474FAAF821E3511579090AF989096ADBDB5F7BFAA03AA0A9AC80296B2">>}, + {<<"eini">>, <<"DFFA48476FD89FB6E41CEEA0ADFA1BC6E7862CCD6584417442F8BB37E5D34715">>}, {<<"jsx">>, <<"D2F6E5F069C00266CAD52FB15D87C428579EA4D7D73A33669E12679E203329DD">>}, {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} ]. From c5b020c30098abc7d0f8d1370a6d513dbfb6b69e Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Mon, 4 Mar 2019 15:06:08 -0600 Subject: [PATCH 046/310] erlcloud_ddb_streams: add approximate_creation_date_time to #ddb_streams_stream_record{} Adds [ApproximateCreationDateTime](https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_streams_StreamRecord.html#DDB-Type-streams_StreamRecord-ApproximateCreationDateTime) field to `erlcloud_ddb_streams:get_records` function output. --- include/erlcloud_ddb_streams.hrl | 3 ++- src/erlcloud_ddb_streams.erl | 3 ++- test/erlcloud_ddb_streams_tests.erl | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/include/erlcloud_ddb_streams.hrl b/include/erlcloud_ddb_streams.hrl index 9fda7eebd..2ecfde30a 100644 --- a/include/erlcloud_ddb_streams.hrl +++ b/include/erlcloud_ddb_streams.hrl @@ -23,7 +23,8 @@ table_name :: undefined | erlcloud_ddb_streams:table_name() }). -record(ddb_streams_stream_record, - {keys :: undefined | erlcloud_ddb_streams:key(), + {approximate_creation_date_time :: undefined | number(), + keys :: undefined | erlcloud_ddb_streams:key(), new_image :: undefined | erlcloud_ddb_streams:item(), old_image :: undefined | erlcloud_ddb_streams:item(), sequence_number :: undefined | erlcloud_ddb_streams:sequence_number(), diff --git a/src/erlcloud_ddb_streams.erl b/src/erlcloud_ddb_streams.erl index af6cde2c8..aa17fc371 100644 --- a/src/erlcloud_ddb_streams.erl +++ b/src/erlcloud_ddb_streams.erl @@ -467,7 +467,8 @@ stream_description_record() -> -spec stream_record_record() -> record_desc(). stream_record_record() -> {#ddb_streams_stream_record{}, - [{<<"Keys">>, #ddb_streams_stream_record.keys, fun undynamize_key/2}, + [{<<"ApproximateCreationDateTime">>, #ddb_streams_stream_record.approximate_creation_date_time, fun id/2}, + {<<"Keys">>, #ddb_streams_stream_record.keys, fun undynamize_key/2}, {<<"NewImage">>, #ddb_streams_stream_record.new_image, fun undynamize_item/2}, {<<"OldImage">>, #ddb_streams_stream_record.old_image, fun undynamize_item/2}, {<<"SequenceNumber">>, #ddb_streams_stream_record.sequence_number, fun id/2}, diff --git a/test/erlcloud_ddb_streams_tests.erl b/test/erlcloud_ddb_streams_tests.erl index e94a78789..6b86cb040 100644 --- a/test/erlcloud_ddb_streams_tests.erl +++ b/test/erlcloud_ddb_streams_tests.erl @@ -369,6 +369,7 @@ get_records_output_tests(_) -> { \"awsRegion\": \"us-west-2\", \"dynamodb\": { + \"ApproximateCreationDateTime\": 1551727994, \"Keys\": { \"ForumName\": {\"S\": \"DynamoDB\"}, \"Subject\": {\"S\": \"DynamoDB Thread 3\"} @@ -385,6 +386,7 @@ get_records_output_tests(_) -> { \"awsRegion\": \"us-west-2\", \"dynamodb\": { + \"ApproximateCreationDateTime\": 1551727994, \"Keys\": { \"ForumName\": {\"S\": \"DynamoDB\"}, \"Subject\": {\"S\": \"DynamoDB Thread 1\"} @@ -401,6 +403,7 @@ get_records_output_tests(_) -> { \"awsRegion\": \"us-west-2\", \"dynamodb\": { + \"ApproximateCreationDateTime\": 1551727994, \"Keys\": { \"ForumName\": {\"S\": \"DynamoDB\"}, \"Subject\": {\"S\": \"DynamoDB Thread 2\"} @@ -419,6 +422,7 @@ get_records_output_tests(_) -> {ok, [#ddb_streams_record{ aws_region = <<"us-west-2">>, dynamodb = #ddb_streams_stream_record{ + approximate_creation_date_time = 1551727994, keys = [{<<"ForumName">>, <<"DynamoDB">>}, {<<"Subject">>, <<"DynamoDB Thread 3">>}], new_image = undefined, @@ -433,6 +437,7 @@ get_records_output_tests(_) -> #ddb_streams_record{ aws_region = <<"us-west-2">>, dynamodb = #ddb_streams_stream_record{ + approximate_creation_date_time = 1551727994, keys = [{<<"ForumName">>, <<"DynamoDB">>}, {<<"Subject">>, <<"DynamoDB Thread 1">>}], new_image = undefined, @@ -447,6 +452,7 @@ get_records_output_tests(_) -> #ddb_streams_record{ aws_region = <<"us-west-2">>, dynamodb = #ddb_streams_stream_record{ + approximate_creation_date_time = 1551727994, keys = [{<<"ForumName">>, <<"DynamoDB">>}, {<<"Subject">>, <<"DynamoDB Thread 2">>}], new_image = undefined, From 1d49fb2806ce029c5aef08c3c17d097ca64445fd Mon Sep 17 00:00:00 2001 From: "david.escher" Date: Fri, 29 Mar 2019 12:08:03 +0100 Subject: [PATCH 047/310] Export list_multipart_uploads/4, fixes #587 --- src/erlcloud_s3.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 0c067339a..5fe5b50d9 100755 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -35,7 +35,7 @@ upload_part/5, upload_part/7, complete_multipart/4, complete_multipart/6, abort_multipart/3, abort_multipart/6, - list_multipart_uploads/1, list_multipart_uploads/2, + list_multipart_uploads/1, list_multipart_uploads/2, list_multipart_uploads/4, get_object_url/2, get_object_url/3, get_bucket_and_key/1, list_bucket_inventory/1, list_bucket_inventory/2, list_bucket_inventory/3, From 967292cddb98c97bb8ebea61b18ffc22c406bfef Mon Sep 17 00:00:00 2001 From: Kozlov Yakov Date: Mon, 8 Apr 2019 18:49:16 +0300 Subject: [PATCH 048/310] Retry lambda call in case of x-amz-function-error exists --- src/erlcloud.app.src | 7 ++++++- src/erlcloud_retry.erl | 12 +++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/erlcloud.app.src b/src/erlcloud.app.src index 06268f059..eede120fb 100644 --- a/src/erlcloud.app.src +++ b/src/erlcloud.app.src @@ -18,7 +18,12 @@ %% hackney, lhttpc]}, {modules, []}, - {env, []}, + {env, [ + %% AWS Lambda may response 200 with additional headers in case of errors. + %% Set flag to true to use retry logic for such cases + %% @see https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html + {retry_x_amz_function_error, false} + ]}, {licenses, ["MIT"]}, {links, [{"Github", "https://github.com/erlcloud/erlcloud"}]}, {maintainers, []} diff --git a/src/erlcloud_retry.erl b/src/erlcloud_retry.erl index 97511006f..0c8fc373c 100644 --- a/src/erlcloud_retry.erl +++ b/src/erlcloud_retry.erl @@ -70,8 +70,18 @@ request_and_retry(Config, ResultFun, {retry, Request}, MaxAttempts) -> erlcloud_aws:get_timeout(Config), Config), case Rsp of {ok, {{Status, StatusLine}, ResponseHeaders, ResponseBody}} -> + ResponseType = case Status >= 200 andalso Status < 300 of + true -> + {ok, RetryFunctionErrors} = application:get_env(erlcloud, retry_x_amz_function_error), + case RetryFunctionErrors andalso lists:keymember("x-amz-function-error", 1, ResponseHeaders) of + true -> error; + false -> ok + end; + false -> + error + end, Request3 = Request2#aws_request{ - response_type = if Status >= 200, Status < 300 -> ok; true -> error end, + response_type = ResponseType, error_type = aws, response_status = Status, response_status_line = StatusLine, From b531f5ad3ab6d4d407f4e3af0d179ab93e0fb4f1 Mon Sep 17 00:00:00 2001 From: Kozlov Yakov Date: Mon, 8 Apr 2019 18:55:03 +0300 Subject: [PATCH 049/310] Fix eunit crash --- src/erlcloud_retry.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_retry.erl b/src/erlcloud_retry.erl index 0c8fc373c..8e63830b0 100644 --- a/src/erlcloud_retry.erl +++ b/src/erlcloud_retry.erl @@ -72,7 +72,7 @@ request_and_retry(Config, ResultFun, {retry, Request}, MaxAttempts) -> {ok, {{Status, StatusLine}, ResponseHeaders, ResponseBody}} -> ResponseType = case Status >= 200 andalso Status < 300 of true -> - {ok, RetryFunctionErrors} = application:get_env(erlcloud, retry_x_amz_function_error), + RetryFunctionErrors = application:get_env(erlcloud, retry_x_amz_function_error, false), case RetryFunctionErrors andalso lists:keymember("x-amz-function-error", 1, ResponseHeaders) of true -> error; false -> ok From bebafe207b5b7aebec4b7be13dde73ab4c433ff4 Mon Sep 17 00:00:00 2001 From: Kozlov Yakov Date: Tue, 9 Apr 2019 00:38:01 +0300 Subject: [PATCH 050/310] Allow to customize response_status logic --- include/erlcloud_aws.hrl | 5 +++++ src/erlcloud.app.src | 7 +------ src/erlcloud_retry.erl | 44 +++++++++++++++++++++++++++------------- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index b1fff2bda..1363e1213 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -137,6 +137,11 @@ %% If you provide a custom function be aware of this anticipated change. %% See erlcloud_retry for full documentation. retry=fun erlcloud_retry:no_retry/1::erlcloud_retry:retry_fun(), + + %% By default treat all non 2xx http error codes as errors. + %% But in some cases, like lambda call it useful to override such + %% behaviour by custom one. + response_type=fun erlcloud_retry:only_http_errors/1::erlcloud_retry:response_type_fun(), %% Currently matches DynamoDB retry %% It's likely this is too many retries for other services retry_num=10::non_neg_integer(), diff --git a/src/erlcloud.app.src b/src/erlcloud.app.src index eede120fb..06268f059 100644 --- a/src/erlcloud.app.src +++ b/src/erlcloud.app.src @@ -18,12 +18,7 @@ %% hackney, lhttpc]}, {modules, []}, - {env, [ - %% AWS Lambda may response 200 with additional headers in case of errors. - %% Set flag to true to use retry logic for such cases - %% @see https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html - {retry_x_amz_function_error, false} - ]}, + {env, []}, {licenses, ["MIT"]}, {links, [{"Github", "https://github.com/erlcloud/erlcloud"}]}, {maintainers, []} diff --git a/src/erlcloud_retry.erl b/src/erlcloud_retry.erl index 8e63830b0..fc46e8402 100644 --- a/src/erlcloud_retry.erl +++ b/src/erlcloud_retry.erl @@ -20,12 +20,16 @@ %% Helpers -export([backoff/1, no_retry/1, - default_retry/1 + default_retry/1, + + only_http_errors/1, + aws_api_errors/1 ]). --export_type([should_retry/0, retry_fun/0]). +-export_type([should_retry/0, retry_fun/0, response_type_fun/0]). -type should_retry() :: {retry | error, #aws_request{}}. -type retry_fun() :: fun((#aws_request{}) -> should_retry()). +-type response_type_fun() :: fun((#aws_request{}) -> ok | error). %% Internal impl api -export([request/3]). @@ -52,6 +56,27 @@ request(Config, #aws_request{attempt = 0} = Request, ResultFun) -> MaxAttempts = Config#aws_config.retry_num, request_and_retry(Config, ResultFun, {retry, Request}, MaxAttempts). +-spec only_http_errors(#aws_request{}) -> ok | error. +only_http_errors(#aws_request{response_status=Status}) + when Status >= 200, Status < 300 + -> + ok; +only_http_errors(_) -> + error. + +-spec aws_api_errors(#aws_request{}) -> ok | error. +aws_api_errors(#aws_request{response_status=Status, response_headers=ResponseHeaders}) + when Status >= 200, Status < 300 + -> + case lists:keymember("x-amz-function-error", 1, ResponseHeaders) of + true -> + error; + false -> + ok + end; +aws_api_errors(_) -> + error. + request_and_retry(_, _, {_, Request}, 0) -> Request; request_and_retry(_, _, {error, Request}, _) -> @@ -66,28 +91,19 @@ request_and_retry(Config, ResultFun, {retry, Request}, MaxAttempts) -> } = Request, Request2 = Request#aws_request{attempt = Attempt + 1}, RetryFun = Config#aws_config.retry, + ResponseTypeFun = Config#aws_config.response_type, Rsp = erlcloud_httpc:request(URI, Method, Headers, Body, erlcloud_aws:get_timeout(Config), Config), case Rsp of {ok, {{Status, StatusLine}, ResponseHeaders, ResponseBody}} -> - ResponseType = case Status >= 200 andalso Status < 300 of - true -> - RetryFunctionErrors = application:get_env(erlcloud, retry_x_amz_function_error, false), - case RetryFunctionErrors andalso lists:keymember("x-amz-function-error", 1, ResponseHeaders) of - true -> error; - false -> ok - end; - false -> - error - end, Request3 = Request2#aws_request{ - response_type = ResponseType, error_type = aws, response_status = Status, response_status_line = StatusLine, response_headers = ResponseHeaders, response_body = ResponseBody}, - Request4 = ResultFun(Request3), + ResponseType = ResponseTypeFun(Request3), + Request4 = ResultFun(Request3#aws_request{response_type=ResponseType}), case Request4#aws_request.response_type of ok -> Request4; From 5a208c92aae2caa0dc59ede0821d0b0437a41f11 Mon Sep 17 00:00:00 2001 From: Kozlov Yakov Date: Tue, 9 Apr 2019 12:02:10 +0300 Subject: [PATCH 051/310] Rename aws_api_errors/1 to lambda_fun_errors/1 --- src/erlcloud_retry.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_retry.erl b/src/erlcloud_retry.erl index fc46e8402..4039e4804 100644 --- a/src/erlcloud_retry.erl +++ b/src/erlcloud_retry.erl @@ -23,7 +23,7 @@ default_retry/1, only_http_errors/1, - aws_api_errors/1 + lambda_fun_errors/1 ]). -export_type([should_retry/0, retry_fun/0, response_type_fun/0]). @@ -64,8 +64,8 @@ only_http_errors(#aws_request{response_status=Status}) only_http_errors(_) -> error. --spec aws_api_errors(#aws_request{}) -> ok | error. -aws_api_errors(#aws_request{response_status=Status, response_headers=ResponseHeaders}) +-spec lambda_fun_errors(#aws_request{}) -> ok | error. +lambda_fun_errors(#aws_request{response_status=Status, response_headers=ResponseHeaders}) when Status >= 200, Status < 300 -> case lists:keymember("x-amz-function-error", 1, ResponseHeaders) of @@ -74,7 +74,7 @@ aws_api_errors(#aws_request{response_status=Status, response_headers=ResponseHea false -> ok end; -aws_api_errors(_) -> +lambda_fun_errors(_) -> error. request_and_retry(_, _, {_, Request}, 0) -> From eb6ce2e0fe6a9dea6f00e396c7bd43efbabb285e Mon Sep 17 00:00:00 2001 From: Kozlov Yakov Date: Tue, 9 Apr 2019 12:44:29 +0300 Subject: [PATCH 052/310] Rename #aws_config.response_type to retry_response_type --- include/erlcloud_aws.hrl | 2 +- src/erlcloud_retry.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 1363e1213..1b060b2f2 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -141,7 +141,7 @@ %% By default treat all non 2xx http error codes as errors. %% But in some cases, like lambda call it useful to override such %% behaviour by custom one. - response_type=fun erlcloud_retry:only_http_errors/1::erlcloud_retry:response_type_fun(), + retry_response_type=fun erlcloud_retry:only_http_errors/1::erlcloud_retry:response_type_fun(), %% Currently matches DynamoDB retry %% It's likely this is too many retries for other services retry_num=10::non_neg_integer(), diff --git a/src/erlcloud_retry.erl b/src/erlcloud_retry.erl index 4039e4804..4d2944584 100644 --- a/src/erlcloud_retry.erl +++ b/src/erlcloud_retry.erl @@ -91,7 +91,7 @@ request_and_retry(Config, ResultFun, {retry, Request}, MaxAttempts) -> } = Request, Request2 = Request#aws_request{attempt = Attempt + 1}, RetryFun = Config#aws_config.retry, - ResponseTypeFun = Config#aws_config.response_type, + ResponseTypeFun = Config#aws_config.retry_response_type, Rsp = erlcloud_httpc:request(URI, Method, Headers, Body, erlcloud_aws:get_timeout(Config), Config), case Rsp of From 72697eb3b2b56f99ef77ce815497e52a4780cdd6 Mon Sep 17 00:00:00 2001 From: Pavel Puchkin Date: Fri, 12 Apr 2019 20:15:47 +0200 Subject: [PATCH 053/310] Implement get_queue_url function --- src/erlcloud_sqs.erl | 15 +++++++++++++++ test/erlcloud_sqs_tests.erl | 22 ++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index 0c6194209..d4ce66337 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -12,6 +12,7 @@ delete_message/2, delete_message/3, delete_queue/1, delete_queue/2, purge_queue/1, purge_queue/2, + get_queue_url/1, get_queue_url/2, get_queue_attributes/1, get_queue_attributes/2, get_queue_attributes/3, list_queues/0, list_queues/1, list_queues/2, receive_message/1, receive_message/2, receive_message/3, receive_message/4, @@ -186,6 +187,20 @@ purge_queue(QueueName, Config) when is_list(QueueName), is_record(Config, aws_config) -> sqs_simple_request(Config, QueueName, "PurgeQueue", []). +-spec get_queue_url(string()) -> proplist() | no_return(). +get_queue_url(QueueName) -> + get_queue_url(QueueName, default_config()). + +-spec get_queue_url(string(), aws_config()) -> proplist() | no_return(). +get_queue_url(QueueName, Config) -> + Doc = sqs_xml_request(Config, "/", "GetQueueUrl", [{"QueueName", QueueName}]), + erlcloud_xml:decode( + [ + {queue_url, "GetQueueUrlResult/QueueUrl", text} + ], + Doc + ). + -spec get_queue_attributes(string()) -> proplist() | no_return(). get_queue_attributes(QueueName) -> get_queue_attributes(QueueName, all). diff --git a/test/erlcloud_sqs_tests.erl b/test/erlcloud_sqs_tests.erl index 5e71a9814..f98efd596 100644 --- a/test/erlcloud_sqs_tests.erl +++ b/test/erlcloud_sqs_tests.erl @@ -18,6 +18,7 @@ erlcloud_api_test_() -> fun stop/1, [ fun set_queue_attributes/1, + fun get_queue_url/1, fun send_message_with_message_opts/1, fun send_message_with_message_attributes/1, fun receive_messages_with_message_attributes/1, @@ -162,6 +163,27 @@ set_queue_attributes(_) -> ", input_tests(Response, Tests). +get_queue_url(_) -> + Expected = [ + {"Action", "GetQueueUrl"}, + {"QueueName", "Queue"} + ], + Tests = + [?_sqs_test( + {"Test queue URL getting.", + ?_f(erlcloud_sqs:get_queue_url("Queue")), + Expected})], + Response = " + + + https://sqs.us-east-2.amazonaws.com/123456789012/Queue + + + 470a6f13-2ed9-4181-ad8a-2fdea142988e + +", + input_tests(Response, Tests). + send_message_with_message_opts(_) -> MessageBody = "Hello", MessageOpts = [ From 3fd915904896b8388102bf0a2ae03f5d7a26babf Mon Sep 17 00:00:00 2001 From: Hugues Martel Date: Wed, 6 Feb 2019 10:45:17 -0600 Subject: [PATCH 054/310] Add TransactWriteItems and TransactGetItems wrappers. --- .gitignore | 1 + include/erlcloud_ddb2.hrl | 20 +++- src/erlcloud_ddb2.erl | 237 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b9000b235..920c52d3f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ ebin/*.app .rebar/ rebar rebar3 +.rebar3 *.sublime-* deps/ doc/ diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index 493f8d145..3cc3f47b9 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -309,6 +309,24 @@ time_to_live_description :: undefined | #ddb2_time_to_live_description{} }). +-record(ddb2_transact_write_items, + {consumed_capacity :: undefined | [#ddb2_consumed_capacity{}], + item_collection_metrics :: undefined | [{erlcloud_ddb2:table_name(), [#ddb2_item_collection_metrics{}]}], + % AWS documentation infer that it should be possible to get old return + % values upon condition failure, but I have been unable to do so. + % Still, let's put a field so we can return {ok, []} on success. + attributes :: undefined + }). + +-record(ddb2_transact_get_items_response, + {item :: erlcloud_ddb2:out_item() + }). + +-record(ddb2_transact_get_items, + {consumed_capacity :: undefined | [#ddb2_consumed_capacity{}], + responses :: undefined | [#ddb2_transact_get_items_response{}] + }). + -record(ddb2_backup_description, {backup_details :: undefined | #ddb2_backup_details{}, source_table_details :: undefined | #ddb2_source_table_details{}, @@ -345,4 +363,4 @@ -record(ddb2_restore_table_to_point_in_time, {table_description :: undefined | #ddb2_table_description{} }). --endif. \ No newline at end of file +-endif. diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 0d5d90d28..6a03bb932 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -113,6 +113,8 @@ restore_table_to_point_in_time/2, restore_table_to_point_in_time/3, restore_table_to_point_in_time/4, scan/1, scan/2, scan/3, tag_resource/2, tag_resource/3, + transact_get_items/1, transact_get_items/2, transact_get_items/3, + transact_write_items/1, transact_write_items/2, transact_write_items/3, untag_resource/2, untag_resource/3, update_continuous_backups/2, update_continuous_backups/3, update_continuous_backups/4, update_item/3, update_item/4, update_item/5, @@ -378,6 +380,7 @@ default_config() -> erlcloud_aws:default_config(). -type out_attr() :: {attr_name(), out_attr_value()}. -type out_item() :: [out_attr() | in_attr()]. % in_attr in the case of typed_record -type ok_return(T) :: {ok, T} | {error, term()}. +-type client_request_token() :: binary(). %%%------------------------------------------------------------------------------ %%% Shared Dynamizers @@ -1028,6 +1031,12 @@ dynamize_expression({contains, Path, Operand}) -> return_consumed_capacity_opt() -> {return_consumed_capacity, <<"ReturnConsumedCapacity">>, fun dynamize_return_consumed_capacity/1}. +-type client_request_token_opt() :: {client_request_token, client_request_token()}. + +-spec client_request_token_opt() -> opt_table_entry(). +client_request_token_opt() -> + {client_request_token, <<"ClientRequestToken">>, fun id/1}. + -type return_item_collection_metrics_opt() :: {return_item_collection_metrics, return_item_collection_metrics()}. -spec return_item_collection_metrics_opt() -> opt_table_entry(). @@ -1513,6 +1522,234 @@ batch_write_item(RequestItems, Opts, Config) -> {error, _} = Out -> Out end. +-type transact_get_items_transact_item_opts() :: expression_attribute_names_opt() | + projection_expression_opt(). +-type transact_get_items_opts() :: [transact_get_items_transact_item_opts()]. + +-type transact_get_items_get_item() :: {table_name(), key(), transact_get_items_transact_item_opts()}. + +-type transact_get_items_get() :: {get, transact_get_items_get_item()}. + +-type transact_get_items_transact_item() :: transact_get_items_get(). +-type transact_get_items_transact_items() :: maybe_list(transact_get_items_transact_item()). + +-type transact_get_items_return() :: ddb_return(#ddb2_transact_get_items{}, out_item()). + +-spec dynamize_transact_get_items_transact_items(transact_write_items_transact_items()) + -> json_pair(). +dynamize_transact_get_items_transact_items(TransactItems) -> + dynamize_maybe_list(fun dynamize_transact_get_items_transact_item/1, TransactItems). + +-spec transact_get_items_transact_item_opts() -> opt_table(). +transact_get_items_transact_item_opts() -> + [expression_attribute_names_opt(), + projection_expression_opt()]. + +-spec dynamize_transact_get_items_transact_item(transact_get_items_transact_item()) -> jsx:json_term(). +dynamize_transact_get_items_transact_item({get, {TableName, Key}}) -> + dynamize_transact_get_items_transact_item({get, {TableName, Key, []}}); +dynamize_transact_get_items_transact_item({get, {TableName, Key, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_get_items_transact_item_opts(), Opts), + [{<<"Get">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]. + +undynamize_transact_get_items_responses(Response, Opts) -> + lists:map(fun(R) -> + Item = proplists:get_value(<<"Item">>, R), + #ddb2_transact_get_items_response{item = undynamize_item(Item, Opts)} + end, Response). + +-spec transact_get_items_record() -> record_desc(). +transact_get_items_record() -> + {#ddb2_transact_get_items{}, + [{<<"ConsumedCapacity">>, #ddb2_transact_get_items.consumed_capacity, fun undynamize_consumed_capacity_list/2}, + {<<"Responses">>, #ddb2_transact_get_items.responses, fun undynamize_transact_get_items_responses/2} + ]}. + +-spec transact_get_items_opts() -> opt_table(). +transact_get_items_opts() -> + [return_consumed_capacity_opt()]. + +-spec transact_get_items(transact_get_items_transact_items()) -> transact_get_items_return(). +transact_get_items(RequestItems) -> + transact_get_items(RequestItems, [], default_config()). + +-spec transact_get_items(transact_get_items_transact_items(), transact_get_items_opts()) -> transact_get_items_return(). +transact_get_items(RequestItems, Opts) -> + transact_get_items(RequestItems, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% DynamoDB API: +%% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactGetItems.html] +%% +%% ===Example=== +%% +%% Put two items in a transaction. +%% +%% ` +%% {ok, Record} = +%% erlcloud_ddb2:transact_get_items( +%% [{get, {<<"PersonalInfo">>, [{<<"Name">>, {s, <<"John Smith">>}}, +%% {<<"DOB">>, {s, <<"11/11/2011">>}}]}}, +%% {get, {<<"EmployeeRecord">>, [{<<"Name">>, {s, <<"John Smith">>}}, +%% {<<"DOH">>, {s, <<"11/11/2018">>}}]}}], +%% [{return_consumed_capacity, total}, +%% {out, record}]), +%% ' +%% @end +%%------------------------------------------------------------------------------ +-spec transact_get_items(transact_get_items_transact_items(), transact_get_items_opts(), aws_config()) -> + transact_get_items_return(). +transact_get_items(TransactItems, Opts, Config) -> + {AwsOpts, DdbOpts} = opts(transact_get_items_opts(), Opts), + Return = erlcloud_ddb_impl:request( + Config, + "DynamoDB_20120810.TransactGetItems", + [{<<"TransactItems">>, dynamize_transact_get_items_transact_items(TransactItems)}] + ++ AwsOpts), + case out(Return, + fun(Json, UOpts) -> undynamize_record(transact_get_items_record(), Json, UOpts) end, DdbOpts) of + {simple, #ddb2_transact_get_items{responses = Responses}} -> + %% Simple return for transact_get_items is all items from all tables in a single list + {ok, lists:map(fun(#ddb2_transact_get_items_response{item = I}) -> I end, Responses)}; + {ok, _} = Out -> Out; + {error, _} = Out -> Out + end. + +%%%------------------------------------------------------------------------------ +%%% TransactWriteItem +%%%------------------------------------------------------------------------------ +-type transact_write_items_opt() :: client_request_token_opt() | + return_consumed_capacity_opt() | + return_item_collection_metrics_opt() | + out_opt(). +-type transact_write_items_opts() :: [transact_write_items_opt()]. + +-type return_value_on_condition_check_failure_opt() :: {return_values_on_condition_check_failure, return_value()}. + +-spec return_value_on_condition_check_failure_opt() -> opt_table_entry(). +return_value_on_condition_check_failure_opt() -> + {return_values_on_condition_check_failure, <<"ReturnValuesOnConditionCheckFailure">>, fun dynamize_return_value/1}. + +-type transact_write_items_transact_item_opt() :: expression_attribute_names_opt() | + expression_attribute_values_opt() | + condition_expression_opt() | + return_value_on_condition_check_failure_opt(). +-type transact_write_items_condition_check_item() :: {table_name(), key(), transact_write_items_transact_item_opt()}. +-type transact_write_items_delete_item() :: {table_name(), key(), transact_write_items_transact_item_opts()}. +-type transact_write_items_put_item() :: {table_name(), in_item(), transact_write_items_transact_item_opts()}. +-type transact_write_items_update_item() :: {table_name(), key(), expression(), transact_write_items_transact_item_opts()}. + +-type transact_write_items_condition_check() :: {condition_check, transact_write_items_condition_check_item()}. +-type transact_write_items_delete() :: {delete, transact_write_items_delete_item()}. +-type transact_write_items_put() :: {put, transact_write_items_put_item()}. +-type transact_write_items_update() :: {update, transact_write_items_update_item()}. + +-type transact_write_items_transact_item() :: transact_write_items_condition_check() | transact_write_items_delete() | transact_write_items_put() | transact_write_items_update(). +-type transact_write_items_transact_items() :: maybe_list(transact_write_items_transact_item()). + +-type transact_write_items_transact_item_opts() :: [transact_write_items_transact_item_opt()]. + +-spec transact_write_items_transact_item_opts() -> opt_table(). +transact_write_items_transact_item_opts() -> + [expression_attribute_names_opt(), + expression_attribute_values_opt(), + condition_expression_opt(), + return_value_on_condition_check_failure_opt()]. + +-spec dynamize_transact_write_items_transact_item(transact_write_items_transact_item()) -> jsx:json_term(). +dynamize_transact_write_items_transact_item({condition_check, {TableName, Key, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), + [{<<"ConditionCheck">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]; +dynamize_transact_write_items_transact_item({delete, {TableName, Key}}) -> + dynamize_transact_write_items_transact_item({delete, {TableName, Key, []}}); +dynamize_transact_write_items_transact_item({delete, {TableName, Key, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), + [{<<"Delete">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]; +dynamize_transact_write_items_transact_item({put, {TableName, Item}}) -> + dynamize_transact_write_items_transact_item({put, {TableName, Item, []}}); +dynamize_transact_write_items_transact_item({put, {TableName, Item, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), + [{<<"Put">>, [{<<"TableName">>, TableName}, {<<"Item">>, dynamize_item(Item)} | AwsOpts]}]; +dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression}}) -> + dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression, []}}); +dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), + [{<<"Update">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)}, {<<"UpdateExpression">>, dynamize_expression(UpdateExpression)} | AwsOpts]}]. + +-spec dynamize_transact_write_items_transact_items(transact_write_items_transact_items()) + -> json_pair(). +dynamize_transact_write_items_transact_items(TransactItems) -> + dynamize_maybe_list(fun dynamize_transact_write_items_transact_item/1, TransactItems). + +-spec transact_write_items_opts() -> opt_table(). +transact_write_items_opts() -> + [client_request_token_opt(), + return_consumed_capacity_opt(), + return_item_collection_metrics_opt()]. + + +-spec transact_write_items_record() -> record_desc(). +transact_write_items_record() -> + {#ddb2_transact_write_items{}, + [{<<"ConsumedCapacity">>, #ddb2_transact_write_items.consumed_capacity, fun undynamize_consumed_capacity_list/2}, + {<<"ItemCollectionMetrics">>, #ddb2_transact_write_items.item_collection_metrics, + fun(V, Opts) -> undynamize_object( + fun({Table, Json}, Opts2) -> + undynamize_item_collection_metric_list(Table, Json, Opts2) + end, V, Opts) + end} + ]}. + +-type transact_write_items_return() :: ddb_return(#ddb2_transact_write_items{}, out_item()). + +-spec transact_write_items(transact_write_items_transact_items()) -> transact_write_items_return(). +transact_write_items(RequestItems) -> + transact_write_items(RequestItems, [], default_config()). + +-spec transact_write_items(transact_write_items_transact_items(), transact_write_items_opts()) -> transact_write_items_return(). +transact_write_items(RequestItems, Opts) -> + transact_write_items(RequestItems, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% DynamoDB API: +%% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html] +%% +%% ===Example=== +%% +%% Put two items in a transaction. +%% +%% ` +%% {ok, Record} = +%% erlcloud_ddb2:transact_write_items( +%% [{put, {<<"PersonalInfo">>, [{<<"Name">>, {s, <<"John Smith">>}}, +%% {<<"DOB">>, {s, <<"11/11/2011">>}}]}}, +%% {put, {<<"EmployeeRecord">>, [{<<"Name">>, {s, <<"John Smith">>}}, +%% {<<"DOH">>, {s, <<"11/11/2018">>}}]}}], +%% [{return_consumed_capacity, total}, +%% {out, record}]), +%% ' +%% @end +%%------------------------------------------------------------------------------ +-spec transact_write_items(transact_write_items_transact_items(), transact_write_items_opts(), aws_config()) -> + transact_write_items_return(). +transact_write_items(TransactItems, Opts, Config) -> + {AwsOpts, DdbOpts} = opts(transact_write_items_opts(), Opts), + Return = erlcloud_ddb_impl:request( + Config, + "DynamoDB_20120810.TransactWriteItems", + [{<<"TransactItems">>, dynamize_transact_write_items_transact_items(TransactItems)}] + ++ AwsOpts), + case out(Return, + fun(Json, UOpts) -> undynamize_record(transact_write_items_record(), Json, UOpts) end, DdbOpts, + #ddb2_transact_write_items.attributes, {ok, []}) of + {simple, Record} -> {ok, Record}; + {ok, _} = Out -> Out; + {error, _} = Out -> Out + end. + + %%%------------------------------------------------------------------------------ %%% CreateBackup %%%------------------------------------------------------------------------------ From a72db6434ca970efeea52ef316c3e44cef3b8bf3 Mon Sep 17 00:00:00 2001 From: hmartel Date: Mon, 29 Apr 2019 09:16:24 -0500 Subject: [PATCH 055/310] Address review comments --- include/erlcloud_ddb2.hrl | 4 +- src/erlcloud_ddb2.erl | 460 ++++++++++++++++++----------------- test/erlcloud_ddb2_tests.erl | 417 ++++++++++++++++++++++++++++++- 3 files changed, 650 insertions(+), 231 deletions(-) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index 3cc3f47b9..d211c1153 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -318,13 +318,13 @@ attributes :: undefined }). --record(ddb2_transact_get_items_response, +-record(ddb2_item_response, {item :: erlcloud_ddb2:out_item() }). -record(ddb2_transact_get_items, {consumed_capacity :: undefined | [#ddb2_consumed_capacity{}], - responses :: undefined | [#ddb2_transact_get_items_response{}] + responses :: undefined | [#ddb2_item_response{}] }). -record(ddb2_backup_description, diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 6a03bb932..ca239eb27 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1522,234 +1522,6 @@ batch_write_item(RequestItems, Opts, Config) -> {error, _} = Out -> Out end. --type transact_get_items_transact_item_opts() :: expression_attribute_names_opt() | - projection_expression_opt(). --type transact_get_items_opts() :: [transact_get_items_transact_item_opts()]. - --type transact_get_items_get_item() :: {table_name(), key(), transact_get_items_transact_item_opts()}. - --type transact_get_items_get() :: {get, transact_get_items_get_item()}. - --type transact_get_items_transact_item() :: transact_get_items_get(). --type transact_get_items_transact_items() :: maybe_list(transact_get_items_transact_item()). - --type transact_get_items_return() :: ddb_return(#ddb2_transact_get_items{}, out_item()). - --spec dynamize_transact_get_items_transact_items(transact_write_items_transact_items()) - -> json_pair(). -dynamize_transact_get_items_transact_items(TransactItems) -> - dynamize_maybe_list(fun dynamize_transact_get_items_transact_item/1, TransactItems). - --spec transact_get_items_transact_item_opts() -> opt_table(). -transact_get_items_transact_item_opts() -> - [expression_attribute_names_opt(), - projection_expression_opt()]. - --spec dynamize_transact_get_items_transact_item(transact_get_items_transact_item()) -> jsx:json_term(). -dynamize_transact_get_items_transact_item({get, {TableName, Key}}) -> - dynamize_transact_get_items_transact_item({get, {TableName, Key, []}}); -dynamize_transact_get_items_transact_item({get, {TableName, Key, Opts}}) -> - {AwsOpts, _DdbOpts} = opts(transact_get_items_transact_item_opts(), Opts), - [{<<"Get">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]. - -undynamize_transact_get_items_responses(Response, Opts) -> - lists:map(fun(R) -> - Item = proplists:get_value(<<"Item">>, R), - #ddb2_transact_get_items_response{item = undynamize_item(Item, Opts)} - end, Response). - --spec transact_get_items_record() -> record_desc(). -transact_get_items_record() -> - {#ddb2_transact_get_items{}, - [{<<"ConsumedCapacity">>, #ddb2_transact_get_items.consumed_capacity, fun undynamize_consumed_capacity_list/2}, - {<<"Responses">>, #ddb2_transact_get_items.responses, fun undynamize_transact_get_items_responses/2} - ]}. - --spec transact_get_items_opts() -> opt_table(). -transact_get_items_opts() -> - [return_consumed_capacity_opt()]. - --spec transact_get_items(transact_get_items_transact_items()) -> transact_get_items_return(). -transact_get_items(RequestItems) -> - transact_get_items(RequestItems, [], default_config()). - --spec transact_get_items(transact_get_items_transact_items(), transact_get_items_opts()) -> transact_get_items_return(). -transact_get_items(RequestItems, Opts) -> - transact_get_items(RequestItems, Opts, default_config()). - -%%------------------------------------------------------------------------------ -%% @doc -%% DynamoDB API: -%% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactGetItems.html] -%% -%% ===Example=== -%% -%% Put two items in a transaction. -%% -%% ` -%% {ok, Record} = -%% erlcloud_ddb2:transact_get_items( -%% [{get, {<<"PersonalInfo">>, [{<<"Name">>, {s, <<"John Smith">>}}, -%% {<<"DOB">>, {s, <<"11/11/2011">>}}]}}, -%% {get, {<<"EmployeeRecord">>, [{<<"Name">>, {s, <<"John Smith">>}}, -%% {<<"DOH">>, {s, <<"11/11/2018">>}}]}}], -%% [{return_consumed_capacity, total}, -%% {out, record}]), -%% ' -%% @end -%%------------------------------------------------------------------------------ --spec transact_get_items(transact_get_items_transact_items(), transact_get_items_opts(), aws_config()) -> - transact_get_items_return(). -transact_get_items(TransactItems, Opts, Config) -> - {AwsOpts, DdbOpts} = opts(transact_get_items_opts(), Opts), - Return = erlcloud_ddb_impl:request( - Config, - "DynamoDB_20120810.TransactGetItems", - [{<<"TransactItems">>, dynamize_transact_get_items_transact_items(TransactItems)}] - ++ AwsOpts), - case out(Return, - fun(Json, UOpts) -> undynamize_record(transact_get_items_record(), Json, UOpts) end, DdbOpts) of - {simple, #ddb2_transact_get_items{responses = Responses}} -> - %% Simple return for transact_get_items is all items from all tables in a single list - {ok, lists:map(fun(#ddb2_transact_get_items_response{item = I}) -> I end, Responses)}; - {ok, _} = Out -> Out; - {error, _} = Out -> Out - end. - -%%%------------------------------------------------------------------------------ -%%% TransactWriteItem -%%%------------------------------------------------------------------------------ --type transact_write_items_opt() :: client_request_token_opt() | - return_consumed_capacity_opt() | - return_item_collection_metrics_opt() | - out_opt(). --type transact_write_items_opts() :: [transact_write_items_opt()]. - --type return_value_on_condition_check_failure_opt() :: {return_values_on_condition_check_failure, return_value()}. - --spec return_value_on_condition_check_failure_opt() -> opt_table_entry(). -return_value_on_condition_check_failure_opt() -> - {return_values_on_condition_check_failure, <<"ReturnValuesOnConditionCheckFailure">>, fun dynamize_return_value/1}. - --type transact_write_items_transact_item_opt() :: expression_attribute_names_opt() | - expression_attribute_values_opt() | - condition_expression_opt() | - return_value_on_condition_check_failure_opt(). --type transact_write_items_condition_check_item() :: {table_name(), key(), transact_write_items_transact_item_opt()}. --type transact_write_items_delete_item() :: {table_name(), key(), transact_write_items_transact_item_opts()}. --type transact_write_items_put_item() :: {table_name(), in_item(), transact_write_items_transact_item_opts()}. --type transact_write_items_update_item() :: {table_name(), key(), expression(), transact_write_items_transact_item_opts()}. - --type transact_write_items_condition_check() :: {condition_check, transact_write_items_condition_check_item()}. --type transact_write_items_delete() :: {delete, transact_write_items_delete_item()}. --type transact_write_items_put() :: {put, transact_write_items_put_item()}. --type transact_write_items_update() :: {update, transact_write_items_update_item()}. - --type transact_write_items_transact_item() :: transact_write_items_condition_check() | transact_write_items_delete() | transact_write_items_put() | transact_write_items_update(). --type transact_write_items_transact_items() :: maybe_list(transact_write_items_transact_item()). - --type transact_write_items_transact_item_opts() :: [transact_write_items_transact_item_opt()]. - --spec transact_write_items_transact_item_opts() -> opt_table(). -transact_write_items_transact_item_opts() -> - [expression_attribute_names_opt(), - expression_attribute_values_opt(), - condition_expression_opt(), - return_value_on_condition_check_failure_opt()]. - --spec dynamize_transact_write_items_transact_item(transact_write_items_transact_item()) -> jsx:json_term(). -dynamize_transact_write_items_transact_item({condition_check, {TableName, Key, Opts}}) -> - {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), - [{<<"ConditionCheck">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]; -dynamize_transact_write_items_transact_item({delete, {TableName, Key}}) -> - dynamize_transact_write_items_transact_item({delete, {TableName, Key, []}}); -dynamize_transact_write_items_transact_item({delete, {TableName, Key, Opts}}) -> - {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), - [{<<"Delete">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]; -dynamize_transact_write_items_transact_item({put, {TableName, Item}}) -> - dynamize_transact_write_items_transact_item({put, {TableName, Item, []}}); -dynamize_transact_write_items_transact_item({put, {TableName, Item, Opts}}) -> - {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), - [{<<"Put">>, [{<<"TableName">>, TableName}, {<<"Item">>, dynamize_item(Item)} | AwsOpts]}]; -dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression}}) -> - dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression, []}}); -dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression, Opts}}) -> - {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), - [{<<"Update">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)}, {<<"UpdateExpression">>, dynamize_expression(UpdateExpression)} | AwsOpts]}]. - --spec dynamize_transact_write_items_transact_items(transact_write_items_transact_items()) - -> json_pair(). -dynamize_transact_write_items_transact_items(TransactItems) -> - dynamize_maybe_list(fun dynamize_transact_write_items_transact_item/1, TransactItems). - --spec transact_write_items_opts() -> opt_table(). -transact_write_items_opts() -> - [client_request_token_opt(), - return_consumed_capacity_opt(), - return_item_collection_metrics_opt()]. - - --spec transact_write_items_record() -> record_desc(). -transact_write_items_record() -> - {#ddb2_transact_write_items{}, - [{<<"ConsumedCapacity">>, #ddb2_transact_write_items.consumed_capacity, fun undynamize_consumed_capacity_list/2}, - {<<"ItemCollectionMetrics">>, #ddb2_transact_write_items.item_collection_metrics, - fun(V, Opts) -> undynamize_object( - fun({Table, Json}, Opts2) -> - undynamize_item_collection_metric_list(Table, Json, Opts2) - end, V, Opts) - end} - ]}. - --type transact_write_items_return() :: ddb_return(#ddb2_transact_write_items{}, out_item()). - --spec transact_write_items(transact_write_items_transact_items()) -> transact_write_items_return(). -transact_write_items(RequestItems) -> - transact_write_items(RequestItems, [], default_config()). - --spec transact_write_items(transact_write_items_transact_items(), transact_write_items_opts()) -> transact_write_items_return(). -transact_write_items(RequestItems, Opts) -> - transact_write_items(RequestItems, Opts, default_config()). - -%%------------------------------------------------------------------------------ -%% @doc -%% DynamoDB API: -%% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html] -%% -%% ===Example=== -%% -%% Put two items in a transaction. -%% -%% ` -%% {ok, Record} = -%% erlcloud_ddb2:transact_write_items( -%% [{put, {<<"PersonalInfo">>, [{<<"Name">>, {s, <<"John Smith">>}}, -%% {<<"DOB">>, {s, <<"11/11/2011">>}}]}}, -%% {put, {<<"EmployeeRecord">>, [{<<"Name">>, {s, <<"John Smith">>}}, -%% {<<"DOH">>, {s, <<"11/11/2018">>}}]}}], -%% [{return_consumed_capacity, total}, -%% {out, record}]), -%% ' -%% @end -%%------------------------------------------------------------------------------ --spec transact_write_items(transact_write_items_transact_items(), transact_write_items_opts(), aws_config()) -> - transact_write_items_return(). -transact_write_items(TransactItems, Opts, Config) -> - {AwsOpts, DdbOpts} = opts(transact_write_items_opts(), Opts), - Return = erlcloud_ddb_impl:request( - Config, - "DynamoDB_20120810.TransactWriteItems", - [{<<"TransactItems">>, dynamize_transact_write_items_transact_items(TransactItems)}] - ++ AwsOpts), - case out(Return, - fun(Json, UOpts) -> undynamize_record(transact_write_items_record(), Json, UOpts) end, DdbOpts, - #ddb2_transact_write_items.attributes, {ok, []}) of - {simple, Record} -> {ok, Record}; - {ok, _} = Out -> Out; - {error, _} = Out -> Out - end. - - %%%------------------------------------------------------------------------------ %%% CreateBackup %%%------------------------------------------------------------------------------ @@ -3377,6 +3149,238 @@ tag_resource(ResourceArn, Tags, Config) -> [{<<"ResourceArn">>, ResourceArn}, {<<"Tags">>, dynamize_tags(Tags)}]). +%%%------------------------------------------------------------------------------ +%%% TransactGetItem +%%%------------------------------------------------------------------------- + +-type transact_get_items_transact_item_opts() :: expression_attribute_names_opt() | + projection_expression_opt(). +-type transact_get_items_opts() :: [transact_get_items_transact_item_opts()]. + +-type transact_get_items_get_item() :: {table_name(), key(), transact_get_items_transact_item_opts()}. + +-type transact_get_items_get() :: {get, transact_get_items_get_item()}. + +-type transact_get_items_transact_item() :: transact_get_items_get(). +-type transact_get_items_transact_items() :: maybe_list(transact_get_items_transact_item()). + +-type transact_get_items_return() :: ddb_return(#ddb2_transact_get_items{}, out_item()). + +-spec dynamize_transact_get_items_transact_items(transact_write_items_transact_items()) + -> json_pair(). +dynamize_transact_get_items_transact_items(TransactItems) -> + dynamize_maybe_list(fun dynamize_transact_get_items_transact_item/1, TransactItems). + +-spec transact_get_items_transact_item_opts() -> opt_table(). +transact_get_items_transact_item_opts() -> + [expression_attribute_names_opt(), + projection_expression_opt()]. + +-spec dynamize_transact_get_items_transact_item(transact_get_items_transact_item()) -> jsx:json_term(). +dynamize_transact_get_items_transact_item({get, {TableName, Key}}) -> + dynamize_transact_get_items_transact_item({get, {TableName, Key, []}}); +dynamize_transact_get_items_transact_item({get, {TableName, Key, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_get_items_transact_item_opts(), Opts), + [{<<"Get">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]. + +undynamize_transact_get_items_responses(Response, Opts) -> + lists:map(fun(R) -> + Item = proplists:get_value(<<"Item">>, R), + #ddb2_item_response{item = undynamize_item(Item, Opts)} + end, Response). + +-spec transact_get_items_record() -> record_desc(). +transact_get_items_record() -> + {#ddb2_transact_get_items{}, + [{<<"ConsumedCapacity">>, #ddb2_transact_get_items.consumed_capacity, fun undynamize_consumed_capacity_list/2}, + {<<"Responses">>, #ddb2_transact_get_items.responses, fun undynamize_transact_get_items_responses/2} + ]}. + +-spec transact_get_items_opts() -> opt_table(). +transact_get_items_opts() -> + [return_consumed_capacity_opt()]. + +-spec transact_get_items(transact_get_items_transact_items()) -> transact_get_items_return(). +transact_get_items(RequestItems) -> + transact_get_items(RequestItems, [], default_config()). + +-spec transact_get_items(transact_get_items_transact_items(), transact_get_items_opts()) -> transact_get_items_return(). +transact_get_items(RequestItems, Opts) -> + transact_get_items(RequestItems, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% DynamoDB API: +%% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactGetItems.html] +%% +%% ===Example=== +%% +%% Get two items in a transaction. +%% +%% ` +%% {ok, Record} = +%% erlcloud_ddb2:transact_get_items( +%% [{get, {<<"PersonalInfo">>, [{<<"Name">>, {s, <<"John Smith">>}}, +%% {<<"DOB">>, {s, <<"11/11/2011">>}}]}}, +%% {get, {<<"EmployeeRecord">>, [{<<"Name">>, {s, <<"John Smith">>}}, +%% {<<"DOH">>, {s, <<"11/11/2018">>}}]}}], +%% [{return_consumed_capacity, total}, +%% {out, record}]), +%% ' +%% @end +%%------------------------------------------------------------------------------ +-spec transact_get_items(transact_get_items_transact_items(), transact_get_items_opts(), aws_config()) -> + transact_get_items_return(). +transact_get_items(TransactItems, Opts, Config) -> + {AwsOpts, DdbOpts} = opts(transact_get_items_opts(), Opts), + Return = erlcloud_ddb_impl:request( + Config, + "DynamoDB_20120810.TransactGetItems", + [{<<"TransactItems">>, dynamize_transact_get_items_transact_items(TransactItems)}] + ++ AwsOpts), + case out(Return, + fun(Json, UOpts) -> undynamize_record(transact_get_items_record(), Json, UOpts) end, DdbOpts) of + {simple, #ddb2_transact_get_items{responses = Responses}} -> + %% Simple return for transact_get_items is all items from all tables in a single list + {ok, lists:map(fun(#ddb2_item_response{item = I}) -> I end, Responses)}; + {ok, _} = Out -> Out; + {error, _} = Out -> Out + end. + +%%%------------------------------------------------------------------------------ +%%% TransactWriteItem +%%%------------------------------------------------------------------------------ + +-type transact_write_items_opt() :: client_request_token_opt() | + return_consumed_capacity_opt() | + return_item_collection_metrics_opt() | + out_opt(). +-type transact_write_items_opts() :: [transact_write_items_opt()]. + +-type return_value_on_condition_check_failure_opt() :: {return_values_on_condition_check_failure, return_value()}. + +-spec return_value_on_condition_check_failure_opt() -> opt_table_entry(). +return_value_on_condition_check_failure_opt() -> + {return_values_on_condition_check_failure, <<"ReturnValuesOnConditionCheckFailure">>, fun dynamize_return_value/1}. + +-type transact_write_items_transact_item_opt() :: expression_attribute_names_opt() | + expression_attribute_values_opt() | + condition_expression_opt() | + return_value_on_condition_check_failure_opt(). +-type transact_write_items_condition_check_item() :: {table_name(), key(), transact_write_items_transact_item_opt()}. +-type transact_write_items_delete_item() :: {table_name(), key(), transact_write_items_transact_item_opts()}. +-type transact_write_items_put_item() :: {table_name(), in_item(), transact_write_items_transact_item_opts()}. +-type transact_write_items_update_item() :: {table_name(), key(), expression(), transact_write_items_transact_item_opts()}. + +-type transact_write_items_condition_check() :: {condition_check, transact_write_items_condition_check_item()}. +-type transact_write_items_delete() :: {delete, transact_write_items_delete_item()}. +-type transact_write_items_put() :: {put, transact_write_items_put_item()}. +-type transact_write_items_update() :: {update, transact_write_items_update_item()}. + +-type transact_write_items_transact_item() :: transact_write_items_condition_check() | transact_write_items_delete() | transact_write_items_put() | transact_write_items_update(). +-type transact_write_items_transact_items() :: maybe_list(transact_write_items_transact_item()). + +-type transact_write_items_transact_item_opts() :: [transact_write_items_transact_item_opt()]. + +-spec transact_write_items_transact_item_opts() -> opt_table(). +transact_write_items_transact_item_opts() -> + [expression_attribute_names_opt(), + expression_attribute_values_opt(), + condition_expression_opt(), + return_value_on_condition_check_failure_opt()]. + +-spec dynamize_transact_write_items_transact_item(transact_write_items_transact_item()) -> jsx:json_term(). +dynamize_transact_write_items_transact_item({condition_check, {TableName, Key, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), + [{<<"ConditionCheck">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]; +dynamize_transact_write_items_transact_item({delete, {TableName, Key}}) -> + dynamize_transact_write_items_transact_item({delete, {TableName, Key, []}}); +dynamize_transact_write_items_transact_item({delete, {TableName, Key, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), + [{<<"Delete">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)} | AwsOpts]}]; +dynamize_transact_write_items_transact_item({put, {TableName, Item}}) -> + dynamize_transact_write_items_transact_item({put, {TableName, Item, []}}); +dynamize_transact_write_items_transact_item({put, {TableName, Item, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), + [{<<"Put">>, [{<<"TableName">>, TableName}, {<<"Item">>, dynamize_item(Item)} | AwsOpts]}]; +dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression}}) -> + dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression, []}}); +dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpression, Opts}}) -> + {AwsOpts, _DdbOpts} = opts(transact_write_items_transact_item_opts(), Opts), + [{<<"Update">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)}, {<<"UpdateExpression">>, dynamize_expression(UpdateExpression)} | AwsOpts]}]. + +-spec dynamize_transact_write_items_transact_items(transact_write_items_transact_items()) + -> json_pair(). +dynamize_transact_write_items_transact_items(TransactItems) -> + dynamize_maybe_list(fun dynamize_transact_write_items_transact_item/1, TransactItems). + +-spec transact_write_items_opts() -> opt_table(). +transact_write_items_opts() -> + [client_request_token_opt(), + return_consumed_capacity_opt(), + return_item_collection_metrics_opt()]. + + +-spec transact_write_items_record() -> record_desc(). +transact_write_items_record() -> + {#ddb2_transact_write_items{}, + [{<<"ConsumedCapacity">>, #ddb2_transact_write_items.consumed_capacity, fun undynamize_consumed_capacity_list/2}, + {<<"ItemCollectionMetrics">>, #ddb2_transact_write_items.item_collection_metrics, + fun(V, Opts) -> undynamize_object( + fun({Table, Json}, Opts2) -> + undynamize_item_collection_metric_list(Table, Json, Opts2) + end, V, Opts) + end} + ]}. + +-type transact_write_items_return() :: ddb_return(#ddb2_transact_write_items{}, out_item()). + +-spec transact_write_items(transact_write_items_transact_items()) -> transact_write_items_return(). +transact_write_items(RequestItems) -> + transact_write_items(RequestItems, [], default_config()). + +-spec transact_write_items(transact_write_items_transact_items(), transact_write_items_opts()) -> transact_write_items_return(). +transact_write_items(RequestItems, Opts) -> + transact_write_items(RequestItems, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% DynamoDB API: +%% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html] +%% +%% ===Example=== +%% +%% Put two items in a transaction. +%% +%% ` +%% {ok, Record} = +%% erlcloud_ddb2:transact_write_items( +%% [{put, {<<"PersonalInfo">>, [{<<"Name">>, {s, <<"John Smith">>}}, +%% {<<"DOB">>, {s, <<"11/11/2011">>}}]}}, +%% {put, {<<"EmployeeRecord">>, [{<<"Name">>, {s, <<"John Smith">>}}, +%% {<<"DOH">>, {s, <<"11/11/2018">>}}]}}], +%% [{return_consumed_capacity, total}, +%% {out, record}]), +%% ' +%% @end +%%------------------------------------------------------------------------------ +-spec transact_write_items(transact_write_items_transact_items(), transact_write_items_opts(), aws_config()) -> + transact_write_items_return(). +transact_write_items(TransactItems, Opts, Config) -> + {AwsOpts, DdbOpts} = opts(transact_write_items_opts(), Opts), + Return = erlcloud_ddb_impl:request( + Config, + "DynamoDB_20120810.TransactWriteItems", + [{<<"TransactItems">>, dynamize_transact_write_items_transact_items(TransactItems)}] + ++ AwsOpts), + case out(Return, + fun(Json, UOpts) -> undynamize_record(transact_write_items_record(), Json, UOpts) end, DdbOpts, + #ddb2_transact_write_items.attributes, {ok, []}) of + {simple, Record} -> {ok, Record}; + {ok, _} = Out -> Out; + {error, _} = Out -> Out + end. + %%%------------------------------------------------------------------------------ %%% UntagResource %%%------------------------------------------------------------------------------ diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index a22352aeb..81fcb4369 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -67,7 +67,7 @@ operation_test_() -> fun list_tables_input_tests/1, fun list_tables_output_tests/1, fun list_tags_of_resource_input_tests/1, - fun list_tags_of_resource_output_tests/1, + fun list_tags_of_resource_output_tests/1, fun put_item_input_tests/1, fun put_item_output_tests/1, fun q_input_tests/1, @@ -80,6 +80,10 @@ operation_test_() -> fun scan_output_tests/1, fun tag_resource_input_tests/1, fun tag_resource_output_tests/1, + fun transact_get_items_input_tests/1, + fun transact_get_items_output_tests/1, + fun transact_write_items_input_tests/1, + fun transact_write_items_output_tests/1, fun untag_resource_input_tests/1, fun untag_resource_output_tests/1, fun update_continuous_backups_input_tests/1, @@ -4767,6 +4771,417 @@ tag_resource_output_tests(_) -> [{<<"example_key1">>, <<"example_value1">>}, {<<"example_key2">>, <<"example_value2">>}])), Tests). + +%% TransactGetItem test using synthetic data (there are no AWS provided examples): +%% https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactGetItems.html +transact_get_items_input_tests(_) -> + Tests = + [?_ddb_test( + {"TransactGetItems request", + ?_f(erlcloud_ddb2:transact_get_items( + [{get, {<<"PersonalInfo">>, [{<<"Name">>, {s, <<"John Smith">>}}, + {<<"DOB">>, {s, <<"11/11/2011">>}}]}}, + {get, {<<"EmployeeRecord">>, [{<<"Name">>, {s, <<"John Smith">>}}, + {<<"DOH">>, {s, <<"11/11/2018">>}}]}}], + [{return_consumed_capacity, total}])), " +{ + \"TransactItems\": [ + {\"Get\": { + \"Key\": { + \"Name\": { + \"S\": \"John Smith\" + }, + \"DOB\": { + \"S\": \"11/11/2011\" + } + }, + \"TableName\": \"PersonalInfo\" + }}, + {\"Get\": { + \"Key\": { + \"Name\": { + \"S\": \"John Smith\" + }, + \"DOH\": { + \"S\": \"11/11/2018\" + } + }, + \"TableName\": \"EmployeeRecord\" + }} + ], + \"ReturnConsumedCapacity\": \"TOTAL\" +}" + }) + ], + + Response = " +{ + \"ConsumedCapacity\": [ + { + \"TableName\": \"PersonalInfo\", + \"CapacityUnits\": 2 + }, + { + \"TableName\": \"EmployeeRecord\", + \"CapacityUnits\": 3 + } + ], + \"Responses\": [ + {\"Item\": { + \"Name\":{ + \"S\":\"John Smith\" + }, + \"DOB\":{ + \"S\":\"11/11/2011\" + }, + \"Creation_ts\":{ + \"N\":\"1500000000\" + } + }}, + {\"Item\": { + \"Name\":{ + \"S\":\"John Smith\" + }, + \"DOH\":{ + \"S\":\"11/11/2018\" + }, + \"Id\":{ + \"N\":\"19\" + } + }} + ] +}", + input_tests(Response, Tests). + +transact_get_items_output_tests(_) -> + Tests = + [?_ddb_test( + {"TransactGetItems response only", " +{ + \"Responses\": [ + {\"Item\": { + \"Name\":{ + \"S\":\"John Smith\" + }, + \"DOB\":{ + \"S\":\"11/11/2011\" + }, + \"Creation_ts\":{ + \"N\":\"1500000000\" + } + }}, + {\"Item\": { + \"Name\":{ + \"S\":\"John Smith\" + }, + \"DOH\":{ + \"S\":\"11/11/2018\" + }, + \"Id\":{ + \"N\":\"19\" + } + }} + ] +}", + {ok, #ddb2_transact_get_items{ + responses = [ + #ddb2_item_response{ + item = [ + {<<"Name">>, <<"John Smith">>}, + {<<"DOB">>, <<"11/11/2011">>}, + {<<"Creation_ts">>, 1500000000} + ] + }, + #ddb2_item_response{ + item = [ + {<<"Name">>, <<"John Smith">>}, + {<<"DOH">>, <<"11/11/2018">>}, + {<<"Id">>, 19} + ] + } + ] + }}}), + ?_ddb_test( + {"TransactGetItems consumed capacity", " +{ + \"ConsumedCapacity\": [ + { + \"TableName\": \"PersonalInfo\", + \"CapacityUnits\": 2 + }, + { + \"TableName\": \"EmployeeRecord\", + \"CapacityUnits\": 3 + } + ], + \"Responses\": [ + {\"Item\": { + \"Name\":{ + \"S\":\"John Smith\" + }, + \"DOB\":{ + \"S\":\"11/11/2011\" + }, + \"Creation_ts\":{ + \"N\":\"1500000000\" + } + }}, + {\"Item\": { + \"Name\":{ + \"S\":\"John Smith\" + }, + \"DOH\":{ + \"S\":\"11/11/2018\" + }, + \"Id\":{ + \"N\":\"19\" + } + }} + ] +}", + {ok, #ddb2_transact_get_items{ + responses = [ + #ddb2_item_response{ + item = [ + {<<"Name">>, <<"John Smith">>}, + {<<"DOB">>, <<"11/11/2011">>}, + {<<"Creation_ts">>, 1500000000} + ] + }, + #ddb2_item_response{ + item = [ + {<<"Name">>, <<"John Smith">>}, + {<<"DOH">>, <<"11/11/2018">>}, + {<<"Id">>, 19} + ] + } + ], + consumed_capacity = [ + #ddb2_consumed_capacity{ + capacity_units = 2, table_name = <<"PersonalInfo">> + }, + #ddb2_consumed_capacity{ + capacity_units = 3, table_name = <<"EmployeeRecord">> + } + ] + }}}) + ], + + output_tests(?_f(erlcloud_ddb2:transact_get_items([], [{out, record}])), + Tests). + +%% TransactWriteItems test using synthetic data (there are no AWS provided examples): +%% https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html +transact_write_items_input_tests(_) -> + Tests = + [?_ddb_test( + {"TransactWriteItems request", + ?_f(erlcloud_ddb2:transact_write_items( + [{update, { + <<"PersonalInfo">>, + [{<<"Name">>, {s, <<"John Smith">>}}, {<<"DOB">>, {s, <<"11/11/2011">>}}], + <<"SET eye_color = :c">>, + [{expression_attribute_values, [{<<":c">>, <<"brown">>}]}]}}, + {condition_check, { + <<"Scratchpad">>, + [{<<"Name">>, {s, <<"John Smith">>}}, {<<"DOB">>, {s, <<"11/11/2011">>}}], + [{condition_expression, <<"approved = :a">>}, + {expression_attribute_values, [{<<":a">>, <<"yes">>}]}]}}, + {delete, {<<"Scratchpad">>, [{<<"Name">>, {s, <<"John Smith">>}}, + {<<"DOB">>, {s, <<"11/11/2011">>}}]}}, + {put, {<<"EmployeeRecord">>, [{<<"Name">>, {s, <<"John Smith">>}}, + {<<"DOH">>, {s, <<"11/11/2018">>}}]}}], + [{return_consumed_capacity, total}, + {return_item_collection_metrics, size}])), " +{ + \"TransactItems\": [ + {\"Update\": { + \"Key\": { + \"Name\": { + \"S\": \"John Smith\" + }, + \"DOB\": { + \"S\": \"11/11/2011\" + } + }, + \"ExpressionAttributeValues\": { + \":c\": {\"S\": \"brown\"} + }, + \"UpdateExpression\": \"SET eye_color = :c\", + \"TableName\": \"PersonalInfo\" + }}, + {\"ConditionCheck\": { + \"Key\": { + \"Name\": { + \"S\": \"John Smith\" + }, + \"DOB\": { + \"S\": \"11/11/2011\" + } + }, + \"ExpressionAttributeValues\": { + \":a\": {\"S\": \"yes\"} + }, + \"ConditionExpression\": \"approved = :a\", + \"TableName\": \"Scratchpad\" + }}, + {\"Delete\": { + \"Key\": { + \"Name\": { + \"S\": \"John Smith\" + }, + \"DOB\": { + \"S\": \"11/11/2011\" + } + }, + \"TableName\": \"Scratchpad\" + }}, + {\"Put\": { + \"Item\": { + \"Name\": { + \"S\": \"John Smith\" + }, + \"DOH\": { + \"S\": \"11/11/2018\" + } + }, + \"TableName\": \"EmployeeRecord\" + }} + ], + \"ReturnConsumedCapacity\": \"TOTAL\", + \"ReturnItemCollectionMetrics\": \"SIZE\" +}" + }) + ], + + Response = " +{ + \"ItemCollectionMetrics\": { + \"PersonalInfo\": [{ + \"ItemCollectionKey\": { + \"Name\": { + \"S\": \"John Smith\" + } + }, + \"SizeEstimateRangeGB\": [1, 2] + }], + \"Scratchpad\": [{ + \"ItemCollectionKey\": { + \"Name\": { + \"S\": \"John Smith\" + } + }, + \"SizeEstimateRangeGB\": [0, 1] + }], + \"EmployeeRecord\": [{ + \"ItemCollectionKey\": { + \"Name\": { + \"S\": \"John Smith\" + } + }, + \"SizeEstimateRangeGB\": [2, 3] + }] + }, + \"ConsumedCapacity\": [ + { + \"TableName\": \"PersonalInfo\", + \"CapacityUnits\": 3 + }, + { + \"TableName\": \"Scratchpad\", + \"CapacityUnits\": 4 + }, + { + \"TableName\": \"EmployeeRecord\", + \"CapacityUnits\": 3 + } + ] +}", + input_tests(Response, Tests). + +transact_write_items_output_tests(_) -> + Tests = + [?_ddb_test( + {"TransactWriteItems consumed capacity", " +{ + \"ConsumedCapacity\": [ + { + \"TableName\": \"PersonalInfo\", + \"CapacityUnits\": 3 + }, + { + \"TableName\": \"Scratchpad\", + \"CapacityUnits\": 4 + }, + { + \"TableName\": \"EmployeeRecord\", + \"CapacityUnits\": 3 + } + ] +}", + {ok, #ddb2_transact_write_items{ + consumed_capacity = [ + #ddb2_consumed_capacity{ + capacity_units = 3, table_name = <<"PersonalInfo">> + }, + #ddb2_consumed_capacity{ + capacity_units = 4, table_name = <<"Scratchpad">> + }, + #ddb2_consumed_capacity{ + capacity_units = 3, table_name = <<"EmployeeRecord">> + } + ] + }}}), + ?_ddb_test( + {"TransactWriteItems item collection metrics", " +{ + \"ItemCollectionMetrics\": { + \"PersonalInfo\": [{ + \"ItemCollectionKey\": { + \"Name\": { + \"S\": \"John Smith\" + } + }, + \"SizeEstimateRangeGB\": [1, 2] + }], + \"Scratchpad\": [{ + \"ItemCollectionKey\": { + \"Name\": { + \"S\": \"John Smith\" + } + }, + \"SizeEstimateRangeGB\": [0, 1] + }], + \"EmployeeRecord\": [{ + \"ItemCollectionKey\": { + \"Name\": { + \"S\": \"John Smith\" + } + }, + \"SizeEstimateRangeGB\": [2, 3] + }] + } +}", + {ok, #ddb2_transact_write_items{ + item_collection_metrics = [ + {<<"PersonalInfo">>, + [#ddb2_item_collection_metrics + {item_collection_key = <<"John Smith">>, + size_estimate_range_gb = {1, 2}}]}, + {<<"Scratchpad">>, + [#ddb2_item_collection_metrics + {item_collection_key = <<"John Smith">>, + size_estimate_range_gb = {0, 1}}]}, + {<<"EmployeeRecord">>, + [#ddb2_item_collection_metrics + {item_collection_key = <<"John Smith">>, + size_estimate_range_gb = {2, 3}} + ]}]}}}) + ], + + output_tests(?_f(erlcloud_ddb2:transact_write_items([], [{out, record}])), + Tests). + %% UntagResource test based on the API: %% https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UntagResource.html untag_resource_input_tests(_) -> From 704bade75deb1e746cb4868d16b9f36572c91696 Mon Sep 17 00:00:00 2001 From: Evgeny Bob Date: Wed, 15 May 2019 09:53:34 +0100 Subject: [PATCH 056/310] address missing comment --- src/erlcloud_ddb2.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index ca239eb27..34e547600 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -3149,9 +3149,9 @@ tag_resource(ResourceArn, Tags, Config) -> [{<<"ResourceArn">>, ResourceArn}, {<<"Tags">>, dynamize_tags(Tags)}]). -%%%------------------------------------------------------------------------------ -%%% TransactGetItem -%%%------------------------------------------------------------------------- +%%%----------------------------------------------------------------------------- +%%% TransactGetItems +%%%----------------------------------------------------------------------------- -type transact_get_items_transact_item_opts() :: expression_attribute_names_opt() | projection_expression_opt(). From cd04fdb33ac8b17922a063e63b0279bca7b581a9 Mon Sep 17 00:00:00 2001 From: hmartel Date: Sat, 29 Jun 2019 09:30:30 -0500 Subject: [PATCH 057/310] Add erlclloud_kinesis:list_shards/2|3 --- src/erlcloud_kinesis.erl | 123 +++++++++++++++++++++++++++++- test/erlcloud_kinesis_tests.erl | 128 ++++++++++++++++++++++++++++++++ 2 files changed, 247 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index c6c83199b..80c89e3ef 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -7,6 +7,7 @@ -export([create_stream/2, create_stream/3, delete_stream/1, delete_stream/2, + list_shards/1, list_shards/2, list_shards/3, list_streams/0, list_streams/1, list_streams/2, list_streams/3, describe_stream/1, describe_stream/2, describe_stream/3, describe_stream/4, describe_stream_summary/1, describe_stream_summary/2, @@ -30,6 +31,11 @@ -type get_records_limit() :: 1..10000. +-type exclusive_start_shard_id_opt() :: {exclusive_start_shard_id, string()}. +-type max_results_opt() :: {max_results, non_neg_integer()}. +-type next_token_opt() :: {next_token, string()}. +-type stream_creation_timestamp_opt() :: {stream_creation_timestamp, integer()}. + -spec new(string(), string()) -> aws_config(). new(AccessKeyID, SecretAccessKey) -> @@ -72,6 +78,20 @@ configure(AccessKeyID, SecretAccessKey, Host, Port) -> default_config() -> erlcloud_aws:default_config(). +dynamize_option({exclusive_start_shard_id, Value}) -> + {<<"ExclusiveStartShardId">>, Value}; +dynamize_option({max_results, Value}) when Value >= 1, Value =< 10000 -> + {<<"MaxResults">>, Value}; +dynamize_option({next_token, Value}) -> + {<<"NextToken">>, Value}; +dynamize_option({stream_creation_timestamp, Value}) -> + {<<"StreamCreationTimestamp">>, Value}; +dynamize_option(Option) -> + error({erlcloud_kinesis, {invalid_option, Option}}). + +dynamize_options(Options) -> + [dynamize_option(Option) || Option <- Options]. + %%------------------------------------------------------------------------------ %% @doc %% Kinesis API: @@ -130,6 +150,101 @@ delete_stream(StreamName, Config) when is_record(Config, aws_config) -> Json = [{<<"StreamName">>, StreamName}], erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.DeleteStream", Json). + +-type list_shards_opts() :: [ + exclusive_start_shard_id_opt() | + max_results_opt() | + next_token_opt() | + stream_creation_timestamp_opt() +]. + +%%------------------------------------------------------------------------------ +%% @doc +%% Kinesis API: +%% [https://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html] +%% +%% ===Example=== +%% +%% This operation returns the following information about the stream: an array of shard objects that comprise the stream. +%% +%% ` +%% erlcloud_kinesis:list_shards(<<"staging">>). +%% {ok, [ +%% {<<"NextToken">>, <<"AAAAAAAAAAGK9EEG0sJqVhCUS2JsgigQ5dcpB4q9PYswrH2oK44Skbjtm+WR0xA7/hrAFFsohevH1/OyPnbzKBS1byPyCZuVcokYtQe/b1m4c0SCI7jctPT0oUTLRdwSRirKm9dp9YC/EL+kZHOvYAUnztVGsOAPEFC3ECf/bVC927bDZBbRRzy/44OHfWmrCLcbcWqehRh5D14WnL3yLsumhiHDkyuxSlkBepauvMnNLtTOlRtmQ5Q5reoujfq2gzeCSOtLcfXgBMztJqohPdgMzjTQSbwB9Am8rMpHLsDbSdMNXmITvw==">>}, +%% {<<"Shards">>, [ +%% [ +%% {<<"ShardId">>, <<"shardId-000000000001">>}, +%% {<<"HashKeyRange">>, [ +%% {<<"EndingHashKey">>, <<"68056473384187692692674921486353642280">>}, +%% {<<"StartingHashKey">>, <<"34028236692093846346337460743176821145">>} +%% ]}, +%% {<<"SequenceNumberRange">>, [ +%% {<<"StartingSequenceNumber">>, <<"49579844037727333356165064238440708846556371693205002258">>} +%% ]} +%% ], [ +%% {<<"ShardId">>, <<"shardId-000000000002">>}, +%% {<<"HashKeyRange">>, [ +%% {<<"EndingHashKey">>, <<"102084710076281539039012382229530463436">>}, +%% {<<"StartingHashKey">>, <<"68056473384187692692674921486353642281">>} +%% ]}, +%% {<<"SequenceNumberRange">>, [ +%% {<<"StartingSequenceNumber">>, <<"49579844037749634101363594861582244564829020124710982690">>} +%% ]} +%% ], [ +%% {<<"ShardId">>, <<"shardId-000000000003">>}, +%% {<<"HashKeyRange">>, [ +%% {<<"EndingHashKey">>, <<"136112946768375385385349842972707284581">>}, +%% {<<"StartingHashKey">>, <<"102084710076281539039012382229530463437">>} +%% ]}, +%% {<<"SequenceNumberRange">>, [ +%% {<<"StartingSequenceNumber">>, <<"49579844037771934846562125484723780283101668556216963122">>} +%% ]} +%% ] +%% ]} +%% ]} +%% ' +%% +%% @end +%%------------------------------------------------------------------------------ + +-spec list_shards(binary()) -> erlcloud_kinesis_impl:json_return(). +list_shards(StreamName) -> + list_shards(StreamName, default_config()). + +-spec list_shards(binary(), list_shards_opts() | aws_config()) -> erlcloud_kinesis_impl:json_return(). +list_shards(StreamName, Config) when is_record(Config, aws_config) -> + list_shards(StreamName, [], Config); +list_shards(StreamName, Options) -> + list_shards(StreamName, Options, default_config()). + +-spec list_shards(binary(), list_shards_opts(), aws_config()) -> erlcloud_kinesis_impl:json_return(). +list_shards(StreamName, Options, Config) when is_record(Config, aws_config) -> + case lists:keymember(next_token, 1, Options) of + false -> + ok; + true -> + case lists:keymember(stream_creation_timestamp, 1, Options) of + false -> + ok; + true -> + error({erlcloud_kinesis, {incompatible_options, [ + next_token, + stream_creation_timestamp + ]}}) + end, + case lists:keymember(exclusive_start_shard_id, 1, Options) of + false -> + ok; + true -> + error({erlcloud_kinesis, {incompatible_options, [ + next_token, + exclusive_start_shard_id + ]}}) + end + end, + Json = [{<<"StreamName">>, StreamName} | dynamize_options(Options)], + erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", Json). + %%------------------------------------------------------------------------------ %% @doc %% Kinesis API: @@ -234,15 +349,15 @@ describe_stream(StreamName, Limit, Config) Limit >= 1, Limit =< 10000 -> Json = [{<<"StreamName">>, StreamName}, {<<"Limit">>, Limit}], erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.DescribeStream", Json); -describe_stream(StreamName, Limit, ExcludeShard) -> - describe_stream(StreamName, Limit, ExcludeShard, default_config()). +describe_stream(StreamName, Limit, ExclusiveStartShardId) -> + describe_stream(StreamName, Limit, ExclusiveStartShardId, default_config()). -spec describe_stream(binary(), get_records_limit(), string(), aws_config()) -> erlcloud_kinesis_impl:json_return(). -describe_stream(StreamName, Limit, ExcludeShard, Config) +describe_stream(StreamName, Limit, ExclusiveStartShardId, Config) when is_record(Config, aws_config), is_integer(Limit), Limit >= 1, Limit =< 10000 -> - Json = [{<<"StreamName">>, StreamName}, {<<"Limit">>, Limit}, {<<"ExclusiveStartShardId">>, ExcludeShard}], + Json = [{<<"StreamName">>, StreamName}, {<<"Limit">>, Limit}, {<<"ExclusiveStartShardId">>, ExclusiveStartShardId}], erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.DescribeStream", Json). diff --git a/test/erlcloud_kinesis_tests.erl b/test/erlcloud_kinesis_tests.erl index 3e45dd14e..dbed58eb3 100644 --- a/test/erlcloud_kinesis_tests.erl +++ b/test/erlcloud_kinesis_tests.erl @@ -33,6 +33,8 @@ operation_test_() -> fun create_stream_output_tests/1, fun delete_stream_input_tests/1, fun delete_stream_output_tests/1, + fun list_shards_input_tests/1, + fun list_shards_output_tests/1, fun list_streams_input_tests/1, fun list_streams_output_tests/1, fun describe_stream_input_tests/1, @@ -223,6 +225,132 @@ delete_stream_output_tests(_) -> output_tests(?_f(erlcloud_kinesis:delete_stream(<<"streamName">>)), Tests). +%% ListStreams test based on the API examples: +%% http://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListShards.html +list_shards_input_tests(_) -> + Tests = + [?_kinesis_test( + {"ListShards example request", + ?_f(erlcloud_kinesis:list_shards(<<"exampleStreamName">>, [{max_results, 3}])), "{ + \"StreamName\" : \"exampleStreamName\", + \"MaxResults\" : 3 + }" + }) + ], + + Response = "{ + \"NextToken\": \"AAAAAAAAAAGK9EEG0sJqVhCUS2JsgigQ5dcpB4q9PYswrH2oK44Skbjtm+WR0xA7/hrAFFsohevH1/OyPnbzKBS1byPyCZuVcokYtQe/b1m4c0SCI7jctPT0oUTLRdwSRirKm9dp9YC/EL+kZHOvYAUnztVGsOAPEFC3ECf/bVC927bDZBbRRzy/44OHfWmrCLcbcWqehRh5D14WnL3yLsumhiHDkyuxSlkBepauvMnNLtTOlRtmQ5Q5reoujfq2gzeCSOtLcfXgBMztJqohPdgMzjTQSbwB9Am8rMpHLsDbSdMNXmITvw==\", + \"Shards\": [ + { + \"ShardId\": \"shardId-000000000001\", + \"HashKeyRange\": { + \"EndingHashKey\": \"68056473384187692692674921486353642280\", + \"StartingHashKey\": \"34028236692093846346337460743176821145\" + }, + \"SequenceNumberRange\": { + \"StartingSequenceNumber\": \"49579844037727333356165064238440708846556371693205002258\" + } + }, + { + \"ShardId\": \"shardId-000000000002\", + \"HashKeyRange\": { + \"EndingHashKey\": \"102084710076281539039012382229530463436\", + \"StartingHashKey\": \"68056473384187692692674921486353642281\" + }, + \"SequenceNumberRange\": { + \"StartingSequenceNumber\": \"49579844037749634101363594861582244564829020124710982690\" + } + }, + { + \"ShardId\": \"shardId-000000000003\", + \"HashKeyRange\": { + \"EndingHashKey\": \"136112946768375385385349842972707284581\", + \"StartingHashKey\": \"102084710076281539039012382229530463437\" + }, + \"SequenceNumberRange\": { + \"StartingSequenceNumber\": \"49579844037771934846562125484723780283101668556216963122\" + } + } + ] + }", + input_tests(Response, Tests). + +list_shards_output_tests(_) -> + Tests = + [?_kinesis_test( + {"ListShards example response", "{ + \"NextToken\": \"AAAAAAAAAAGK9EEG0sJqVhCUS2JsgigQ5dcpB4q9PYswrH2oK44Skbjtm+WR0xA7/hrAFFsohevH1/OyPnbzKBS1byPyCZuVcokYtQe/b1m4c0SCI7jctPT0oUTLRdwSRirKm9dp9YC/EL+kZHOvYAUnztVGsOAPEFC3ECf/bVC927bDZBbRRzy/44OHfWmrCLcbcWqehRh5D14WnL3yLsumhiHDkyuxSlkBepauvMnNLtTOlRtmQ5Q5reoujfq2gzeCSOtLcfXgBMztJqohPdgMzjTQSbwB9Am8rMpHLsDbSdMNXmITvw==\", + \"Shards\": [ + { + \"ShardId\": \"shardId-000000000001\", + \"HashKeyRange\": { + \"EndingHashKey\": \"68056473384187692692674921486353642280\", + \"StartingHashKey\": \"34028236692093846346337460743176821145\" + }, + \"SequenceNumberRange\": { + \"StartingSequenceNumber\": \"49579844037727333356165064238440708846556371693205002258\" + } + }, + { + \"ShardId\": \"shardId-000000000002\", + \"HashKeyRange\": { + \"EndingHashKey\": \"102084710076281539039012382229530463436\", + \"StartingHashKey\": \"68056473384187692692674921486353642281\" + }, + \"SequenceNumberRange\": { + \"StartingSequenceNumber\": \"49579844037749634101363594861582244564829020124710982690\" + } + }, + { + \"ShardId\": \"shardId-000000000003\", + \"HashKeyRange\": { + \"EndingHashKey\": \"136112946768375385385349842972707284581\", + \"StartingHashKey\": \"102084710076281539039012382229530463437\" + }, + \"SequenceNumberRange\": { + \"StartingSequenceNumber\": \"49579844037771934846562125484723780283101668556216963122\" + } + } + ] + }", + {ok, [ + {<<"NextToken">>, <<"AAAAAAAAAAGK9EEG0sJqVhCUS2JsgigQ5dcpB4q9PYswrH2oK44Skbjtm+WR0xA7/hrAFFsohevH1/OyPnbzKBS1byPyCZuVcokYtQe/b1m4c0SCI7jctPT0oUTLRdwSRirKm9dp9YC/EL+kZHOvYAUnztVGsOAPEFC3ECf/bVC927bDZBbRRzy/44OHfWmrCLcbcWqehRh5D14WnL3yLsumhiHDkyuxSlkBepauvMnNLtTOlRtmQ5Q5reoujfq2gzeCSOtLcfXgBMztJqohPdgMzjTQSbwB9Am8rMpHLsDbSdMNXmITvw==">>}, + {<<"Shards">>, [ + [ + {<<"ShardId">>, <<"shardId-000000000001">>}, + {<<"HashKeyRange">>, [ + {<<"EndingHashKey">>, <<"68056473384187692692674921486353642280">>}, + {<<"StartingHashKey">>, <<"34028236692093846346337460743176821145">>} + ]}, + {<<"SequenceNumberRange">>, [ + {<<"StartingSequenceNumber">>, <<"49579844037727333356165064238440708846556371693205002258">>} + ]} + ], [ + {<<"ShardId">>, <<"shardId-000000000002">>}, + {<<"HashKeyRange">>, [ + {<<"EndingHashKey">>, <<"102084710076281539039012382229530463436">>}, + {<<"StartingHashKey">>, <<"68056473384187692692674921486353642281">>} + ]}, + {<<"SequenceNumberRange">>, [ + {<<"StartingSequenceNumber">>, <<"49579844037749634101363594861582244564829020124710982690">>} + ]} + ], [ + {<<"ShardId">>, <<"shardId-000000000003">>}, + {<<"HashKeyRange">>, [ + {<<"EndingHashKey">>, <<"136112946768375385385349842972707284581">>}, + {<<"StartingHashKey">>, <<"102084710076281539039012382229530463437">>} + ]}, + {<<"SequenceNumberRange">>, [ + {<<"StartingSequenceNumber">>, <<"49579844037771934846562125484723780283101668556216963122">>} + ]} + ] + ]} + ]} + } + )], + + output_tests(?_f(erlcloud_kinesis:list_shards(<<"staging">>)), Tests). + %% ListStreams test based on the API examples: %% http://docs.aws.amazon.com/kinesis/latest/APIReference/API_ListStreams.html list_streams_input_tests(_) -> From 84cea2e05347bafc71ec62b2b0c02b17ecd1e4ea Mon Sep 17 00:00:00 2001 From: hmartel Date: Mon, 1 Jul 2019 07:47:32 -0500 Subject: [PATCH 058/310] Address review comments. --- src/erlcloud_kinesis.erl | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index 80c89e3ef..f1252c584 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -87,10 +87,15 @@ dynamize_option({next_token, Value}) -> dynamize_option({stream_creation_timestamp, Value}) -> {<<"StreamCreationTimestamp">>, Value}; dynamize_option(Option) -> - error({erlcloud_kinesis, {invalid_option, Option}}). + {error, {invalid_option, Option}}. dynamize_options(Options) -> - [dynamize_option(Option) || Option <- Options]. + lists:foldr(fun + (Option, Acc) when is_list(Acc) -> + [dynamize_option(Option) | Acc]; + (_Option, Error) -> + Error + end, [], Options). %%------------------------------------------------------------------------------ %% @doc @@ -207,7 +212,7 @@ delete_stream(StreamName, Config) when is_record(Config, aws_config) -> %% @end %%------------------------------------------------------------------------------ --spec list_shards(binary()) -> erlcloud_kinesis_impl:json_return(). +-spec list_shards(binary()) -> erlcloud_kinesis_impl:json_return() | {error, any()}. list_shards(StreamName) -> list_shards(StreamName, default_config()). @@ -242,8 +247,13 @@ list_shards(StreamName, Options, Config) when is_record(Config, aws_config) -> ]}}) end end, - Json = [{<<"StreamName">>, StreamName} | dynamize_options(Options)], - erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", Json). + case dynamize_options(Options) of + DynamizedOptions when is_list(DynamizedOptions) -> + Json = [{<<"StreamName">>, StreamName} | dynamize_options(Options)], + erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", Json); + Error -> + Error + end. %%------------------------------------------------------------------------------ %% @doc From a3f1d9c3d832943ea0b4da41600bfa8628a12c79 Mon Sep 17 00:00:00 2001 From: hmartel Date: Mon, 1 Jul 2019 09:36:14 -0500 Subject: [PATCH 059/310] Address review comments. --- src/erlcloud_kinesis.erl | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index f1252c584..0d5b74f2f 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -224,35 +224,29 @@ list_shards(StreamName, Options) -> -spec list_shards(binary(), list_shards_opts(), aws_config()) -> erlcloud_kinesis_impl:json_return(). list_shards(StreamName, Options, Config) when is_record(Config, aws_config) -> - case lists:keymember(next_token, 1, Options) of + DynamizedOptionsOrError = case lists:keymember(next_token, 1, Options) of false -> - ok; + dynamize_options(Options); true -> case lists:keymember(stream_creation_timestamp, 1, Options) of false -> - ok; + dynamize_options(Options); true -> - error({erlcloud_kinesis, {incompatible_options, [ - next_token, - stream_creation_timestamp - ]}}) + {error, {incompatible_options, [next_token, stream_creation_timestamp]}} end, case lists:keymember(exclusive_start_shard_id, 1, Options) of false -> - ok; + dynamize_options(Options); true -> - error({erlcloud_kinesis, {incompatible_options, [ - next_token, - exclusive_start_shard_id - ]}}) + {error, {incompatible_options, [next_token, exclusive_start_shard_id]}} end end, - case dynamize_options(Options) of - DynamizedOptions when is_list(DynamizedOptions) -> - Json = [{<<"StreamName">>, StreamName} | dynamize_options(Options)], + case is_list(DynamizedOptionsOrError) of + true -> + Json = [{<<"StreamName">>, StreamName} | DynamizedOptionsOrError], erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", Json); - Error -> - Error + false -> + DynamizedOptionsOrError end. %%------------------------------------------------------------------------------ From 50da38e8a4742985e8c3b3cf3dcb95aba1019c6f Mon Sep 17 00:00:00 2001 From: hmartel Date: Mon, 1 Jul 2019 11:46:43 -0500 Subject: [PATCH 060/310] Address review comments. --- src/erlcloud_kinesis.erl | 48 +++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index 0d5b74f2f..f582ab7b2 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -92,7 +92,12 @@ dynamize_option(Option) -> dynamize_options(Options) -> lists:foldr(fun (Option, Acc) when is_list(Acc) -> - [dynamize_option(Option) | Acc]; + case dynamize_option(Option) of + {error, _} = Error -> + Error; + DynamizedOption -> + [DynamizedOption | Acc] + end; (_Option, Error) -> Error end, [], Options). @@ -212,43 +217,30 @@ delete_stream(StreamName, Config) when is_record(Config, aws_config) -> %% @end %%------------------------------------------------------------------------------ --spec list_shards(binary()) -> erlcloud_kinesis_impl:json_return() | {error, any()}. -list_shards(StreamName) -> - list_shards(StreamName, default_config()). +-spec list_shards(binary() | list_shards_opts()) -> erlcloud_kinesis_impl:json_return() | {error, any()}. +list_shards(StreamNameOrOptions) -> + list_shards(StreamNameOrOptions, default_config()). --spec list_shards(binary(), list_shards_opts() | aws_config()) -> erlcloud_kinesis_impl:json_return(). -list_shards(StreamName, Config) when is_record(Config, aws_config) -> +-spec list_shards(binary() | list_shards_opts(), list_shards_opts() | aws_config()) -> erlcloud_kinesis_impl:json_return(). +list_shards(StreamName, Config) when is_record(Config, aws_config), is_binary(StreamName) -> list_shards(StreamName, [], Config); +list_shards(Options, Config) when is_record(Config, aws_config), is_list(Options) -> + Json = dynamize_options(Options), + erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", Json); list_shards(StreamName, Options) -> list_shards(StreamName, Options, default_config()). -spec list_shards(binary(), list_shards_opts(), aws_config()) -> erlcloud_kinesis_impl:json_return(). list_shards(StreamName, Options, Config) when is_record(Config, aws_config) -> - DynamizedOptionsOrError = case lists:keymember(next_token, 1, Options) of - false -> - dynamize_options(Options); - true -> - case lists:keymember(stream_creation_timestamp, 1, Options) of - false -> - dynamize_options(Options); - true -> - {error, {incompatible_options, [next_token, stream_creation_timestamp]}} - end, - case lists:keymember(exclusive_start_shard_id, 1, Options) of - false -> - dynamize_options(Options); - true -> - {error, {incompatible_options, [next_token, exclusive_start_shard_id]}} - end - end, - case is_list(DynamizedOptionsOrError) of - true -> - Json = [{<<"StreamName">>, StreamName} | DynamizedOptionsOrError], + case dynamize_options(Options) of + DynamizedOptions when is_list(DynamizedOptions) -> + Json = [{<<"StreamName">>, StreamName} | DynamizedOptions], erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", Json); - false -> - DynamizedOptionsOrError + Error -> + Error end. + %%------------------------------------------------------------------------------ %% @doc %% Kinesis API: From ffdf485e9b8110be5b25de0e74d6cfb582d2a17e Mon Sep 17 00:00:00 2001 From: hmartel Date: Mon, 1 Jul 2019 13:37:25 -0500 Subject: [PATCH 061/310] Address review comments. --- src/erlcloud_kinesis.erl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index f582ab7b2..16583ad4a 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -204,7 +204,7 @@ delete_stream(StreamName, Config) when is_record(Config, aws_config) -> %% {<<"ShardId">>, <<"shardId-000000000003">>}, %% {<<"HashKeyRange">>, [ %% {<<"EndingHashKey">>, <<"136112946768375385385349842972707284581">>}, -%% {<<"StartingHashKey">>, <<"102084710076281539039012382229530463437">>} +%% {<<"StartingHashKey">>, <<"102084710022876281539039012382229530463437">>} %% ]}, %% {<<"SequenceNumberRange">>, [ %% {<<"StartingSequenceNumber">>, <<"49579844037771934846562125484723780283101668556216963122">>} @@ -225,8 +225,12 @@ list_shards(StreamNameOrOptions) -> list_shards(StreamName, Config) when is_record(Config, aws_config), is_binary(StreamName) -> list_shards(StreamName, [], Config); list_shards(Options, Config) when is_record(Config, aws_config), is_list(Options) -> - Json = dynamize_options(Options), - erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", Json); + case dynamize_options(Options) of + DynamizedOptions when is_list(DynamizedOptions) -> + erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", DynamizedOptions); + Error -> + Error + end; list_shards(StreamName, Options) -> list_shards(StreamName, Options, default_config()). From 5704bfb1283be672f3975f15a4b3adea361ada52 Mon Sep 17 00:00:00 2001 From: hmartel Date: Mon, 1 Jul 2019 14:03:04 -0500 Subject: [PATCH 062/310] Address review comments. --- src/erlcloud_kinesis.erl | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index 16583ad4a..09818d8cc 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -78,6 +78,8 @@ configure(AccessKeyID, SecretAccessKey, Host, Port) -> default_config() -> erlcloud_aws:default_config(). +dynamize_option({stream_name, Value}) -> + {<<"StreamName">>, Value}; dynamize_option({exclusive_start_shard_id, Value}) -> {<<"ExclusiveStartShardId">>, Value}; dynamize_option({max_results, Value}) when Value >= 1, Value =< 10000 -> @@ -235,14 +237,10 @@ list_shards(StreamName, Options) -> list_shards(StreamName, Options, default_config()). -spec list_shards(binary(), list_shards_opts(), aws_config()) -> erlcloud_kinesis_impl:json_return(). -list_shards(StreamName, Options, Config) when is_record(Config, aws_config) -> - case dynamize_options(Options) of - DynamizedOptions when is_list(DynamizedOptions) -> - Json = [{<<"StreamName">>, StreamName} | DynamizedOptions], - erlcloud_kinesis_impl:request(Config, "Kinesis_20131202.ListShards", Json); - Error -> - Error - end. +list_shards(StreamName, Options, Config) -> + % Syntaxtic sugar as all other erlcloud_kinesis operating on a stream takes + % a stream name as 1st argument. + list_shards([{stream_name, StreamName} | Options], Config). %%------------------------------------------------------------------------------ From 52386f63874ac5b4d5cce07d36b61caa6a9f1cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos?= Date: Thu, 4 Jul 2019 17:44:08 +0100 Subject: [PATCH 063/310] Implement Application Auto Scaling module Fix name of module Operations added Exporting functions Retry failed attempts with exponential backoff Adding extract to policies Add optional boolean type to json decoder --- src/erlcloud_application_autoscaler.erl | 666 ++++++++++++++++++++++++ src/erlcloud_json.erl | 3 +- 2 files changed, 668 insertions(+), 1 deletion(-) create mode 100644 src/erlcloud_application_autoscaler.erl diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl new file mode 100644 index 000000000..b163e5811 --- /dev/null +++ b/src/erlcloud_application_autoscaler.erl @@ -0,0 +1,666 @@ +-module(erlcloud_application_autoscaler). + +-include("erlcloud_aws.hrl"). +-include("erlcloud_as.hrl"). +-include("erlcloud_xmerl.hrl"). + +%% ------------------------------------------------------------------ +%% AWS Application Autoscaling Function Exports +%% ------------------------------------------------------------------ + +-export([delete_scaling_policy/5]). + +-export([delete_scheduled_action/2]). +-export([delete_scheduled_action/5]). + +-export([deregister_scalable_target/2]). +-export([deregister_scalable_target/4]). + +-export([describe_scalable_targets/2]). + +-export([describe_scaling_activities/2]). + +-export([describe_scaling_policies/2]). + +-export([describe_scheduled_actions/2]). + +-export([put_scaling_policy/2]). +-export([put_scaling_policy/5]). +-export([put_scaling_policy/7]). + +-export([put_scheduled_action/2]). +-export([put_scheduled_action/5]). +-export([put_scheduled_action/6]). +-export([put_scheduled_action/7]). +-export([put_scheduled_action/8]). + +-export([register_scalable_target/2]). +-export([register_scalable_target/4]). +-export([register_scalable_target/5]). +-export([register_scalable_target/7]). + +%% ------------------------------------------------------------------ +%% AWS Application Autoscaling Library Initialization Function Exports +%% ------------------------------------------------------------------ + +-export([configure/2, configure/3, configure/4, configure/5, + new/2, new/3, new/4, new/5]). +-export([default_config/0]). + +%% ------------------------------------------------------------------ +%% Macro Definitions +%% ------------------------------------------------------------------ + +-define(NUM_ATTEMPTS, 10). +-define(DEFAULT_MAX_RECORDS, 20). + +%% ------------------------------------------------------------------ +%% Type Definitions +%% ------------------------------------------------------------------ + +-type response_attribute() :: binary(). +-type response_key() :: atom(). +-type response() :: [{response_key(), response_attribute()} | proplist:proplist()]. + +-type aws_aas_request_body() :: [proplist:proplist()]. + +-spec extract_alarm(J :: binary()) -> response(). +extract_alarm(J) -> + erlcloud_json:decode([ + {alarm_name, <<"AlarmName">>, optional_string}, + {alarm_arn, <<"AlarmARN">>, optional_string} + ], J). + +-spec extract_step_adjustments(J :: binary) -> response(). +extract_step_adjustments(J) -> + erlcloud_json:decode([ + {metric_interval_lower_bound, <<"MetricIntervalLowerBound">>, optional_integer}, + {metric_interval_upper_bound, <<"MetricIntervalupperBound">>, optional_integer}, + {scaling_adjustment, <<"ScalingAdjustment">>, optional_integer} + ], J). + +-spec extract_step_scaling_policy(J :: binary) -> response(). +extract_step_scaling_policy(J) -> + Scaling = erlcloud_json:decode([ + {adjustment_type, <<"AdjustmentType">>, optional_string}, + {cooldown, <<"Cooldown">>, optional_integer}, + {metric_aggregation_type, <<"MetricAggregationType">>, optional_string}, + {min_adjustment_magnitude, <<"MinAdjustmentMagnitude">>, optional_integer} + ], J), + case proplists:get_value(<<"StepAdjustments">>, J, undefined) of + undefined -> + Scaling; + Steps -> + [{step_adjustments, [ extract_step_adjustments(Step) || Step <- Steps]} | Scaling] + end. + +-spec extract_dimensions(J :: binary) -> response(). +extract_dimensions(J) -> + erlcloud_json:decode([ + {name, <<"Name">>, optional_string}, + {value, <<"Value">>, optional_string} + ], J). + +-spec extract_predefined_metric_specifications(J :: binary) -> response(). +extract_predefined_metric_specifications(J) -> + erlcloud_json:decode([ + {predefined_metric_type, <<"PredefinedMetricType">>, optional_string}, + {resource_label, <<"ResourceLabel">>, optional_string} + ], J). + +-spec extract_customized_metric_specification(J :: binary) -> response(). +extract_customized_metric_specification(J) -> + CustomizedMetricSpecification = erlcloud_json:decode([ + {metric_name, <<"MetricName">>, optional_string}, + {namespace, <<"Namespace">>, optional_string}, + {statistic, <<"Statistic">>, optional_string}, + {unit, <<"Unit">>, optional_string} + ], J), + MaybeHasDimension = case proplists:get_value(<<"Dimensions">>, J, undefined) of + undefined -> + []; + Dimensions -> + [{dimension, [extract_dimensions(Dim) || Dim <- Dimensions ]}] + end, + CustomizedMetricSpecification ++ MaybeHasDimension. + +-spec extract_target_tracking_policy(J :: binary) -> response(). +extract_target_tracking_policy(J) -> + Target = erlcloud_json:decode([ + {disable_scale_in, <<"DisableScaleIn">>, optional_boolean}, + {scale_in_cooldown, <<"ScaleInCooldown">>, optional_integer}, + {scale_out_cooldown, <<"ScaleOutCooldown">>, optional_integer}, + {target_value, <<"TargetValue">>, optional_integer} + ], J), + MaybeHasCustomizedMetrics = case proplists:get_value(<<"CustomizedMetricsSpecification">>, J, undefined) of + undefined -> + []; + CustomMetrics -> + [{customized_metrics_specification, extract_customized_metric_specification(CustomMetrics)}] + end, + MaybeHasPredefinedMetrics = case proplists:get_value(<<"PredefinedMetricSpecification">>, J, undefined) of + undefined -> + []; + PredefinedMetrics -> + [{predefined_metric_specification, extract_predefined_metric_specifications(PredefinedMetrics)}] + end, + Target ++ MaybeHasCustomizedMetrics ++ MaybeHasPredefinedMetrics. + +-spec extract_scaling_policies(J :: binary()) -> response(). +extract_scaling_policies(J) -> + Policy = erlcloud_json:decode([ + {creation_time, <<"CreationTime">>, optional_integer}, + {policy_arn, <<"PolicyARN">>, optional_string}, + {policy_name, <<"PolicyName">>, optional_string}, + {policy_type, <<"PolicyType">>, optional_string}, + {resource_id, <<"ResourceId">>, optional_string}, + {scalable_dimension, <<"ScalableDimension">>, optional_string}, + {service_namespace, <<"ServiceNamespace">>, optional_string} + ], J), + MaybeHasAlarms = case proplists:get_value(<<"Alarms">>, J, undefined) of + undefined -> + []; + AlarmJSON -> + [{alarms, [extract_alarm(Alarm) || Alarm <- AlarmJSON]}] + end, + MaybeHasStepScaling = case proplists:get_value(<<"StepScalingPolicyConfiguration">>, J, undefined) of + undefined -> + []; + StepScaling -> + [{step_scaling_policy_configuration, extract_step_scaling_policy(StepScaling)}] + end, + MaybeHasTargetTracking = case proplists:get_value(<<"TargetTrackingScalingPolicyConfiguration">>, J, undefined) of + undefined -> + []; + TargetTracking -> + [{target_tracking_scaling_policy_configuration, extract_target_tracking_policy(TargetTracking)}] + end, + Policy ++ MaybeHasAlarms ++ MaybeHasStepScaling ++ MaybeHasTargetTracking. + + +-spec extract_scalable_targets(J :: binary()) -> response(). +extract_scalable_targets(J) -> + erlcloud_json:decode([ + {creation_time, <<"CreationTime">>, optional_integer}, + {max_capacity, <<"MaxCapacity">>, optional_integer}, + {min_capacity, <<"MinCapacity">>, optional_integer}, + {resource_id, <<"ResourceId">>, optional_string}, + {role_arn, <<"RoleARN">>, optional_string}, + {scalable_dimension, <<"ScalableDimension">>, optional_string}, + {service_namespace, <<"ServiceNamespace">>, optional_string} + ], J). + +-spec extract_scaling_activities(J :: binary) -> response(). +extract_scaling_activities(J) -> + erlcloud_json:decode([ + {activity_id, <<"ActivityId">>, optional_string}, + {cause, <<"Cause">>, optional_string}, + {description, <<"Description">>, optional_string}, + {resource_id, <<"ResourceId">>, optional_string}, + {scalable_dimension, <<"ScalableDimension">>, optional_string}, + {service_namespace, <<"ServiceNamespace">>, optional_string}, + {start_time, <<"StartTime">>, optional_integer}, + {status_code, <<"StatusCode">>, optional_string}, + {status_message, <<"StatusMessage">>, optional_string} + ], J). + +-spec extract_scheduled_action(J :: binary) -> response(). +extract_scheduled_action(J) -> + Res = erlcloud_json:decode([ + {creation_time, <<"CreationTime">>, optional_integer}, + {resource_id, <<"ResourceId">>, optional_string}, + {scalable_dimension, <<"ScalableDimension">>, optional_string}, + {service_namespace, <<"ServiceNamespace">>, optional_string}, + {scheduled_action_arn, <<"ScheduledActionARN">>, optional_string}, + {scheduled_action_name, <<"ScheduledActionName">>, optional_string} + ], J), + Action = proplists:get_value(<<"ScalableTargetAction">>, J), + PropAction = erlcloud_json:decode([ + {max_capacity, <<"MaxCapacity">>, optional_integer}, + {min_capacity, <<"MinCapacity">>, optional_integer} + ], Action), + [{scalable_target_action, PropAction} | Res]. + + +%%%------------------------------------------------------------------------------ +%%% Library initialization. +%%%------------------------------------------------------------------------------ + +-spec new(string(), string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey) -> + #aws_config{access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey}. + +-spec new(string(), string(), string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey, Host) -> + #aws_config{access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey, + autoscaling_host=Host}. + +-spec new(string(), string(), string(), non_neg_integer()) -> aws_config(). +new(AccessKeyID, SecretAccessKey, Host, Port) -> + #aws_config{access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey, + autoscaling_host=Host, + autoscaling_port=Port}. + +-spec new(string(), string(), string(), non_neg_integer(), string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey, Host, Port, Scheme) -> + #aws_config{access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey, + autoscaling_host=Host, + autoscaling_port=Port, + autoscaling_scheme=Scheme}. + +-spec configure(string(), string()) -> ok. +configure(AccessKeyID, SecretAccessKey) -> + erlcloud_config:configure(AccessKeyID, SecretAccessKey, fun new/2). + +-spec configure(string(), string(), string()) -> ok. +configure(AccessKeyID, SecretAccessKey, Host) -> + erlcloud_config:configure(AccessKeyID, SecretAccessKey, Host, fun new/3). + +-spec configure(string(), string(), string(), non_neg_integer()) -> ok. +configure(AccessKeyID, SecretAccessKey, Host, Port) -> + erlcloud_config:configure(AccessKeyID, SecretAccessKey, Host, Port, fun new/4). + +-spec configure(string(), string(), string(), non_neg_integer(), string()) -> ok. +configure(AccessKeyID, SecretAccessKey, Host, Port, Scheme) -> + erlcloud_config:configure(AccessKeyID, SecretAccessKey, Host, Port, Scheme, fun new/5). + +default_config() -> erlcloud_aws:default_config(). + +%%%------------------------------------------------------------------------------ +%%% AWS Application Autoscaling functions. +%%%------------------------------------------------------------------------------ + +%% DeleteScalingPolicy + +-spec delete_scaling_policy(Configuration :: aws_config(), + PolicyName :: binary(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary() + ) -> term(). +delete_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, ServiceNamespace) -> + BodyProps = [{<<"PolicyName">>, PolicyName}, + {<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}], + delete_scaling_policy(Configuration, BodyProps). + +-spec delete_scaling_policy(Configuration :: aws_config(), + BodyConfigurations :: aws_aas_request_body() + ) -> term(). +delete_scaling_policy(Configuration, BodyConfigurations) -> + request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingPolicies"). + +%% DeleteScheduledAction + +-spec delete_scheduled_action( + Configuration :: erlcloud_aws:aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ScheduledActionName :: binary(), + ServiceNamespace :: binary() + ) -> response(). +delete_scheduled_action(Configuration, ResourceId, ScalableDimension, ScheduledActionName, ServiceNamespace) -> + BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, + {<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}], + delete_scheduled_action(Configuration, BodyProps). + + +-spec delete_scheduled_action(Configuration :: aws_config(), + BodyConfigurations :: aws_aas_request_body() + ) -> response(). +delete_scheduled_action(Configuration, BodyConfigurations) -> + request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DeleteScheduledAction"). + +%% DeregisterScalableTarget + +-spec deregister_scalable_target( + Configuration :: erlcloud_aws:aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary() + ) -> response(). +deregister_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace) -> + BodyProps = [{<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}], + deregister_scalable_target(Configuration, BodyProps). + +-spec deregister_scalable_target( + Configuration :: erlcloud_aws:aws_config(), + BodyConfigurations :: aws_aas_request_body() + ) -> response(). +deregister_scalable_target(Configuration, BodyConfigurations) -> + request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DeregisterScalableTarget"). + +%% DescribeScalableTarget + +-spec describe_scalable_targets( + erlcloud_aws:aws_config(), + aws_aas_request_body() | binary() + ) -> response(). +describe_scalable_targets(Configuration, ServiceNamespace) when is_binary(ServiceNamespace)-> + describe_scalable_targets(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); +describe_scalable_targets(Configuration, BodyConfigurations) -> + {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalableTargets"), + ScalableTargets = proplists:get_value(<<"ScalableTargets">>, Result), + PropRes = [extract_scalable_targets(E) || E <- ScalableTargets], + NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), + MaybeHasNext = case NextToken of + undefined -> []; + Token -> [{next_token, Token}] + end, + {ok, PropRes ++ MaybeHasNext}. + +%% DescribeScalingActivities + +-spec describe_scaling_activities( + erlcloud_aws:aws_config(), + aws_aas_request_body() | binary() + ) -> response(). +describe_scaling_activities(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> + describe_scaling_activities(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); +describe_scaling_activities(Configuration, BodyConfigurations) -> + {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingActivities"), + ScalingActivities = proplists:get_value(<<"ScalingActivities">>, Result), + PropRes = [extract_scaling_activities(E) || E <- ScalingActivities], + NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), + MaybeHasNext = case NextToken of + undefined -> []; + Token -> [{next_token, Token}] + end, + {ok, PropRes ++ MaybeHasNext}. + +%% DescribeScalingPolicies + +-spec describe_scaling_policies( + erlcloud_aws:aws_config(), + aws_aas_request_body() | binary() + ) -> response(). +describe_scaling_policies(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> + describe_scaling_policies(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); +describe_scaling_policies(Configuration, BodyConfigurations) -> + {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingPolicies"), + ScalingPolicies = proplists:get_value(<<"ScalingPolicies">>, Result), + PropRes = [extract_scaling_policies(Extracted) || Extracted <- ScalingPolicies], + NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), + MaybeHasNext = case NextToken of + undefined -> []; + Token -> [{next_token, Token}] + end, + {ok, PropRes ++ MaybeHasNext}. + + +%% DescribeScheduledActions + +-spec describe_scheduled_actions( + erlcloud_aws:aws_config(), + aws_aas_request_body() | binary() + ) -> response(). +describe_scheduled_actions(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> + describe_scheduled_actions(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); +describe_scheduled_actions(Configuration, BodyConfigurations) -> + {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScheduledActions"), + ScheduledActions = proplists:get_value(<<"ScheduledActions">>, Result), + PropRes = [extract_scheduled_action(Extracted) || Extracted <- ScheduledActions], + NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), + MaybeHasNext = case NextToken of + undefined -> []; + Token -> [{next_token, Token}] + end, + {ok, PropRes ++ MaybeHasNext}. + + +%% PutScalingPolicy + +-spec put_scaling_policy( + Configuration :: aws_config(), + PolicyName :: binary(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary() + ) -> response(). +put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, ServiceNamespace) -> + BodyProps = [{<<"PolicyName">>, PolicyName}, + {<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}], + put_scaling_policy(Configuration, BodyProps). + +-spec put_scaling_policy( + Configuration :: aws_config(), + PolicyName :: binary(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary(), + PolicyType :: binary(), + Policy :: [proplist:proplist()] + ) -> response(). +put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, ServiceNamespace, PolicyType, Policy) -> + BodyProps = [{<<"PolicyName">>, PolicyName}, + {<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}, + {<<"PolicyType">>, PolicyType}], + case PolicyType of + <<"StepScaling">> -> + put_scaling_policy(Configuration, [{<<"StepScalingPolicyConfiguration">>, Policy} | BodyProps]); + <<"TargetTrackingScaling">> -> + put_scaling_policy(Configuration, [{<<"TargetTrackingScalingPolicyConfiguration">>, Policy} | BodyProps]) + end. + +-spec put_scaling_policy( + Configuration :: erlcloud_aws:aws_config(), + BodyConfigurations :: aws_aas_request_body() + ) -> response(). +put_scaling_policy(Configuration, BodyConfigurations) -> + {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.PutScalingPolicy"), + Alarms = proplists:get_value(<<"Alarms">>, Result), + PropAlarms = [extract_alarm(E) || E <- Alarms], + [{policy_arn, proplists:get_value(<<"PolicyARN">>, Result)} | PropAlarms]. + +%% PutScheduledAction + +-spec put_scheduled_action( + Configuration :: aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary(), + ScheduledActionName :: binary() + ) -> response(). +put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName) -> + BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, + {<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}], + put_scheduled_action(Configuration, BodyProps). + +-spec put_scheduled_action( + Configuration :: aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary(), + ScheduledActionName :: binary(), + Schedule :: binary() + ) -> response(). +put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule) -> + BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, + {<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}, + {<<"Schedule">>, Schedule}], + put_scheduled_action(Configuration, BodyProps). + +-spec put_scheduled_action( + Configuration :: aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary(), + ScheduledActionName :: binary(), + Schedule :: binary(), + StartTime :: pos_integer() + ) -> response(). +put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule, StartTime) -> + BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, + {<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}, + {<<"Schedule">>, Schedule}, + {<<"StartTime">>, StartTime}], + put_scheduled_action(Configuration, BodyProps). + +-spec put_scheduled_action( + Configuration :: aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary(), + ScheduledActionName :: binary(), + Schedule :: binary(), + StartTime :: pos_integer(), + EndTime :: pos_integer() + ) -> response(). +put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule, StartTime, EndTime) -> + BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, + {<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}, + {<<"Schedule">>, Schedule}, + {<<"StartTime">>, StartTime}, + {<<"EndTime">>, EndTime}], + put_scheduled_action(Configuration, BodyProps). + +-spec put_scheduled_action(Configuration :: aws_config(), + BodyConfigurations :: aws_aas_request_body() + ) -> response(). +put_scheduled_action(Configuration, BodyConfigurations) -> + request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.PutScheduledAction"). + +%% RegisterScalableTarget + +-spec register_scalable_target(Configuration :: aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary()) -> response(). +register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace) -> + BodyProps = [{<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}], + register_scalable_target(Configuration, BodyProps). + +-spec register_scalable_target(Configuration :: aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary(), + ResourceARN :: binary()) -> response(). +register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ResourceARN) -> + BodyProps = [{<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}, + {<<"ResourceARN">>, ResourceARN}], + register_scalable_target(Configuration, BodyProps). + +-spec register_scalable_target(Configuration :: aws_config(), + ResourceId :: binary(), + ScalableDimension :: binary(), + ServiceNamespace :: binary(), + ResourceARN :: binary(), + MinCapacity :: integer() | undefined, + MaxCapacity :: integer() | undefined) -> response(). +register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ResourceARN, MinCapacity, MaxCapacity) -> + BodyProps = [{<<"ResourceId">>, ResourceId}, + {<<"ScalableDimension">>, ScalableDimension}, + {<<"ServiceNamespace">>, ServiceNamespace}, + {<<"ResourceARN">>, ResourceARN}], + MaybeBodyWithMax = case MaxCapacity of + undefined -> []; + Max -> [{<<"MaxCapacity">>, Max}] + end, + MaybeBodyWithMin = case MinCapacity of + undefined -> []; + Min -> [{<<"MinCapacity">>, Min}] + end, + register_scalable_target(Configuration, BodyProps ++ MaybeBodyWithMax ++ MaybeBodyWithMin). + +-spec register_scalable_target( + Configurations :: erlcloud_aws:aws_config(), + BodyConfigurations :: aws_aas_request_body() +) -> response(). +register_scalable_target(Configurations, BodyConfigurations) -> + request_with_action(Configurations, BodyConfigurations, "AnyScaleFrontendService.RegisterScalableTarget"). + + +%%%------------------------------------------------------------------------------ +%%% Internal functions. +%%%------------------------------------------------------------------------------ + +request_with_action(Configuration, BodyConfigurations, Action) -> + Body = jsx:encode(BodyConfigurations), + case erlcloud_aws:update_config(Configuration) of + {ok, Config} -> + HeadersPrev = headers(Config, Action, Body), + Headers = [{"content-type", "application/x-amz-json-1.1"} | HeadersPrev], + request_and_retry(Config, Headers, Body, {attempt, 0}); + {error, Reason} -> + {error, Reason} + end. + +request_and_retry(_, _, _, {error, Reason}) -> + {error, Reason}; +request_and_retry(_, _, _, {attempt, Attempt}) when Attempt =:= 11 -> + {error, no_answer}; +request_and_retry(Config, Headers, Body, {attempt, Attempt}) -> + case erlcloud_httpc:request(url(Config), post, Headers, Body, timeout(Attempt, Config), Config) of + + {ok, {{200, _}, _, <<>>}} -> + ok; + + {ok, {{200, _}, _, RespBody}} -> + {ok, jsx:decode(RespBody)}; + + _Error -> + request_and_retry(Config, Headers, Body, retry({attempt, Attempt + 1})) + end. + +retry({attempt, Attempt}) when Attempt >= ?NUM_ATTEMPTS -> + {error, max_limit_of_attempts}; +retry({attempt, Attempt}) -> + backoff(Attempt), + {attempt, Attempt + 1}. + +%% Sleep after an attempt +-spec backoff(pos_integer()) -> ok. +backoff(1) -> ok; +backoff(Attempt) -> + timer:sleep(erlcloud_util:rand_uniform((1 bsl (Attempt - 1)) * 100)). + +-spec timeout(pos_integer(), aws_config()) -> pos_integer(). +%% HTTPC timeout for a request +timeout(1, #aws_config{timeout = undefined}) -> + %% Shorter timeout on first request. This is to avoid long (5s) failover when first DDB + %% endpoint doesn't respond + 1000; +timeout(_, #aws_config{} = Cfg) -> + erlcloud_aws:get_timeout(Cfg). + + +-spec url(aws_config()) -> []. +url(#aws_config{autoscaling_scheme = Scheme, autoscaling_host = Host} = Config) -> + lists:flatten([Scheme, Host, port_spec(Config)]). + +port_spec(#aws_config{autoscaling_port=80}) -> + ""; +port_spec(#aws_config{autoscaling_port=Port}) -> + [":", erlang:integer_to_list(Port)]. + + +headers(Config, Operation, Body) -> + Headers = [{"host", Config#aws_config.autoscaling_host}, + {"x-amz-target", Operation}], + + erlcloud_aws:sign_v4_headers(Config, Headers, Body, erlcloud_aws:aws_region_from_host(Config#aws_config.autoscaling_host), "application-autoscaling"). diff --git a/src/erlcloud_json.erl b/src/erlcloud_json.erl index c6351154a..4825808e0 100644 --- a/src/erlcloud_json.erl +++ b/src/erlcloud_json.erl @@ -4,7 +4,7 @@ -include("erlcloud.hrl"). -type decode_return() :: [{Name :: atom(), Value :: string() | integer()}]. --type decode_value_type() :: optional_string | optional_integer | {optional_map, fun(([{Key :: binary(), Value :: string() | integer()}]) -> decode_return())}. +-type decode_value_type() :: optional_string | optional_integer | optional_boolean | {optional_map, fun(([{Key :: binary(), Value :: string() | integer()}]) -> decode_return())}. -type decode_value() :: {atom(), JsonField :: binary(), Type :: decode_value_type()}. -type decode_value_r() :: {pos_integer(), JsonField :: binary(), Type :: atom()}. @@ -33,6 +33,7 @@ get_value(JsonField, Type, Json) -> case Type of optional_string -> proplists:get_value(JsonField, Json, undefined); optional_integer -> proplists:get_value(JsonField, Json, undefined); + optional_boolean -> proplists:get_value(JsonField, Json, undefined); string -> proplists:get_value(JsonField, Json, ""); integer -> proplists:get_value(JsonField, Json, 0); Fun when is_function(Fun, 1) -> From b3be61e6f51a375dc8aa35930e0f5dc8a5b46dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos?= Date: Tue, 9 Jul 2019 12:02:55 +0100 Subject: [PATCH 064/310] Uncovering errors --- src/erlcloud_application_autoscaler.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index b163e5811..53ef38baa 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -623,8 +623,10 @@ request_and_retry(Config, Headers, Body, {attempt, Attempt}) -> {ok, {{200, _}, _, RespBody}} -> {ok, jsx:decode(RespBody)}; - _Error -> - request_and_retry(Config, Headers, Body, retry({attempt, Attempt + 1})) + {error, {_, timeout}} -> + request_and_retry(Config, Headers, Body, retry({attempt, Attempt + 1})); + Error -> + request_and_retry(Config, Headers, Body, retry(Error)) end. retry({attempt, Attempt}) when Attempt >= ?NUM_ATTEMPTS -> From c902202e9b0630f8f4a1bb3350a9f34166edacec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos?= Date: Tue, 9 Jul 2019 14:46:37 +0100 Subject: [PATCH 065/310] Add coherency to other modules of the lib --- src/erlcloud_application_autoscaler.erl | 103 ++++++++++++++---------- 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 53ef38baa..4d241ed5f 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -348,15 +348,19 @@ deregister_scalable_target(Configuration, BodyConfigurations) -> describe_scalable_targets(Configuration, ServiceNamespace) when is_binary(ServiceNamespace)-> describe_scalable_targets(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scalable_targets(Configuration, BodyConfigurations) -> - {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalableTargets"), - ScalableTargets = proplists:get_value(<<"ScalableTargets">>, Result), - PropRes = [extract_scalable_targets(E) || E <- ScalableTargets], - NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), - MaybeHasNext = case NextToken of - undefined -> []; - Token -> [{next_token, Token}] - end, - {ok, PropRes ++ MaybeHasNext}. + case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalableTargets") of + {ok, Result} -> + ScalableTargets = proplists:get_value(<<"ScalableTargets">>, Result), + PropRes = [extract_scalable_targets(E) || E <- ScalableTargets], + NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), + MaybeHasNext = case NextToken of + undefined -> []; + Token -> [{next_token, Token}] + end, + {ok, PropRes ++ MaybeHasNext}; + {error, Error} -> + {error, Error} + end. %% DescribeScalingActivities @@ -367,16 +371,20 @@ describe_scalable_targets(Configuration, BodyConfigurations) -> describe_scaling_activities(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scaling_activities(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scaling_activities(Configuration, BodyConfigurations) -> - {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingActivities"), - ScalingActivities = proplists:get_value(<<"ScalingActivities">>, Result), - PropRes = [extract_scaling_activities(E) || E <- ScalingActivities], - NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), - MaybeHasNext = case NextToken of - undefined -> []; - Token -> [{next_token, Token}] - end, - {ok, PropRes ++ MaybeHasNext}. - + case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingActivities") of + {ok, Result} -> + {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingActivities"), + ScalingActivities = proplists:get_value(<<"ScalingActivities">>, Result), + PropRes = [extract_scaling_activities(E) || E <- ScalingActivities], + NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), + MaybeHasNext = case NextToken of + undefined -> []; + Token -> [{next_token, Token}] + end, + {ok, PropRes ++ MaybeHasNext}; + {error, Error} -> + {error, Error} + end. %% DescribeScalingPolicies -spec describe_scaling_policies( @@ -386,15 +394,19 @@ describe_scaling_activities(Configuration, BodyConfigurations) -> describe_scaling_policies(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scaling_policies(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scaling_policies(Configuration, BodyConfigurations) -> - {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingPolicies"), - ScalingPolicies = proplists:get_value(<<"ScalingPolicies">>, Result), - PropRes = [extract_scaling_policies(Extracted) || Extracted <- ScalingPolicies], - NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), - MaybeHasNext = case NextToken of - undefined -> []; - Token -> [{next_token, Token}] - end, - {ok, PropRes ++ MaybeHasNext}. + case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingPolicies") of + {ok, Result} -> + ScalingPolicies = proplists:get_value(<<"ScalingPolicies">>, Result), + PropRes = [extract_scaling_policies(Extracted) || Extracted <- ScalingPolicies], + NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), + MaybeHasNext = case NextToken of + undefined -> []; + Token -> [{next_token, Token}] + end, + {ok, PropRes ++ MaybeHasNext}; + {error, Error} -> + {error, Error} + end. %% DescribeScheduledActions @@ -406,15 +418,20 @@ describe_scaling_policies(Configuration, BodyConfigurations) -> describe_scheduled_actions(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scheduled_actions(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scheduled_actions(Configuration, BodyConfigurations) -> - {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScheduledActions"), - ScheduledActions = proplists:get_value(<<"ScheduledActions">>, Result), - PropRes = [extract_scheduled_action(Extracted) || Extracted <- ScheduledActions], - NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), - MaybeHasNext = case NextToken of - undefined -> []; - Token -> [{next_token, Token}] - end, - {ok, PropRes ++ MaybeHasNext}. + case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScheduledActions") of + {ok, Result} -> + ScheduledActions = proplists:get_value(<<"ScheduledActions">>, Result), + PropRes = [extract_scheduled_action(Extracted) || Extracted <- ScheduledActions], + NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), + MaybeHasNext = case NextToken of + undefined -> []; + Token -> [{next_token, Token}] + end, + {ok, PropRes ++ MaybeHasNext}; + {error, Error} -> + {error, Error} + + end. %% PutScalingPolicy @@ -460,10 +477,14 @@ put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, Ser BodyConfigurations :: aws_aas_request_body() ) -> response(). put_scaling_policy(Configuration, BodyConfigurations) -> - {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.PutScalingPolicy"), - Alarms = proplists:get_value(<<"Alarms">>, Result), - PropAlarms = [extract_alarm(E) || E <- Alarms], - [{policy_arn, proplists:get_value(<<"PolicyARN">>, Result)} | PropAlarms]. + case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.PutScalingPolicy") of + {ok, Result} -> + Alarms = proplists:get_value(<<"Alarms">>, Result), + PropAlarms = [extract_alarm(E) || E <- Alarms], + {ok, [{policy_arn, proplists:get_value(<<"PolicyARN">>, Result)} | PropAlarms]}; + {error, Error} -> + {error, Error} + end. %% PutScheduledAction From fe4e2074fe75df0d4534d5f73023224cfbf253ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos?= Date: Sat, 13 Jul 2019 16:35:26 +0100 Subject: [PATCH 066/310] Refactor to application autoscaler to use retry module --- src/erlcloud_application_autoscaler.erl | 99 ++++++++++++++++--------- 1 file changed, 64 insertions(+), 35 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 4d241ed5f..ee90f2ada 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -272,6 +272,8 @@ default_config() -> erlcloud_aws:default_config(). %%%------------------------------------------------------------------------------ %%% AWS Application Autoscaling functions. +%%% +%%% API Documentation on AWS: https://docs.aws.amazon.com/autoscaling/application/APIReference/Welcome.html %%%------------------------------------------------------------------------------ %% DeleteScalingPolicy @@ -290,10 +292,10 @@ delete_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, delete_scaling_policy(Configuration, BodyProps). -spec delete_scaling_policy(Configuration :: aws_config(), - BodyConfigurations :: aws_aas_request_body() + BodyConfiguration :: aws_aas_request_body() ) -> term(). -delete_scaling_policy(Configuration, BodyConfigurations) -> - request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingPolicies"). +delete_scaling_policy(Configuration, BodyConfiguration) -> + request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DescribeScalingPolicies"). %% DeleteScheduledAction @@ -313,10 +315,10 @@ delete_scheduled_action(Configuration, ResourceId, ScalableDimension, ScheduledA -spec delete_scheduled_action(Configuration :: aws_config(), - BodyConfigurations :: aws_aas_request_body() + BodyConfiguration :: aws_aas_request_body() ) -> response(). -delete_scheduled_action(Configuration, BodyConfigurations) -> - request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DeleteScheduledAction"). +delete_scheduled_action(Configuration, BodyConfiguration) -> + request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DeleteScheduledAction"). %% DeregisterScalableTarget @@ -334,10 +336,10 @@ deregister_scalable_target(Configuration, ResourceId, ScalableDimension, Service -spec deregister_scalable_target( Configuration :: erlcloud_aws:aws_config(), - BodyConfigurations :: aws_aas_request_body() + BodyConfiguration :: aws_aas_request_body() ) -> response(). -deregister_scalable_target(Configuration, BodyConfigurations) -> - request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DeregisterScalableTarget"). +deregister_scalable_target(Configuration, BodyConfiguration) -> + request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DeregisterScalableTarget"). %% DescribeScalableTarget @@ -347,8 +349,8 @@ deregister_scalable_target(Configuration, BodyConfigurations) -> ) -> response(). describe_scalable_targets(Configuration, ServiceNamespace) when is_binary(ServiceNamespace)-> describe_scalable_targets(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); -describe_scalable_targets(Configuration, BodyConfigurations) -> - case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalableTargets") of +describe_scalable_targets(Configuration, BodyConfiguration) -> + case request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DescribeScalableTargets") of {ok, Result} -> ScalableTargets = proplists:get_value(<<"ScalableTargets">>, Result), PropRes = [extract_scalable_targets(E) || E <- ScalableTargets], @@ -370,10 +372,10 @@ describe_scalable_targets(Configuration, BodyConfigurations) -> ) -> response(). describe_scaling_activities(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scaling_activities(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); -describe_scaling_activities(Configuration, BodyConfigurations) -> - case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingActivities") of +describe_scaling_activities(Configuration, BodyConfiguration) -> + case request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DescribeScalingActivities") of {ok, Result} -> - {ok, Result} = request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingActivities"), + {ok, Result} = request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DescribeScalingActivities"), ScalingActivities = proplists:get_value(<<"ScalingActivities">>, Result), PropRes = [extract_scaling_activities(E) || E <- ScalingActivities], NextToken = proplists:get_value(<<"NextToken">>, Result, undefined), @@ -393,8 +395,8 @@ describe_scaling_activities(Configuration, BodyConfigurations) -> ) -> response(). describe_scaling_policies(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scaling_policies(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); -describe_scaling_policies(Configuration, BodyConfigurations) -> - case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScalingPolicies") of +describe_scaling_policies(Configuration, BodyConfiguration) -> + case request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DescribeScalingPolicies") of {ok, Result} -> ScalingPolicies = proplists:get_value(<<"ScalingPolicies">>, Result), PropRes = [extract_scaling_policies(Extracted) || Extracted <- ScalingPolicies], @@ -417,8 +419,8 @@ describe_scaling_policies(Configuration, BodyConfigurations) -> ) -> response(). describe_scheduled_actions(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scheduled_actions(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); -describe_scheduled_actions(Configuration, BodyConfigurations) -> - case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.DescribeScheduledActions") of +describe_scheduled_actions(Configuration, BodyConfiguration) -> + case request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DescribeScheduledActions") of {ok, Result} -> ScheduledActions = proplists:get_value(<<"ScheduledActions">>, Result), PropRes = [extract_scheduled_action(Extracted) || Extracted <- ScheduledActions], @@ -474,10 +476,10 @@ put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, Ser -spec put_scaling_policy( Configuration :: erlcloud_aws:aws_config(), - BodyConfigurations :: aws_aas_request_body() + BodyConfiguration :: aws_aas_request_body() ) -> response(). -put_scaling_policy(Configuration, BodyConfigurations) -> - case request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.PutScalingPolicy") of +put_scaling_policy(Configuration, BodyConfiguration) -> + case request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.PutScalingPolicy") of {ok, Result} -> Alarms = proplists:get_value(<<"Alarms">>, Result), PropAlarms = [extract_alarm(E) || E <- Alarms], @@ -557,10 +559,10 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp put_scheduled_action(Configuration, BodyProps). -spec put_scheduled_action(Configuration :: aws_config(), - BodyConfigurations :: aws_aas_request_body() + BodyConfiguration :: aws_aas_request_body() ) -> response(). -put_scheduled_action(Configuration, BodyConfigurations) -> - request_with_action(Configuration, BodyConfigurations, "AnyScaleFrontendService.PutScheduledAction"). +put_scheduled_action(Configuration, BodyConfiguration) -> + request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.PutScheduledAction"). %% RegisterScalableTarget @@ -609,41 +611,69 @@ register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNa register_scalable_target(Configuration, BodyProps ++ MaybeBodyWithMax ++ MaybeBodyWithMin). -spec register_scalable_target( - Configurations :: erlcloud_aws:aws_config(), - BodyConfigurations :: aws_aas_request_body() + Configuration :: erlcloud_aws:aws_config(), + BodyConfiguration :: aws_aas_request_body() ) -> response(). -register_scalable_target(Configurations, BodyConfigurations) -> - request_with_action(Configurations, BodyConfigurations, "AnyScaleFrontendService.RegisterScalableTarget"). +register_scalable_target(Configuration, BodyConfiguration) -> + request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.RegisterScalableTarget"). %%%------------------------------------------------------------------------------ %%% Internal functions. %%%------------------------------------------------------------------------------ -request_with_action(Configuration, BodyConfigurations, Action) -> - Body = jsx:encode(BodyConfigurations), +aas_result_fun(#aws_request{response_type = ok} = Request) -> + Request; +aas_result_fun(#aws_request{response_type = error, + error_type = aws, + response_status = Status} = Request) when +%% Retry conflicting operations 409,Conflict and 500s +%% including 503, SlowDown, Reduce your request rate. + Status =:= 409; Status >= 500 -> + Request#aws_request{should_retry = true}; +aas_result_fun(#aws_request{response_type = error, error_type = aws} = Request) -> + Request#aws_request{should_retry = false}. + + +request_with_action(Configuration, BodyConfiguration, Action) -> + Body = jsx:encode(BodyConfiguration), case erlcloud_aws:update_config(Configuration) of {ok, Config} -> HeadersPrev = headers(Config, Action, Body), Headers = [{"content-type", "application/x-amz-json-1.1"} | HeadersPrev], - request_and_retry(Config, Headers, Body, {attempt, 0}); + Request = prepare_record(Config, post, Headers, Body, Action), + erlcloud_retry:request(Config, Request, aas_result_fun/1); + %request_and_retry(Config, Headers, Body, {attempt, 0}); {error, Reason} -> {error, Reason} end. +prepare_record(Config, Method, Headers, Body, Action) -> + %%% URI: awsConfig.Scheme + awsConfig.Host + [awsConfig.Port] + %%% URI: https://autoscaling.us-west-2.amazonaws.com/ + + RequestURI = lists:flatten([ + Config#aws_config.autoscaling_scheme, + Config#aws_config.autoscaling_host, + port_spec(Config) + ]), + + #aws_request{service = application_autoscaling, + method = Method, + request_headers = Headers, + request_body = Body, + uri = RequestURI}. + request_and_retry(_, _, _, {error, Reason}) -> {error, Reason}; request_and_retry(_, _, _, {attempt, Attempt}) when Attempt =:= 11 -> {error, no_answer}; request_and_retry(Config, Headers, Body, {attempt, Attempt}) -> case erlcloud_httpc:request(url(Config), post, Headers, Body, timeout(Attempt, Config), Config) of - {ok, {{200, _}, _, <<>>}} -> ok; - {ok, {{200, _}, _, RespBody}} -> {ok, jsx:decode(RespBody)}; - {error, {_, timeout}} -> request_and_retry(Config, Headers, Body, retry({attempt, Attempt + 1})); Error -> @@ -685,5 +715,4 @@ port_spec(#aws_config{autoscaling_port=Port}) -> headers(Config, Operation, Body) -> Headers = [{"host", Config#aws_config.autoscaling_host}, {"x-amz-target", Operation}], - erlcloud_aws:sign_v4_headers(Config, Headers, Body, erlcloud_aws:aws_region_from_host(Config#aws_config.autoscaling_host), "application-autoscaling"). From 8d8f7ed7ed08f11d571ff0ad31e704b54b2ff15c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos?= Date: Mon, 15 Jul 2019 09:39:49 +0100 Subject: [PATCH 067/310] Update to adhere to comments --- include/erlcloud_aws.hrl | 3 + src/erlcloud_application_autoscaler.erl | 135 +++++++++++++----------- 2 files changed, 74 insertions(+), 64 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 3f65a4a94..097e0df73 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -88,6 +88,9 @@ autoscaling_scheme="https://"::string(), autoscaling_host="autoscaling.us-east-1.amazonaws.com"::string(), autoscaling_port=80::non_neg_integer(), + application_autoscaling_scheme="https://"::string(), + application_autoscaling_host="autoscaling.us-east-1.amazonaws.com"::string(), + application_autoscaling_port=80::non_neg_integer(), directconnect_scheme="https://"::string(), directconnect_host="directconnect.us-east-1.amazonaws.com"::string(), directconnect_port=80::non_neg_integer(), diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index ee90f2ada..61edf4d5e 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -235,22 +235,22 @@ new(AccessKeyID, SecretAccessKey) -> new(AccessKeyID, SecretAccessKey, Host) -> #aws_config{access_key_id=AccessKeyID, secret_access_key=SecretAccessKey, - autoscaling_host=Host}. + application_autoscaling_host=Host}. -spec new(string(), string(), string(), non_neg_integer()) -> aws_config(). new(AccessKeyID, SecretAccessKey, Host, Port) -> #aws_config{access_key_id=AccessKeyID, secret_access_key=SecretAccessKey, - autoscaling_host=Host, - autoscaling_port=Port}. + application_autoscaling_host=Host, + application_autoscaling_port=Port}. -spec new(string(), string(), string(), non_neg_integer(), string()) -> aws_config(). new(AccessKeyID, SecretAccessKey, Host, Port, Scheme) -> #aws_config{access_key_id=AccessKeyID, secret_access_key=SecretAccessKey, - autoscaling_host=Host, - autoscaling_port=Port, - autoscaling_scheme=Scheme}. + application_autoscaling_host=Host, + application_autoscaling_port=Port, + application_autoscaling_scheme=Scheme}. -spec configure(string(), string()) -> ok. configure(AccessKeyID, SecretAccessKey) -> @@ -276,7 +276,12 @@ default_config() -> erlcloud_aws:default_config(). %%% API Documentation on AWS: https://docs.aws.amazon.com/autoscaling/application/APIReference/Welcome.html %%%------------------------------------------------------------------------------ -%% DeleteScalingPolicy +%%------------------------------------------------------------------------------ +%% @doc. +%% DeleteScalingPolicy. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_DeleteScalingPolicy.html +%%------------------------------------------------------------------------------ -spec delete_scaling_policy(Configuration :: aws_config(), PolicyName :: binary(), @@ -297,7 +302,12 @@ delete_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, delete_scaling_policy(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DescribeScalingPolicies"). -%% DeleteScheduledAction +%%------------------------------------------------------------------------------ +%% @doc. +%% DeleteScheduledAction. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_DeleteScheduledAction.html +%%------------------------------------------------------------------------------ -spec delete_scheduled_action( Configuration :: erlcloud_aws:aws_config(), @@ -320,7 +330,12 @@ delete_scheduled_action(Configuration, ResourceId, ScalableDimension, ScheduledA delete_scheduled_action(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DeleteScheduledAction"). -%% DeregisterScalableTarget +%%------------------------------------------------------------------------------ +%% @doc. +%% DeregisterScalableTarget. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_DeregisterScalableTarget.html +%%------------------------------------------------------------------------------ -spec deregister_scalable_target( Configuration :: erlcloud_aws:aws_config(), @@ -341,7 +356,12 @@ deregister_scalable_target(Configuration, ResourceId, ScalableDimension, Service deregister_scalable_target(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DeregisterScalableTarget"). -%% DescribeScalableTarget +%%------------------------------------------------------------------------------ +%% @doc. +%% DescribeScalableTargets. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_DescribeScalableTargets.html +%%------------------------------------------------------------------------------ -spec describe_scalable_targets( erlcloud_aws:aws_config(), @@ -364,7 +384,12 @@ describe_scalable_targets(Configuration, BodyConfiguration) -> {error, Error} end. -%% DescribeScalingActivities +%%------------------------------------------------------------------------------ +%% @doc. +%% DescribeScalingActivities. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_DescribeScalingActivities.html +%%------------------------------------------------------------------------------ -spec describe_scaling_activities( erlcloud_aws:aws_config(), @@ -387,7 +412,13 @@ describe_scaling_activities(Configuration, BodyConfiguration) -> {error, Error} -> {error, Error} end. -%% DescribeScalingPolicies + +%%------------------------------------------------------------------------------ +%% @doc. +%% DescribeScalingPolicies. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_DescribeScalingPolicies.html +%%------------------------------------------------------------------------------ -spec describe_scaling_policies( erlcloud_aws:aws_config(), @@ -411,7 +442,12 @@ describe_scaling_policies(Configuration, BodyConfiguration) -> end. -%% DescribeScheduledActions +%%------------------------------------------------------------------------------ +%% @doc. +%% DescribeScheduledActions. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_DescribeScheduledActions.html +%%------------------------------------------------------------------------------ -spec describe_scheduled_actions( erlcloud_aws:aws_config(), @@ -435,8 +471,12 @@ describe_scheduled_actions(Configuration, BodyConfiguration) -> end. - -%% PutScalingPolicy +%%------------------------------------------------------------------------------ +%% @doc. +%% PutScalingPolicy. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_PutScalingPolicy.html +%%------------------------------------------------------------------------------ -spec put_scaling_policy( Configuration :: aws_config(), @@ -488,7 +528,12 @@ put_scaling_policy(Configuration, BodyConfiguration) -> {error, Error} end. -%% PutScheduledAction +%%------------------------------------------------------------------------------ +%% @doc. +%% PutScheduledAction. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_PutScheduledAction.html +%%------------------------------------------------------------------------------ -spec put_scheduled_action( Configuration :: aws_config(), @@ -564,7 +609,12 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp put_scheduled_action(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.PutScheduledAction"). -%% RegisterScalableTarget +%%------------------------------------------------------------------------------ +%% @doc. +%% RegisterScalableTarget. +%% +%% https://docs.aws.amazon.com/autoscaling/application/APIReference/API_RegisterScalableTarget.html +%%------------------------------------------------------------------------------ -spec register_scalable_target(Configuration :: aws_config(), ResourceId :: binary(), @@ -641,14 +691,13 @@ request_with_action(Configuration, BodyConfiguration, Action) -> {ok, Config} -> HeadersPrev = headers(Config, Action, Body), Headers = [{"content-type", "application/x-amz-json-1.1"} | HeadersPrev], - Request = prepare_record(Config, post, Headers, Body, Action), + Request = prepare_record(Config, post, Headers, Body), erlcloud_retry:request(Config, Request, aas_result_fun/1); - %request_and_retry(Config, Headers, Body, {attempt, 0}); {error, Reason} -> {error, Reason} end. -prepare_record(Config, Method, Headers, Body, Action) -> +prepare_record(Config, Method, Headers, Body) -> %%% URI: awsConfig.Scheme + awsConfig.Host + [awsConfig.Port] %%% URI: https://autoscaling.us-west-2.amazonaws.com/ @@ -664,48 +713,6 @@ prepare_record(Config, Method, Headers, Body, Action) -> request_body = Body, uri = RequestURI}. -request_and_retry(_, _, _, {error, Reason}) -> - {error, Reason}; -request_and_retry(_, _, _, {attempt, Attempt}) when Attempt =:= 11 -> - {error, no_answer}; -request_and_retry(Config, Headers, Body, {attempt, Attempt}) -> - case erlcloud_httpc:request(url(Config), post, Headers, Body, timeout(Attempt, Config), Config) of - {ok, {{200, _}, _, <<>>}} -> - ok; - {ok, {{200, _}, _, RespBody}} -> - {ok, jsx:decode(RespBody)}; - {error, {_, timeout}} -> - request_and_retry(Config, Headers, Body, retry({attempt, Attempt + 1})); - Error -> - request_and_retry(Config, Headers, Body, retry(Error)) - end. - -retry({attempt, Attempt}) when Attempt >= ?NUM_ATTEMPTS -> - {error, max_limit_of_attempts}; -retry({attempt, Attempt}) -> - backoff(Attempt), - {attempt, Attempt + 1}. - -%% Sleep after an attempt --spec backoff(pos_integer()) -> ok. -backoff(1) -> ok; -backoff(Attempt) -> - timer:sleep(erlcloud_util:rand_uniform((1 bsl (Attempt - 1)) * 100)). - --spec timeout(pos_integer(), aws_config()) -> pos_integer(). -%% HTTPC timeout for a request -timeout(1, #aws_config{timeout = undefined}) -> - %% Shorter timeout on first request. This is to avoid long (5s) failover when first DDB - %% endpoint doesn't respond - 1000; -timeout(_, #aws_config{} = Cfg) -> - erlcloud_aws:get_timeout(Cfg). - - --spec url(aws_config()) -> []. -url(#aws_config{autoscaling_scheme = Scheme, autoscaling_host = Host} = Config) -> - lists:flatten([Scheme, Host, port_spec(Config)]). - port_spec(#aws_config{autoscaling_port=80}) -> ""; port_spec(#aws_config{autoscaling_port=Port}) -> @@ -713,6 +720,6 @@ port_spec(#aws_config{autoscaling_port=Port}) -> headers(Config, Operation, Body) -> - Headers = [{"host", Config#aws_config.autoscaling_host}, + Headers = [{"host", Config#aws_config.application_autoscaling_host}, {"x-amz-target", Operation}], - erlcloud_aws:sign_v4_headers(Config, Headers, Body, erlcloud_aws:aws_region_from_host(Config#aws_config.autoscaling_host), "application-autoscaling"). + erlcloud_aws:sign_v4_headers(Config, Headers, Body, erlcloud_aws:aws_region_from_host(Config#aws_config.application_autoscaling_host), "application-autoscaling"). From d197504e9ee97770ef99888683c9725ba2258bd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos?= Date: Mon, 15 Jul 2019 10:36:39 +0100 Subject: [PATCH 068/310] Update service_config --- src/erlcloud_application_autoscaler.erl | 12 +++++++----- src/erlcloud_aws.erl | 4 ++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 61edf4d5e..4b693d0f6 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -682,6 +682,8 @@ aas_result_fun(#aws_request{response_type = error, Status =:= 409; Status >= 500 -> Request#aws_request{should_retry = true}; aas_result_fun(#aws_request{response_type = error, error_type = aws} = Request) -> + Request#aws_request{should_retry = false}; +aas_result_fun(Request) -> Request#aws_request{should_retry = false}. @@ -692,7 +694,7 @@ request_with_action(Configuration, BodyConfiguration, Action) -> HeadersPrev = headers(Config, Action, Body), Headers = [{"content-type", "application/x-amz-json-1.1"} | HeadersPrev], Request = prepare_record(Config, post, Headers, Body), - erlcloud_retry:request(Config, Request, aas_result_fun/1); + erlcloud_retry:request(Config, Request, fun aas_result_fun/1); {error, Reason} -> {error, Reason} end. @@ -702,8 +704,8 @@ prepare_record(Config, Method, Headers, Body) -> %%% URI: https://autoscaling.us-west-2.amazonaws.com/ RequestURI = lists:flatten([ - Config#aws_config.autoscaling_scheme, - Config#aws_config.autoscaling_host, + Config#aws_config.application_autoscaling_scheme, + Config#aws_config.application_autoscaling_host, port_spec(Config) ]), @@ -713,9 +715,9 @@ prepare_record(Config, Method, Headers, Body) -> request_body = Body, uri = RequestURI}. -port_spec(#aws_config{autoscaling_port=80}) -> +port_spec(#aws_config{application_autoscaling_port=80}) -> ""; -port_spec(#aws_config{autoscaling_port=Port}) -> +port_spec(#aws_config{application_autoscaling_port=Port}) -> [":", erlang:integer_to_list(Port)]. diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 73a114ad3..e405cd5a5 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -765,8 +765,12 @@ service_config( <<"guardduty">> = Service, Region, Config ) -> Config#aws_config{guardduty_host = Host}; service_config( <<"cur">>, Region, Config ) -> Host = service_host(<<"cur">>, Region), + Config#aws_config{cur_host = Host}; +service_config( <<"application-autoscaling">>, Region, Config ) -> + Host = service_host(<<"application-autoscaling">>, Region), Config#aws_config{cur_host = Host}. + %%%--------------------------------------------------------------------------- -spec service_host( Service :: binary(), Region :: binary() ) -> string(). From bf426e48394106ec447f965ba18557afe3eac786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos?= Date: Mon, 15 Jul 2019 11:01:27 +0100 Subject: [PATCH 069/310] Adhere to comments for the pull request --- src/erlcloud_application_autoscaler.erl | 3 ++- src/erlcloud_aws.erl | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 4b693d0f6..a3535ab05 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -694,7 +694,8 @@ request_with_action(Configuration, BodyConfiguration, Action) -> HeadersPrev = headers(Config, Action, Body), Headers = [{"content-type", "application/x-amz-json-1.1"} | HeadersPrev], Request = prepare_record(Config, post, Headers, Body), - erlcloud_retry:request(Config, Request, fun aas_result_fun/1); + Response = erlcloud_retry:request(Config, Request, fun aas_result_fun/1), + {ok, jsx:decode(Response#aws_request.response_body)}; {error, Reason} -> {error, Reason} end. diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index e405cd5a5..a40d69961 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -766,8 +766,8 @@ service_config( <<"guardduty">> = Service, Region, Config ) -> service_config( <<"cur">>, Region, Config ) -> Host = service_host(<<"cur">>, Region), Config#aws_config{cur_host = Host}; -service_config( <<"application-autoscaling">>, Region, Config ) -> - Host = service_host(<<"application-autoscaling">>, Region), +service_config( <<"application_autoscaling">>, Region, Config ) -> + Host = service_host(<<"application_autoscaling">>, Region), Config#aws_config{cur_host = Host}. From 13621724bcf898f1d0d304006f707c1cc073985b Mon Sep 17 00:00:00 2001 From: Sonic Gao Date: Wed, 17 Jul 2019 14:09:19 +0800 Subject: [PATCH 070/310] support all regions for AWS China --- src/erlcloud_aws.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index a40d69961..3aaed9936 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -155,6 +155,7 @@ aws_region_from_host(Host) -> %% we need to account for that: %% us-west-2: s3.us-west-2.amazonaws.com %% cn-north-1 (AWS China): s3.cn-north-1.amazonaws.com.cn + %% cn-northwest-1 (AWS China): s3.cn-northwest-1.amazonaws.com.cn %% it's assumed that the first element is the aws service (s3, ec2, etc), %% the second is the region identifier, the rest is ignored %% the exception (of course) is the dynamodb streams and the marketplace which follows a @@ -688,6 +689,9 @@ service_config( <<"emr">>, Region, Config ) -> service_config( <<"iam">> = Service, <<"cn-north-1">> = Region, Config ) -> Host = service_host( Service, Region ), Config#aws_config{ iam_host = Host }; +service_config( <<"iam">> = Service, <<"cn-northwest-1">> = Region, Config ) -> + Host = service_host( Service, Region ), + Config#aws_config{ iam_host = Host }; service_config( <<"iam">>, _Region, Config ) -> Config; service_config( <<"inspector">> = Service, Region, Config ) -> Host = service_host( Service, Region ), @@ -782,14 +786,20 @@ service_config( <<"application_autoscaling">>, Region, Config ) -> %% names. %% service_host( <<"s3">>, <<"us-east-1">> ) -> "s3-external-1.amazonaws.com"; +service_host( <<"s3">>, <<"us-gov-west-1">> ) -> "s3-fips-us-gov-west-1.amazonaws.com"; service_host( <<"s3">>, <<"cn-north-1">> ) -> "s3.cn-north-1.amazonaws.com.cn"; -service_host( <<"s3">>, <<"us-gov-west-1">> ) -> - "s3-fips-us-gov-west-1.amazonaws.com"; +service_host( <<"s3">>, <<"cn-northwest-1">> ) -> "s3.cn-northwest-1.amazonaws.com.cn"; service_host( <<"s3">>, Region ) -> binary_to_list( <<"s3-", Region/binary, ".amazonaws.com">> ); +service_host( <<"iam">>, <<"cn-north-1">> ) -> "iam.amazonaws.com.cn"; +service_host( <<"iam">>, <<"cn-northwest-1">> ) -> "iam.amazonaws.com.cn"; service_host( <<"sdb">>, <<"us-east-1">> ) -> "sdb.amazonaws.com"; service_host( <<"states">>, Region ) -> binary_to_list( <<"states.", Region/binary, ".amazonaws.com">> ); +service_host( Service, <<"cn-north-1">> = Region ) when is_binary(Service) -> + binary_to_list( <> ); +service_host( Service, <<"cn-northwest-1">> =Region ) when is_binary(Service) -> + binary_to_list( <> ); service_host( Service, Region ) when is_binary(Service) -> binary_to_list( <> ). From b01f06775badc79905eff3260be53097afdf8427 Mon Sep 17 00:00:00 2001 From: Vitor Andriotti Date: Wed, 17 Jul 2019 18:11:42 -0300 Subject: [PATCH 071/310] Make 'TransactionCanceledException' error handling extract 'CancellationReasons' values and return it in the error In order to make transaction error handling more pattern-match friendly, this change makes the `client_error' handler extract `CancellationReasons` values from `TransactionCanceledException` and return the error as `{error, {Type, {Message, CancellationReasons}}}` instead of just `{error, {Type, Message}}`. **NOTE** that this output change applies to `TransactionCanceledException` error type only. --- src/erlcloud_ddb2.erl | 22 +++++++++++++-- src/erlcloud_ddb_impl.erl | 19 +++++++++++-- test/erlcloud_ddb2_tests.erl | 55 +++++++++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 34e547600..2b0af2d95 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -61,12 +61,18 @@ %% contain typed attribute values so that they may be correctly passed %% to subsequent calls. %% -%% DynamoDB errors are return in the form `{error, {ErrorCode, -%% Message}}' where `ErrorCode' and 'Message' are both binary +%% DynamoDB errors for most cases are returned in the form +%% `{error, {ErrorCode, Message}}' where `ErrorCode' and `Message' are both binary %% strings. List of error codes: %% [http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ErrorHandling.html]. So %% to handle conditional check failures, match `{error, %% {<<"ConditionalCheckFailedException">>, _}}'. +%% Note that in the case of a `TransactionCanceledException' DynamoDB error, +%% the error response has the form `{error, {<<"TransactionCanceledException">>, +%% {Message, CancellationReasons}}}' where `Message' is a binary string and +%% `CancellationReasons' is an ordered list in the form `[{Code, Message}]', +%% where `Code' is the status code of the result and `Message' is the cancellation +%% reason message description. %% %% `erlcloud_ddb_util' provides a higher level API that implements common %% operations that may require multiple DynamoDB API calls. @@ -3213,6 +3219,12 @@ transact_get_items(RequestItems, Opts) -> %% DynamoDB API: %% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactGetItems.html] %% +%% Note that in the case of a `TransactionCanceledException' DynamoDB error, the error +%% response has the form `{error, {<<"TransactionCanceledException">>, {Message, CancellationReasons}}}' +%% where `Message' is a binary string and `CancellationReasons' is an ordered list in the form +%% `[{Code, Message}]', where `Code' is the status code of the result and `Message' is the cancellation +%% reason message description. +%% %% ===Example=== %% %% Get two items in a transaction. @@ -3348,6 +3360,12 @@ transact_write_items(RequestItems, Opts) -> %% DynamoDB API: %% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_TransactWriteItems.html] %% +%% Note that in the case of a `TransactionCanceledException' DynamoDB error, the error +%% response has the form `{error, {<<"TransactionCanceledException">>, {Message, CancellationReasons}}}' +%% where `Message' is a binary string and `CancellationReasons' is an ordered list in the form +%% `[{Code, Message}]', where `Code' is the status code of the result and `Message' is the cancellation +%% reason message description. +%% %% ===Example=== %% %% Put two items in a transaction. diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 7ba4c03f1..79bc30150 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -240,11 +240,19 @@ client_error(Body, DDBError) -> [_, Type] when Type =:= <<"ProvisionedThroughputExceededException">> orelse Type =:= <<"ThrottlingException">> -> - DDBError#ddb2_error{error_type = ddb, + DDBError#ddb2_error{error_type = ddb, should_retry = true, reason = {Type, Message}}; + [_, Type] when + Type =:= <<"TransactionCanceledException">> -> + CancellationReasons0 = proplists:get_value(<<"CancellationReasons">>, Json, []), + CancellationReasons = [{proplists:get_value(<<"Code">>, R), + proplists:get_value(<<"Message">>, R, null)} || R <- CancellationReasons0], + DDBError#ddb2_error{error_type = ddb, + should_retry = should_retry_canceled_transaction(CancellationReasons), + reason = {Type, {Message, CancellationReasons}}}; [_, Type] -> - DDBError#ddb2_error{error_type = ddb, + DDBError#ddb2_error{error_type = ddb, should_retry = false, reason = {Type, Message}}; _ -> @@ -268,3 +276,10 @@ port_spec(#aws_config{ddb_port=80}) -> port_spec(#aws_config{ddb_port=Port}) -> [":", erlang:integer_to_list(Port)]. +-spec should_retry_canceled_transaction(proplists:proplist()) -> boolean(). +should_retry_canceled_transaction(CancellationReasons) -> + %% Retry canceled transaction if cancellation reasons are either: + %% `None', `ThrottlingError' and/or `ProvisionedThroughputExceeded' + lists:filter(fun({Reason, _Message}) -> + not lists:member(Reason, [<<"None">>, <<"ThrottlingError">>, <<"ProvisionedThroughputExceeded">>]) + end, CancellationReasons) == []. diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index 81fcb4369..d96c4d18f 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -280,8 +280,61 @@ error_handling_tests(_) -> OkResult}) ], - error_tests(?_f(erlcloud_ddb2:get_item(<<"table">>, {<<"k">>, <<"v">>})), Tests). + error_tests(?_f(erlcloud_ddb2:get_item(<<"table">>, {<<"k">>, <<"v">>})), Tests), + TransactionErrorResult = {error, {<<"TransactionCanceledException">>, + {<<"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]">>, + [{<<"None">>, null}, {<<"ConditionalCheckFailed">>, <<"The conditional request failed">>}]}}}, + + TransactOkResponse = httpc_response(200, "{}"), + TransactOkResult = {ok, []}, + + TransactTests = + [?_ddb_test( + {"Test return output for TransactionCanceledException error", + [httpc_response(400, " +{ + \"__type\":\"com.amazonaws.dynamodb.v20120810#TransactionCanceledException\", + \"CancellationReasons\": [ + { + \"Code\":\"None\", + }, + { + \"Code\":\"ConditionalCheckFailed\", + \"Message\":\"The conditional request failed\", + } + ], + \"message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]\" +}" + )], + TransactionErrorResult}), + ?_ddb_test( + {"Test retry after ProvisionedThroughputExceeded cancellation reason", + [httpc_response(400, " +{ + \"__type\":\"com.amazonaws.dynamodb.v20120810#TransactionCanceledException\", + \"CancellationReasons\": [ + { + \"Code\":\"None\", + }, + { + \"Code\":\"ProvisionedThroughputExceeded\", + \"Message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\", + } + ], + \"message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ProvisionedThroughputExceeded]\" +}" + ), + TransactOkResponse], + TransactOkResult}) + ], + + Transact = [{put, {<<"table">>, [{<<"k">>, {s, <<"v">>}}], []}}, + {condition_check, {<<"table">>, [{<<"k">>, {s, <<"v2">>}}], + [{condition_expression, <<"approved = :a">>}, + {expression_attribute_values, [{<<":a">>, <<"yes">>}]}]}}], + + error_tests(?_f(erlcloud_ddb2:transact_write_items(Transact)), TransactTests). %% BatchGetItem test based on the API examples: %% http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchGetItem.html From 6dce88317e9231bd85f6ce8d9be0db33035520ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joa=CC=83o=20Santos?= Date: Fri, 26 Jul 2019 14:59:17 +0100 Subject: [PATCH 072/310] Add missing fields to scheduled actions --- src/erlcloud_application_autoscaler.erl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index a3535ab05..567ad23a3 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -212,7 +212,10 @@ extract_scheduled_action(J) -> {scalable_dimension, <<"ScalableDimension">>, optional_string}, {service_namespace, <<"ServiceNamespace">>, optional_string}, {scheduled_action_arn, <<"ScheduledActionARN">>, optional_string}, - {scheduled_action_name, <<"ScheduledActionName">>, optional_string} + {scheduled_action_name, <<"ScheduledActionName">>, optional_string}, + {start_time, <<"StartTime">>, optional_integer}, + {end_time, <<"EndTime">>, optional_integer}, + {schedule, <<"Schedule">>, optional_string} ], J), Action = proplists:get_value(<<"ScalableTargetAction">>, J), PropAction = erlcloud_json:decode([ From 168772bc685560d1772c48625c7444aeaeda4d53 Mon Sep 17 00:00:00 2001 From: Stephane Bourque Date: Wed, 31 Jul 2019 10:13:25 -0700 Subject: [PATCH 073/310] Adding coverage for dealing with Custom Verification Templates. Support added for Create,Delete,Update,List,Get --- src/erlcloud_ses.erl | 150 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 146 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index 7aa11521e..6f877d1cd 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -43,8 +43,21 @@ -export([verify_email_identity/1, verify_email_identity/2]). -export([verify_domain_identity/1, verify_domain_identity/2]). --include("erlcloud.hrl"). --include("erlcloud_aws.hrl"). +-export([ create_custom_verification_email_template/6, + create_custom_verification_email_template/7, + update_custom_verification_email_template/2, + update_custom_verification_email_template/3, + send_custom_verification_email/3, + delete_custom_verification_email_template/1, + delete_custom_verification_email_template/2, + get_custom_verification_email_template/2, + get_custom_verification_email_template/1, + list_custom_verification_email_templates/0, + list_custom_verification_email_templates/1]). + + +-include("../include/erlcloud.hrl"). +-include("../include/erlcloud_aws.hrl"). -define(API_VERSION, "2010-12-01"). @@ -132,6 +145,104 @@ delete_identity(Identity, Config) -> {error, Reason} -> {error, Reason} end. +%%%------------------------------------------------------------------------------ +%%% Custom Verification Templates +%%% +%%% Template attributes: +%%% { template_name , string() } +%%% { from_email_address , string() } +%%% { template_subject , string() } +%%% { template_content , string() } -- please see notes in API Guide on what is allowed +%%% { success_redirect_url , string() } +%%% { failure_redirect_url , string() } +%%% +%%% On template creation, all attributes are mandatory. +%%% On template updates, only include the attributes you need to modify +%%%------------------------------------------------------------------------------ + +create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL ) -> + create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL , default_config() ). + +create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL ,Config ) -> + Params = encode_params( + [ { template_name , TemplateName} , + { from_email_address , FromEmailAddress } , + { template_subject , TemplateSubject } , + { template_content , TemplateContent } , + { success_redirect_url , SuccessRedirectionURL} , + { failure_redirect_url , FailureRedirectionURL} ] ), + case ses_request(Config, "CreateCustomVerificationEmailTemplate", Params) of + {ok, _Doc} -> + ok; + {error, Reason} -> {error, Reason} + end. + +update_custom_verification_email_template( TemplateName , Attributes ) -> + update_custom_verification_email_template( TemplateName , Attributes , default_config() ). + +update_custom_verification_email_template( TemplateName , Attributes , Config ) -> + Params = encode_params( [ { template_name , TemplateName } | Attributes ] ) , + case ses_request(Config, "UpdateCustomVerificationEmailTemplate", Params) of + {ok, _Doc} -> + ok ; + {error, Reason} -> {error, Reason} + end. + +send_custom_verification_email( EmailAddress , TemplateName , Config ) -> + Params = encode_params( [ { email_address , EmailAddress} , + { template_name , TemplateName} ] ), + case ses_request(Config, "SendCustomVerificationEmail", Params) of + {ok, Doc} -> + {ok, erlcloud_xml:decode([{message_id, "SendCustomVerificationEmailResult/MessageId", text}], Doc)}; + {error, Reason} -> {error, Reason} + end. + +delete_custom_verification_email_template( TemplateName ) -> + delete_custom_verification_email_template( TemplateName , default_config() ). + +delete_custom_verification_email_template( TemplateName , Config ) -> + Params = encode_params( [ { template_name , TemplateName } ] ), + case ses_request(Config, "DeleteCustomVerificationEmailTemplate", Params) of + { ok , _Doc } -> + ok ; + { error , Reason } -> + { error , Reason } + end. + +get_custom_verification_email_template( TemplateName ) -> + get_custom_verification_email_template( TemplateName , default_config() ). + +get_custom_verification_email_template( TemplateName , Config ) -> + Params = encode_params( [ { template_name , TemplateName } ] ), + case ses_request(Config, "GetCustomVerificationEmailTemplate", Params) of + { ok , Doc } -> + {ok, erlcloud_xml:decode( + [ + {template_name , "GetCustomVerificationEmailTemplateResult/TemplateName", text }, + {from_email_address , "GetCustomVerificationEmailTemplateResult/FromEmailAddress", text }, + {template_subject , "GetCustomVerificationEmailTemplateResult/TemplateSubject", text }, + {template_content , "GetCustomVerificationEmailTemplateResult/TemplateContent", text }, + {success_redirect_url, "GetCustomVerificationEmailTemplateResult/SuccessRedirectionURL", text }, + {failure_redirect_url, "GetCustomVerificationEmailTemplateResult/FailureRedirectionURL", text } + ], + Doc)}; + { error , Reason } -> + {error,Reason} + end. + +list_custom_verification_email_templates() -> + list_custom_verification_email_templates(default_config() ). + +list_custom_verification_email_templates( Config ) -> + Params = [ { "MaxResults" , 50 } ] , + case ses_request(Config, "ListCustomVerificationEmailTemplates", Params) of + { ok , Doc } -> + {ok, erlcloud_xml:decode([{custom_templates, "ListCustomVerificationEmailTemplatesResult/CustomVerificationEmailTemplates/member", fun decode_custom_template_entry/1}, + {next_token, "ListCustomVerificationEmailTemplatesResult/NextToken", optional_text}], + Doc)}; + { error , Reason } -> + {error,Reason} + end. %%%------------------------------------------------------------------------------ %%% GetIdentityDkimAttributes @@ -794,8 +905,23 @@ encode_params([{source, Source} | T], Acc) when is_list(Source); is_binary(Sourc encode_params(T, [{"Source", Source} | Acc]); encode_params([{subject, Subject} | T], Acc) -> encode_params(T, encode_content("Message.Subject", Subject, Acc)); +encode_params([{template_name, TemplateName} | T], Acc) when is_list(TemplateName); is_binary(TemplateName) -> + encode_params(T, [{"TemplateName", TemplateName} | Acc]); +encode_params([{from_email_address, FromEmailAddress} | T], Acc) when is_list(FromEmailAddress); is_binary(FromEmailAddress) -> + encode_params(T, [{"FromEmailAddress", FromEmailAddress} | Acc]); +encode_params([{template_subject, TemplateSubject} | T], Acc) when is_list(TemplateSubject); is_binary(TemplateSubject) -> + encode_params(T, [{"TemplateSubject", TemplateSubject} | Acc]); +encode_params([{template_content, TemplateContent} | T], Acc) when is_list(TemplateContent); is_binary(TemplateContent) -> + encode_params(T, [{"TemplateContent", TemplateContent} | Acc]); +encode_params([{success_redirect_url, SuccessRedirectionURL} | T], Acc) when is_list(SuccessRedirectionURL); is_binary(SuccessRedirectionURL) -> + encode_params(T, [{"SuccessRedirectionURL", SuccessRedirectionURL} | Acc]); +encode_params([{failure_redirect_url, FailureRedirectionURL} | T], Acc) when is_list(FailureRedirectionURL); is_binary(FailureRedirectionURL) -> + encode_params(T, [{"FailureRedirectionURL", FailureRedirectionURL} | Acc]); encode_params([Option | _], _Acc) -> - error({erlcloud_ses, {invalid_parameter, Option}}). + error({erlcloud_ses, {invalid_parameter, Option}}). + + + encode_list(Prefix, List, Acc) -> encode_list(Prefix, List, 1, Acc). @@ -926,6 +1052,15 @@ decode_send_data_points(SendDataPointsDoc) -> Entry) || Entry <- SendDataPointsDoc]. +decode_custom_template_entry(CustomTemplatesDoc) -> + [ erlcloud_xml:decode([ + { template_name , "TemplateName" , text } , + { from_email_address , "FromEmailAddress" , text } , + { template_subject , "TemplateSubject" , text } , + { success_redirect_url , "SuccessRedirectionURL" , text } , + { failure_redirect_url , "FailureRedirectionURL" , text } ], + Entry) + || Entry <- CustomTemplatesDoc ]. decode_error_code("IncompleteSignature") -> incomplete_signature; decode_error_code("InternalFailure") -> internal_failure; @@ -943,7 +1078,14 @@ decode_error_code("OptInRequired") -> opt_in_required; decode_error_code("RequestExpired") -> request_expired; decode_error_code("ServiceUnavailable") -> service_unavailable; decode_error_code("Throttling") -> throttling; -decode_error_code("ValidationError") -> validation_error. +decode_error_code("ValidationError") -> validation_error; +decode_error_code("LimitExceeded") -> limit_exceeded; +decode_error_code("ConfigurationSetDoesNotExist") -> configuration_set_does_not_exist; +decode_error_code("CustomVerificationEmailTemplateDoesNotExist") -> custom_verification_email_template_does_not_exist; +decode_error_code("ProductionAccessNotGranted") -> production_access_not_granted; +decode_error_code("CustomVerificationEmailInvalidContent") -> custom_verification_email_invalid_content; +decode_error_code("FromEmailAddressNotVerified") -> from_email_address_not_verified; +decode_error_code("CustomVerificationEmailTemplateAlreadyExists") -> custom_verification_email_template_already_exists. decode_error(Doc) -> From c6a80bf89d9533b5ba1c1dfd7cdd5bcfb0a5e5b1 Mon Sep 17 00:00:00 2001 From: Stephane Bourque Date: Wed, 31 Jul 2019 10:29:24 -0700 Subject: [PATCH 074/310] Remove duplicate includes --- src/erlcloud_ses.erl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index 6f877d1cd..c7970b9b2 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -55,9 +55,8 @@ list_custom_verification_email_templates/0, list_custom_verification_email_templates/1]). - --include("../include/erlcloud.hrl"). --include("../include/erlcloud_aws.hrl"). +-include("erlcloud.hrl"). +-include("erlcloud_aws.hrl"). -define(API_VERSION, "2010-12-01"). From dbdb731fae4f0439e546c5b076bb2bfd662b1995 Mon Sep 17 00:00:00 2001 From: Stephane Bourque Date: Wed, 31 Jul 2019 11:11:25 -0700 Subject: [PATCH 075/310] Added spec/types for all added fcuntions --- src/erlcloud_ses.erl | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index c7970b9b2..ea1cec878 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -47,6 +47,7 @@ create_custom_verification_email_template/7, update_custom_verification_email_template/2, update_custom_verification_email_template/3, + send_custom_verification_email/2, send_custom_verification_email/3, delete_custom_verification_email_template/1, delete_custom_verification_email_template/2, @@ -159,9 +160,16 @@ delete_identity(Identity, Config) -> %%% On template updates, only include the attributes you need to modify %%%------------------------------------------------------------------------------ +-type custom_template_attribute_names() :: template_name | from_email_address | template_subject | template_content | success_redirect_url | failure_redirect_url . +-type custom_template_attributes() :: [ { custom_template_attribute_names() , string() } ]. + +-type create_custom_verification_email_template_result() :: ok | { error , term() }. + +-spec create_custom_verification_email_template( string() , string() ,string() ,string() ,string() ,string() ) -> create_custom_verification_email_template_result(). create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL ) -> create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL , default_config() ). +-spec create_custom_verification_email_template( string() , string() ,string() ,string() ,string() ,string() , aws_config() ) -> create_custom_verification_email_template_result(). create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL ,Config ) -> Params = encode_params( [ { template_name , TemplateName} , @@ -176,9 +184,13 @@ create_custom_verification_email_template( TemplateName , FromEmailAddress , Tem {error, Reason} -> {error, Reason} end. +-type update_custom_verification_email_template_result() :: ok | { error , term() }. + +-spec update_custom_verification_email_template( string() , [ { atom() , string() }] ) -> update_custom_verification_email_template_result(). update_custom_verification_email_template( TemplateName , Attributes ) -> update_custom_verification_email_template( TemplateName , Attributes , default_config() ). +-spec update_custom_verification_email_template( string() , [ { atom() , string() } ] , aws_config() ) -> update_custom_verification_email_template_result(). update_custom_verification_email_template( TemplateName , Attributes , Config ) -> Params = encode_params( [ { template_name , TemplateName } | Attributes ] ) , case ses_request(Config, "UpdateCustomVerificationEmailTemplate", Params) of @@ -187,6 +199,13 @@ update_custom_verification_email_template( TemplateName , Attributes , Config ) {error, Reason} -> {error, Reason} end. +-type send_custom_verification_email_result() :: { ok , term() } | { error , term() }. + +-spec send_custom_verification_email( string() , string() ) -> send_custom_verification_email_result(). +send_custom_verification_email( EmailAddress , TemplateName ) -> + send_custom_verification_email( EmailAddress , TemplateName , default_config() ). + +-spec send_custom_verification_email( string() , string() , aws_config()) -> send_custom_verification_email_result(). send_custom_verification_email( EmailAddress , TemplateName , Config ) -> Params = encode_params( [ { email_address , EmailAddress} , { template_name , TemplateName} ] ), @@ -196,9 +215,13 @@ send_custom_verification_email( EmailAddress , TemplateName , Config ) -> {error, Reason} -> {error, Reason} end. +-type delete_custom_verification_email_template_result() :: ok | {error,term()}. + +-spec delete_custom_verification_email_template( string() ) -> delete_custom_verification_email_template_result(). delete_custom_verification_email_template( TemplateName ) -> delete_custom_verification_email_template( TemplateName , default_config() ). +-spec delete_custom_verification_email_template( string() , aws_config() ) -> delete_custom_verification_email_template_result(). delete_custom_verification_email_template( TemplateName , Config ) -> Params = encode_params( [ { template_name , TemplateName } ] ), case ses_request(Config, "DeleteCustomVerificationEmailTemplate", Params) of @@ -208,9 +231,13 @@ delete_custom_verification_email_template( TemplateName , Config ) -> { error , Reason } end. +-type get_custom_verification_email_template_result() :: { ok , custom_template_attributes() } | { error , term() }. + +-spec get_custom_verification_email_template( string() ) -> get_custom_verification_email_template_result(). get_custom_verification_email_template( TemplateName ) -> get_custom_verification_email_template( TemplateName , default_config() ). +-spec get_custom_verification_email_template( string() , aws_config() ) -> get_custom_verification_email_template_result(). get_custom_verification_email_template( TemplateName , Config ) -> Params = encode_params( [ { template_name , TemplateName } ] ), case ses_request(Config, "GetCustomVerificationEmailTemplate", Params) of @@ -229,9 +256,13 @@ get_custom_verification_email_template( TemplateName , Config ) -> {error,Reason} end. +-type list_custom_verification_email_templates_result() :: { ok , [ { custom_templates , [ custom_template_attributes() ] } ] } | { error , term() }. + +-spec list_custom_verification_email_templates() -> list_custom_verification_email_templates_result(). list_custom_verification_email_templates() -> list_custom_verification_email_templates(default_config() ). +-spec list_custom_verification_email_templates( aws_config() ) -> list_custom_verification_email_templates_result(). list_custom_verification_email_templates( Config ) -> Params = [ { "MaxResults" , 50 } ] , case ses_request(Config, "ListCustomVerificationEmailTemplates", Params) of @@ -1051,6 +1082,9 @@ decode_send_data_points(SendDataPointsDoc) -> Entry) || Entry <- SendDataPointsDoc]. +%% Added for decoding a custom template entry with the ListCustomVerificationEmailTemplates command +%% Notice that the ListCustomVerificationEmailTemplates API does not return the TemplateContent +%% You must use GetCustomVerificationEmailTemplate to retrieve TemplateContent decode_custom_template_entry(CustomTemplatesDoc) -> [ erlcloud_xml:decode([ { template_name , "TemplateName" , text } , From 2b6d7898e4f3c955494722b752b41b6b7429ac37 Mon Sep 17 00:00:00 2001 From: Stephane Bourque Date: Wed, 31 Jul 2019 11:14:20 -0700 Subject: [PATCH 076/310] Added a link to the actual API --- src/erlcloud_ses.erl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index ea1cec878..237165f24 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -146,6 +146,7 @@ delete_identity(Identity, Config) -> end. %%%------------------------------------------------------------------------------ +%%% %%% Custom Verification Templates %%% %%% Template attributes: @@ -158,6 +159,11 @@ delete_identity(Identity, Config) -> %%% %%% On template creation, all attributes are mandatory. %%% On template updates, only include the attributes you need to modify +%%% +%%% +%%% Please consult: https://docs.aws.amazon.com/ses/latest/APIReference/API_GetCustomVerificationEmailTemplate.html +%%% for a full reference to these APIs +%%% %%%------------------------------------------------------------------------------ -type custom_template_attribute_names() :: template_name | from_email_address | template_subject | template_content | success_redirect_url | failure_redirect_url . From 17ac88146ac83ffaf554982fff0769f9dff66cd3 Mon Sep 17 00:00:00 2001 From: Stephane Bourque Date: Wed, 31 Jul 2019 11:23:18 -0700 Subject: [PATCH 077/310] Some comments --- src/erlcloud_ses.erl | 47 +++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index 237165f24..eff448874 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -145,26 +145,33 @@ delete_identity(Identity, Config) -> {error, Reason} -> {error, Reason} end. -%%%------------------------------------------------------------------------------ -%%% -%%% Custom Verification Templates -%%% -%%% Template attributes: -%%% { template_name , string() } -%%% { from_email_address , string() } -%%% { template_subject , string() } -%%% { template_content , string() } -- please see notes in API Guide on what is allowed -%%% { success_redirect_url , string() } -%%% { failure_redirect_url , string() } -%%% -%%% On template creation, all attributes are mandatory. -%%% On template updates, only include the attributes you need to modify -%%% -%%% -%%% Please consult: https://docs.aws.amazon.com/ses/latest/APIReference/API_GetCustomVerificationEmailTemplate.html -%%% for a full reference to these APIs -%%% -%%%------------------------------------------------------------------------------ +%%------------------------------------------------------------------------------ +%% @doc +%% Custom Verification Templates +%% SES API: +%% [https://docs.aws.amazon.com/ses/latest/APIReference/API_GetCustomVerificationEmailTemplate.html] +%% +%% Template attributes: +%% { template_name , string() } +%% { from_email_address , string() } +%% { template_subject , string() } +%% { template_content , string() } -- please see notes in API Guide on what is allowed +%% { success_redirect_url , string() } +%% { failure_redirect_url , string() } +%% +%% On template creation, all attributes are mandatory. +%% On template updates, only include the attributes you need to modify +%% When listing templates, all attributes are included except template_content, use the get function to retrieve it +%% +%% create_custom_verification_email_template +%% update_custom_verification_email_template +%% send_custom_verification_email +%% delete_custom_verification_email_template +%% get_custom_verification_email_template +%% list_custom_verification_email_templates +%% +%% @end +%%------------------------------------------------------------------------------ -type custom_template_attribute_names() :: template_name | from_email_address | template_subject | template_content | success_redirect_url | failure_redirect_url . -type custom_template_attributes() :: [ { custom_template_attribute_names() , string() } ]. From c6712836fa00af667df8bd3d1d5dc3677741619c Mon Sep 17 00:00:00 2001 From: Stephane Bourque Date: Wed, 31 Jul 2019 21:41:12 -0700 Subject: [PATCH 078/310] Fixed some style issues and formatting. --- src/erlcloud_ses.erl | 189 +++++++++++++++++++++---------------------- 1 file changed, 90 insertions(+), 99 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index eff448874..67e71b6c3 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -43,18 +43,12 @@ -export([verify_email_identity/1, verify_email_identity/2]). -export([verify_domain_identity/1, verify_domain_identity/2]). --export([ create_custom_verification_email_template/6, - create_custom_verification_email_template/7, - update_custom_verification_email_template/2, - update_custom_verification_email_template/3, - send_custom_verification_email/2, - send_custom_verification_email/3, - delete_custom_verification_email_template/1, - delete_custom_verification_email_template/2, - get_custom_verification_email_template/2, - get_custom_verification_email_template/1, - list_custom_verification_email_templates/0, - list_custom_verification_email_templates/1]). +-export([create_custom_verification_email_template/6, create_custom_verification_email_template/7]). +-export([update_custom_verification_email_template/2, update_custom_verification_email_template/3]). +-export([send_custom_verification_email/2, send_custom_verification_email/3]). +-export([delete_custom_verification_email_template/1, delete_custom_verification_email_template/2]). +-export([get_custom_verification_email_template/1, get_custom_verification_email_template/2]). +-export([list_custom_verification_email_templates/0, list_custom_verification_email_templates/1]). -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). @@ -84,6 +78,11 @@ domain/0, verification_status/0]). +-type custom_template_attribute_names() :: template_name | from_email_address | template_subject | template_content | success_redirect_url | failure_redirect_url . +-type custom_template_attributes() :: [{custom_template_attribute_names(), string()}]. + +-export_type([custom_template_attribute_names/0, custom_template_attributes/0]). + %%%------------------------------------------------------------------------------ %%% Library initialization %%%------------------------------------------------------------------------------ @@ -173,55 +172,50 @@ delete_identity(Identity, Config) -> %% @end %%------------------------------------------------------------------------------ --type custom_template_attribute_names() :: template_name | from_email_address | template_subject | template_content | success_redirect_url | failure_redirect_url . --type custom_template_attributes() :: [ { custom_template_attribute_names() , string() } ]. - --type create_custom_verification_email_template_result() :: ok | { error , term() }. - --spec create_custom_verification_email_template( string() , string() ,string() ,string() ,string() ,string() ) -> create_custom_verification_email_template_result(). -create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL ) -> - create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL , default_config() ). - --spec create_custom_verification_email_template( string() , string() ,string() ,string() ,string() ,string() , aws_config() ) -> create_custom_verification_email_template_result(). -create_custom_verification_email_template( TemplateName , FromEmailAddress , TemplateSubject , TemplateContent , SuccessRedirectionURL , FailureRedirectionURL ,Config ) -> - Params = encode_params( - [ { template_name , TemplateName} , - { from_email_address , FromEmailAddress } , - { template_subject , TemplateSubject } , - { template_content , TemplateContent } , - { success_redirect_url , SuccessRedirectionURL} , - { failure_redirect_url , FailureRedirectionURL} ] ), +-type create_custom_verification_email_template_result() :: ok | { error, term() }. + +-spec create_custom_verification_email_template( string(), string(), string(), string(), string(), string() ) -> create_custom_verification_email_template_result(). +create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL) -> + create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL ,default_config()). + +-spec create_custom_verification_email_template(string(), string(), string(), string(), string(), string(), aws_config()) -> + create_custom_verification_email_template_result(). +create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL, Config) -> + Params = encode_params([{template_name, TemplateName}, + {from_email_address, FromEmailAddress}, + {template_subject, TemplateSubject}, + {template_content, TemplateContent}, + {success_redirect_url, SuccessRedirectionURL}, + {failure_redirect_url, FailureRedirectionURL}]), case ses_request(Config, "CreateCustomVerificationEmailTemplate", Params) of - {ok, _Doc} -> - ok; + {ok, _Doc} -> ok; {error, Reason} -> {error, Reason} end. --type update_custom_verification_email_template_result() :: ok | { error , term() }. +-type update_custom_verification_email_template_result() :: ok | { error, term() }. --spec update_custom_verification_email_template( string() , [ { atom() , string() }] ) -> update_custom_verification_email_template_result(). -update_custom_verification_email_template( TemplateName , Attributes ) -> - update_custom_verification_email_template( TemplateName , Attributes , default_config() ). +-spec update_custom_verification_email_template(string(), [{atom(), string()}]) -> update_custom_verification_email_template_result(). +update_custom_verification_email_template(TemplateName, Attributes) -> + update_custom_verification_email_template(TemplateName, Attributes,default_config()). --spec update_custom_verification_email_template( string() , [ { atom() , string() } ] , aws_config() ) -> update_custom_verification_email_template_result(). -update_custom_verification_email_template( TemplateName , Attributes , Config ) -> - Params = encode_params( [ { template_name , TemplateName } | Attributes ] ) , +-spec update_custom_verification_email_template(string(), [{atom(), string()}], aws_config()) -> update_custom_verification_email_template_result(). +update_custom_verification_email_template(TemplateName, Attributes, Config) -> + Params = encode_params([{template_name, TemplateName} | Attributes]), case ses_request(Config, "UpdateCustomVerificationEmailTemplate", Params) of - {ok, _Doc} -> - ok ; + {ok, _Doc} -> ok ; {error, Reason} -> {error, Reason} end. --type send_custom_verification_email_result() :: { ok , term() } | { error , term() }. +-type send_custom_verification_email_result() :: {ok, term()} | {error, term()}. --spec send_custom_verification_email( string() , string() ) -> send_custom_verification_email_result(). -send_custom_verification_email( EmailAddress , TemplateName ) -> - send_custom_verification_email( EmailAddress , TemplateName , default_config() ). +-spec send_custom_verification_email(string(), string()) -> send_custom_verification_email_result(). +send_custom_verification_email(EmailAddress, TemplateName) -> + send_custom_verification_email(EmailAddress, TemplateName, default_config()). --spec send_custom_verification_email( string() , string() , aws_config()) -> send_custom_verification_email_result(). -send_custom_verification_email( EmailAddress , TemplateName , Config ) -> - Params = encode_params( [ { email_address , EmailAddress} , - { template_name , TemplateName} ] ), +-spec send_custom_verification_email(string(), string(), aws_config()) -> send_custom_verification_email_result(). +send_custom_verification_email(EmailAddress, TemplateName, Config) -> + Params = encode_params([{email_address, EmailAddress}, + {template_name, TemplateName}]), case ses_request(Config, "SendCustomVerificationEmail", Params) of {ok, Doc} -> {ok, erlcloud_xml:decode([{message_id, "SendCustomVerificationEmailResult/MessageId", text}], Doc)}; @@ -230,61 +224,55 @@ send_custom_verification_email( EmailAddress , TemplateName , Config ) -> -type delete_custom_verification_email_template_result() :: ok | {error,term()}. --spec delete_custom_verification_email_template( string() ) -> delete_custom_verification_email_template_result(). -delete_custom_verification_email_template( TemplateName ) -> - delete_custom_verification_email_template( TemplateName , default_config() ). +-spec delete_custom_verification_email_template(string()) -> delete_custom_verification_email_template_result(). +delete_custom_verification_email_template(TemplateName) -> + delete_custom_verification_email_template(TemplateName, default_config()). --spec delete_custom_verification_email_template( string() , aws_config() ) -> delete_custom_verification_email_template_result(). -delete_custom_verification_email_template( TemplateName , Config ) -> - Params = encode_params( [ { template_name , TemplateName } ] ), +-spec delete_custom_verification_email_template(string(), aws_config()) -> delete_custom_verification_email_template_result(). +delete_custom_verification_email_template(TemplateName, Config) -> + Params = encode_params([{template_name, TemplateName }]), case ses_request(Config, "DeleteCustomVerificationEmailTemplate", Params) of - { ok , _Doc } -> - ok ; - { error , Reason } -> - { error , Reason } + {ok, _Doc} -> ok; + {error, Reason} -> {error, Reason} end. --type get_custom_verification_email_template_result() :: { ok , custom_template_attributes() } | { error , term() }. +-type get_custom_verification_email_template_result() :: {ok, custom_template_attributes()} | {error, term()}. --spec get_custom_verification_email_template( string() ) -> get_custom_verification_email_template_result(). -get_custom_verification_email_template( TemplateName ) -> - get_custom_verification_email_template( TemplateName , default_config() ). +-spec get_custom_verification_email_template(string()) -> get_custom_verification_email_template_result(). +get_custom_verification_email_template(TemplateName) -> + get_custom_verification_email_template(TemplateName, default_config()). -spec get_custom_verification_email_template( string() , aws_config() ) -> get_custom_verification_email_template_result(). -get_custom_verification_email_template( TemplateName , Config ) -> - Params = encode_params( [ { template_name , TemplateName } ] ), +get_custom_verification_email_template(TemplateName, Config) -> + Params = encode_params([{template_name, TemplateName}]), case ses_request(Config, "GetCustomVerificationEmailTemplate", Params) of { ok , Doc } -> {ok, erlcloud_xml:decode( - [ - {template_name , "GetCustomVerificationEmailTemplateResult/TemplateName", text }, - {from_email_address , "GetCustomVerificationEmailTemplateResult/FromEmailAddress", text }, - {template_subject , "GetCustomVerificationEmailTemplateResult/TemplateSubject", text }, - {template_content , "GetCustomVerificationEmailTemplateResult/TemplateContent", text }, - {success_redirect_url, "GetCustomVerificationEmailTemplateResult/SuccessRedirectionURL", text }, - {failure_redirect_url, "GetCustomVerificationEmailTemplateResult/FailureRedirectionURL", text } - ], - Doc)}; - { error , Reason } -> - {error,Reason} + [{template_name, "GetCustomVerificationEmailTemplateResult/TemplateName", text}, + {from_email_address, "GetCustomVerificationEmailTemplateResult/FromEmailAddress", text}, + {template_subject, "GetCustomVerificationEmailTemplateResult/TemplateSubject", text}, + {template_content, "GetCustomVerificationEmailTemplateResult/TemplateContent", text}, + {success_redirect_url, "GetCustomVerificationEmailTemplateResult/SuccessRedirectionURL", text}, + {failure_redirect_url, "GetCustomVerificationEmailTemplateResult/FailureRedirectionURL", text}], + Doc)}; + {error, Reason} -> {error, Reason} end. --type list_custom_verification_email_templates_result() :: { ok , [ { custom_templates , [ custom_template_attributes() ] } ] } | { error , term() }. +-type list_custom_verification_email_templates_result() :: {ok, [{custom_templates, [custom_template_attributes()]}]} | {error , term()}. -spec list_custom_verification_email_templates() -> list_custom_verification_email_templates_result(). list_custom_verification_email_templates() -> - list_custom_verification_email_templates(default_config() ). + list_custom_verification_email_templates(default_config()). --spec list_custom_verification_email_templates( aws_config() ) -> list_custom_verification_email_templates_result(). -list_custom_verification_email_templates( Config ) -> - Params = [ { "MaxResults" , 50 } ] , +-spec list_custom_verification_email_templates(aws_config()) -> list_custom_verification_email_templates_result(). +list_custom_verification_email_templates(Config) -> + Params = [{"MaxResults",50}], case ses_request(Config, "ListCustomVerificationEmailTemplates", Params) of { ok , Doc } -> {ok, erlcloud_xml:decode([{custom_templates, "ListCustomVerificationEmailTemplatesResult/CustomVerificationEmailTemplates/member", fun decode_custom_template_entry/1}, {next_token, "ListCustomVerificationEmailTemplatesResult/NextToken", optional_text}], Doc)}; - { error , Reason } -> - {error,Reason} + {error, Reason } -> {error, Reason} end. %%%------------------------------------------------------------------------------ @@ -948,23 +936,26 @@ encode_params([{source, Source} | T], Acc) when is_list(Source); is_binary(Sourc encode_params(T, [{"Source", Source} | Acc]); encode_params([{subject, Subject} | T], Acc) -> encode_params(T, encode_content("Message.Subject", Subject, Acc)); -encode_params([{template_name, TemplateName} | T], Acc) when is_list(TemplateName); is_binary(TemplateName) -> - encode_params(T, [{"TemplateName", TemplateName} | Acc]); -encode_params([{from_email_address, FromEmailAddress} | T], Acc) when is_list(FromEmailAddress); is_binary(FromEmailAddress) -> - encode_params(T, [{"FromEmailAddress", FromEmailAddress} | Acc]); -encode_params([{template_subject, TemplateSubject} | T], Acc) when is_list(TemplateSubject); is_binary(TemplateSubject) -> - encode_params(T, [{"TemplateSubject", TemplateSubject} | Acc]); -encode_params([{template_content, TemplateContent} | T], Acc) when is_list(TemplateContent); is_binary(TemplateContent) -> - encode_params(T, [{"TemplateContent", TemplateContent} | Acc]); -encode_params([{success_redirect_url, SuccessRedirectionURL} | T], Acc) when is_list(SuccessRedirectionURL); is_binary(SuccessRedirectionURL) -> - encode_params(T, [{"SuccessRedirectionURL", SuccessRedirectionURL} | Acc]); -encode_params([{failure_redirect_url, FailureRedirectionURL} | T], Acc) when is_list(FailureRedirectionURL); is_binary(FailureRedirectionURL) -> - encode_params(T, [{"FailureRedirectionURL", FailureRedirectionURL} | Acc]); +encode_params([{template_name, TemplateName} | T], Acc) + when is_list(TemplateName); is_binary(TemplateName) -> + encode_params(T, [{"TemplateName", TemplateName} | Acc]); +encode_params([{from_email_address, FromEmailAddress} | T], Acc) + when is_list(FromEmailAddress); is_binary(FromEmailAddress) -> + encode_params(T, [{"FromEmailAddress", FromEmailAddress} | Acc]); +encode_params([{template_subject, TemplateSubject} | T], Acc) + when is_list(TemplateSubject); is_binary(TemplateSubject) -> + encode_params(T, [{"TemplateSubject", TemplateSubject} | Acc]); +encode_params([{template_content, TemplateContent} | T], Acc) + when is_list(TemplateContent); is_binary(TemplateContent) -> + encode_params(T, [{"TemplateContent", TemplateContent} | Acc]); +encode_params([{success_redirect_url, SuccessRedirectionURL} | T], Acc) + when is_list(SuccessRedirectionURL); is_binary(SuccessRedirectionURL) -> + encode_params(T, [{"SuccessRedirectionURL", SuccessRedirectionURL} | Acc]); +encode_params([{failure_redirect_url, FailureRedirectionURL} | T], Acc) + when is_list(FailureRedirectionURL); is_binary(FailureRedirectionURL) -> + encode_params(T, [{"FailureRedirectionURL", FailureRedirectionURL} | Acc]); encode_params([Option | _], _Acc) -> - error({erlcloud_ses, {invalid_parameter, Option}}). - - - + error({erlcloud_ses, {invalid_parameter, Option}}). encode_list(Prefix, List, Acc) -> encode_list(Prefix, List, 1, Acc). From 1174b1af797d386abc467205303dca999c8603de Mon Sep 17 00:00:00 2001 From: Stephane Bourque Date: Thu, 1 Aug 2019 07:51:04 -0700 Subject: [PATCH 079/310] Moving functions around and adding API links --- src/erlcloud_ses.erl | 315 ++++++++++++++++++++++++++++--------------- 1 file changed, 209 insertions(+), 106 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index 67e71b6c3..77750bfd3 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -110,6 +110,80 @@ configure(AccessKeyID, SecretAccessKey, Host) -> default_config() -> erlcloud_aws:default_config(). +%%------------------------------------------------------------------------------ +%% @doc +%% SES API: +%% [https://docs.aws.amazon.com/ses/latest/APIReference/API_CreateCustomVerificationEmailTemplate.html] +%% +%% ===Example=== +%% +%% Creates a new custom verification email template. +%% +%% ` +%% ok = erlcloud_ses:create_custom_verification_email_template_result( +%% "templaneName", +%% "support@example.com", +%% "Welcome to support", +%% "limited html content", +%% "https://www.example.com/success", +%% "https://www.example.com/failure" ). +%% ' +%% +%% Please consult the following for what is and not allowed in the HTML content parameter +%% [https://docs.aws.amazon.com/ses/latest/DeveloperGuide/custom-verification-emails.html#custom-verification-emails-faq] +%% @end +%%------------------------------------------------------------------------------ + +-type create_custom_verification_email_template_result() :: ok | { error, term() }. + +-spec create_custom_verification_email_template( string(), string(), string(), string(), string(), string() ) -> create_custom_verification_email_template_result(). +create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL) -> + create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL ,default_config()). + +-spec create_custom_verification_email_template(string(), string(), string(), string(), string(), string(), aws_config()) -> + create_custom_verification_email_template_result(). +create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL, Config) -> + Params = encode_params([{template_name, TemplateName}, + {from_email_address, FromEmailAddress}, + {template_subject, TemplateSubject}, + {template_content, TemplateContent}, + {success_redirect_url, SuccessRedirectionURL}, + {failure_redirect_url, FailureRedirectionURL}]), + case ses_request(Config, "CreateCustomVerificationEmailTemplate", Params) of + {ok, _Doc} -> ok; + {error, Reason} -> {error, Reason} + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% SES API: +%% [https://docs.aws.amazon.com/ses/latest/APIReference/API_DeleteCustomVerificationEmailTemplate.html] +%% +%% ===Example=== +%% +%% Deletes an existing custom verification email template. +%% +%% ` +%% ok = erlcloud_ses:delete_custom_verification_email_template("templaneName"). +%% ' +%% +%% @end +%%------------------------------------------------------------------------------ + +-type delete_custom_verification_email_template_result() :: ok | {error,term()}. + +-spec delete_custom_verification_email_template(string()) -> delete_custom_verification_email_template_result(). +delete_custom_verification_email_template(TemplateName) -> + delete_custom_verification_email_template(TemplateName, default_config()). + +-spec delete_custom_verification_email_template(string(), aws_config()) -> delete_custom_verification_email_template_result(). +delete_custom_verification_email_template(TemplateName, Config) -> + Params = encode_params([{template_name, TemplateName }]), + case ses_request(Config, "DeleteCustomVerificationEmailTemplate", Params) of + {ok, _Doc} -> ok; + {error, Reason} -> {error, Reason} + end. + %%%------------------------------------------------------------------------------ %%% DeleteIdentity @@ -146,96 +220,29 @@ delete_identity(Identity, Config) -> %%------------------------------------------------------------------------------ %% @doc -%% Custom Verification Templates %% SES API: %% [https://docs.aws.amazon.com/ses/latest/APIReference/API_GetCustomVerificationEmailTemplate.html] %% -%% Template attributes: -%% { template_name , string() } -%% { from_email_address , string() } -%% { template_subject , string() } -%% { template_content , string() } -- please see notes in API Guide on what is allowed -%% { success_redirect_url , string() } -%% { failure_redirect_url , string() } +%% ===Example=== %% -%% On template creation, all attributes are mandatory. -%% On template updates, only include the attributes you need to modify -%% When listing templates, all attributes are included except template_content, use the get function to retrieve it +%% Returns the custom email verification template for the template name you specify. %% -%% create_custom_verification_email_template -%% update_custom_verification_email_template -%% send_custom_verification_email -%% delete_custom_verification_email_template -%% get_custom_verification_email_template -%% list_custom_verification_email_templates +%% ` +%% {ok,[{template_name,"templateName"}, +%% {from_email_address,"support@example.com"}, +%% {template_subject,"Welcome to Example.com"}, +%% {template_content,"\n\n \n

Ready to start with Example.com

\n

Example.com is very happy to +%% welcome you to the ACME system Please click\non the link below to activate +%% your account.

\n\n"}, +%% {success_redirect_url,"https://www.example.com/success"}, +%% {failure_redirect_url,"http://example.arilia.com"}]} = +%% erlcloud_ses:delete_custom_verification_email_template("templaneName"). +%% ' %% %% @end %%------------------------------------------------------------------------------ --type create_custom_verification_email_template_result() :: ok | { error, term() }. - --spec create_custom_verification_email_template( string(), string(), string(), string(), string(), string() ) -> create_custom_verification_email_template_result(). -create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL) -> - create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL ,default_config()). - --spec create_custom_verification_email_template(string(), string(), string(), string(), string(), string(), aws_config()) -> - create_custom_verification_email_template_result(). -create_custom_verification_email_template(TemplateName, FromEmailAddress, TemplateSubject, TemplateContent, SuccessRedirectionURL, FailureRedirectionURL, Config) -> - Params = encode_params([{template_name, TemplateName}, - {from_email_address, FromEmailAddress}, - {template_subject, TemplateSubject}, - {template_content, TemplateContent}, - {success_redirect_url, SuccessRedirectionURL}, - {failure_redirect_url, FailureRedirectionURL}]), - case ses_request(Config, "CreateCustomVerificationEmailTemplate", Params) of - {ok, _Doc} -> ok; - {error, Reason} -> {error, Reason} - end. - --type update_custom_verification_email_template_result() :: ok | { error, term() }. - --spec update_custom_verification_email_template(string(), [{atom(), string()}]) -> update_custom_verification_email_template_result(). -update_custom_verification_email_template(TemplateName, Attributes) -> - update_custom_verification_email_template(TemplateName, Attributes,default_config()). - --spec update_custom_verification_email_template(string(), [{atom(), string()}], aws_config()) -> update_custom_verification_email_template_result(). -update_custom_verification_email_template(TemplateName, Attributes, Config) -> - Params = encode_params([{template_name, TemplateName} | Attributes]), - case ses_request(Config, "UpdateCustomVerificationEmailTemplate", Params) of - {ok, _Doc} -> ok ; - {error, Reason} -> {error, Reason} - end. - --type send_custom_verification_email_result() :: {ok, term()} | {error, term()}. - --spec send_custom_verification_email(string(), string()) -> send_custom_verification_email_result(). -send_custom_verification_email(EmailAddress, TemplateName) -> - send_custom_verification_email(EmailAddress, TemplateName, default_config()). - --spec send_custom_verification_email(string(), string(), aws_config()) -> send_custom_verification_email_result(). -send_custom_verification_email(EmailAddress, TemplateName, Config) -> - Params = encode_params([{email_address, EmailAddress}, - {template_name, TemplateName}]), - case ses_request(Config, "SendCustomVerificationEmail", Params) of - {ok, Doc} -> - {ok, erlcloud_xml:decode([{message_id, "SendCustomVerificationEmailResult/MessageId", text}], Doc)}; - {error, Reason} -> {error, Reason} - end. - --type delete_custom_verification_email_template_result() :: ok | {error,term()}. - --spec delete_custom_verification_email_template(string()) -> delete_custom_verification_email_template_result(). -delete_custom_verification_email_template(TemplateName) -> - delete_custom_verification_email_template(TemplateName, default_config()). - --spec delete_custom_verification_email_template(string(), aws_config()) -> delete_custom_verification_email_template_result(). -delete_custom_verification_email_template(TemplateName, Config) -> - Params = encode_params([{template_name, TemplateName }]), - case ses_request(Config, "DeleteCustomVerificationEmailTemplate", Params) of - {ok, _Doc} -> ok; - {error, Reason} -> {error, Reason} - end. - -type get_custom_verification_email_template_result() :: {ok, custom_template_attributes()} | {error, term()}. -spec get_custom_verification_email_template(string()) -> get_custom_verification_email_template_result(). @@ -248,31 +255,14 @@ get_custom_verification_email_template(TemplateName, Config) -> case ses_request(Config, "GetCustomVerificationEmailTemplate", Params) of { ok , Doc } -> {ok, erlcloud_xml:decode( - [{template_name, "GetCustomVerificationEmailTemplateResult/TemplateName", text}, - {from_email_address, "GetCustomVerificationEmailTemplateResult/FromEmailAddress", text}, - {template_subject, "GetCustomVerificationEmailTemplateResult/TemplateSubject", text}, - {template_content, "GetCustomVerificationEmailTemplateResult/TemplateContent", text}, - {success_redirect_url, "GetCustomVerificationEmailTemplateResult/SuccessRedirectionURL", text}, - {failure_redirect_url, "GetCustomVerificationEmailTemplateResult/FailureRedirectionURL", text}], - Doc)}; - {error, Reason} -> {error, Reason} - end. - --type list_custom_verification_email_templates_result() :: {ok, [{custom_templates, [custom_template_attributes()]}]} | {error , term()}. - --spec list_custom_verification_email_templates() -> list_custom_verification_email_templates_result(). -list_custom_verification_email_templates() -> - list_custom_verification_email_templates(default_config()). - --spec list_custom_verification_email_templates(aws_config()) -> list_custom_verification_email_templates_result(). -list_custom_verification_email_templates(Config) -> - Params = [{"MaxResults",50}], - case ses_request(Config, "ListCustomVerificationEmailTemplates", Params) of - { ok , Doc } -> - {ok, erlcloud_xml:decode([{custom_templates, "ListCustomVerificationEmailTemplatesResult/CustomVerificationEmailTemplates/member", fun decode_custom_template_entry/1}, - {next_token, "ListCustomVerificationEmailTemplatesResult/NextToken", optional_text}], + [{template_name, "GetCustomVerificationEmailTemplateResult/TemplateName", text}, + {from_email_address, "GetCustomVerificationEmailTemplateResult/FromEmailAddress", text}, + {template_subject, "GetCustomVerificationEmailTemplateResult/TemplateSubject", text}, + {template_content, "GetCustomVerificationEmailTemplateResult/TemplateContent", text}, + {success_redirect_url, "GetCustomVerificationEmailTemplateResult/SuccessRedirectionURL", text}, + {failure_redirect_url, "GetCustomVerificationEmailTemplateResult/FailureRedirectionURL", text}], Doc)}; - {error, Reason } -> {error, Reason} + {error, Reason} -> {error, Reason} end. %%%------------------------------------------------------------------------------ @@ -510,6 +500,48 @@ get_send_statistics(Config) -> {error, Reason} -> {error, Reason} end. +%%------------------------------------------------------------------------------ +%% @doc +%% SES API: +%% [https://docs.aws.amazon.com/ses/latest/APIReference/API_ListCustomVerificationEmailTemplates.html] +%% +%% ===Example=== +%% +%% Lists the existing custom verification email templates for your account in the current AWS Region. +%% +%% ` +%% {ok,[{custom_templates,[[{template_name,"template1"}, +%% {from_email_address,"support@example.com"}, +%% {template_subject,"Welcome to support"}, +%% {success_redirect_url,"https://www.example.com/success"}, +%% {failure_redirect_url,"https://www.example.com/failure"}], +%% [{template_name,"template2"}, +%% {from_email_address,"applications@example.com"}, +%% {template_subject,"Welcome to Applications"}, +%% {success_redirect_url,"https://www.example.com/success"}, +%% {failure_redirect_url,"https://www.example.com/failure"}]]}]} = +%% erlcloud_ses:list_custom_verification_email_templates(). +%% ' +%% +%% @end +%%------------------------------------------------------------------------------ + +-type list_custom_verification_email_templates_result() :: {ok, [{custom_templates, [custom_template_attributes()]}]} | {error , term()}. + +-spec list_custom_verification_email_templates() -> list_custom_verification_email_templates_result(). +list_custom_verification_email_templates() -> + list_custom_verification_email_templates(default_config()). + +-spec list_custom_verification_email_templates(aws_config()) -> list_custom_verification_email_templates_result(). +list_custom_verification_email_templates(Config) -> + Params = [{"MaxResults",50}], + case ses_request(Config, "ListCustomVerificationEmailTemplates", Params) of + { ok , Doc } -> + {ok, erlcloud_xml:decode([{custom_templates, "ListCustomVerificationEmailTemplatesResult/CustomVerificationEmailTemplates/member", fun decode_custom_template_entry/1}, + {next_token, "ListCustomVerificationEmailTemplatesResult/NextToken", optional_text}], + Doc)}; + {error, Reason } -> {error, Reason} + end. %%%------------------------------------------------------------------------------ %%% ListIdentities @@ -573,6 +605,38 @@ list_identities(Opts, Config) -> {error, Reason} -> {error, Reason} end. +%%------------------------------------------------------------------------------ +%% @doc +%% SES API: +%% [https://docs.aws.amazon.com/ses/latest/APIReference/API_SendCustomVerificationEmail.html] +%% +%% ===Example=== +%% +%% Send a custom verification email. +%% +%% ` +%% {ok, [{message_id, "abcdefghijkllkjdlkj"}]} = +%% erlcloud_ses:send_custom_verification_email_result("newclient@newco.com", "templateName"). +%% ' +%% +%% @end +%%------------------------------------------------------------------------------ + +-type send_custom_verification_email_result() :: {ok, term()} | {error, term()}. + +-spec send_custom_verification_email(string(), string()) -> send_custom_verification_email_result(). +send_custom_verification_email(EmailAddress, TemplateName) -> + send_custom_verification_email(EmailAddress, TemplateName, default_config()). + +-spec send_custom_verification_email(string(), string(), aws_config()) -> send_custom_verification_email_result(). +send_custom_verification_email(EmailAddress, TemplateName, Config) -> + Params = encode_params([{email_address, EmailAddress}, + {template_name, TemplateName}]), + case ses_request(Config, "SendCustomVerificationEmail", Params) of + {ok, Doc} -> + {ok, erlcloud_xml:decode([{message_id, "SendCustomVerificationEmailResult/MessageId", text}], Doc)}; + {error, Reason} -> {error, Reason} + end. %%%------------------------------------------------------------------------------ %%% SendEmail @@ -784,6 +848,46 @@ set_identity_notification_topic(Identity, NotificationType, SnsTopic, Config) -> {error, Reason} -> {error, Reason} end. +%%------------------------------------------------------------------------------ +%% @doc +%% SES API: +%% [https://docs.aws.amazon.com/ses/latest/APIReference/API_UpdateCustomVerificationEmailTemplate.html] +%% +%% Template attributes +%% { template_name , string() } +%% { from_email_address , string() } +%% { template_subject , string() } +%% { template_content , string() } +%% { success_redirect_url , string() } +%% { failure_redirect_url , string() } +%% +%% ===Example=== +%% +%% Updates an existing custom verification email template. +%% +%% ` +%% ok = erlcloud_ses:update_custom_verification_email_template("templateName", +%% [{template_subject, "New subject"}, +%% {from_email_address, "support2@example.com"}]). +%% ' +%% Please consult the following for what is and not allowed in the HTML content parameter +%% [https://docs.aws.amazon.com/ses/latest/DeveloperGuide/custom-verification-emails.html#custom-verification-emails-faq] +%% @end +%%------------------------------------------------------------------------------ + +-type update_custom_verification_email_template_result() :: ok | { error, term() }. + +-spec update_custom_verification_email_template(string(), [{atom(), string()}]) -> update_custom_verification_email_template_result(). +update_custom_verification_email_template(TemplateName, Attributes) -> + update_custom_verification_email_template(TemplateName, Attributes,default_config()). + +-spec update_custom_verification_email_template(string(), [{atom(), string()}], aws_config()) -> update_custom_verification_email_template_result(). +update_custom_verification_email_template(TemplateName, Attributes, Config) -> + Params = encode_params([{template_name, TemplateName} | Attributes]), + case ses_request(Config, "UpdateCustomVerificationEmailTemplate", Params) of + {ok, _Doc} -> ok ; + {error, Reason} -> {error, Reason} + end. %%%------------------------------------------------------------------------------ %%% VerifyDomainDkim @@ -860,7 +964,6 @@ verify_domain_identity(Domain, Config) -> {error, Reason} -> {error, Reason} end. - %%%------------------------------------------------------------------------------ %%% VerifyEmailIdentity %%%------------------------------------------------------------------------------ From 400dd79b9b421a6460873d17d463d8c4f59be3af Mon Sep 17 00:00:00 2001 From: Stephane Bourque Date: Sat, 3 Aug 2019 11:04:34 -0700 Subject: [PATCH 080/310] Reordering exports, export_types, and types. --- src/erlcloud_ses.erl | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index 77750bfd3..c59de0fa7 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -21,9 +21,14 @@ -module(erlcloud_ses). -export([configure/2, configure/3, new/2, new/3]). +-export([create_custom_verification_email_template/6, create_custom_verification_email_template/7]). + +-export([delete_custom_verification_email_template/1, delete_custom_verification_email_template/2]). -export([delete_identity/1, delete_identity/2]). +-export([get_custom_verification_email_template/1, get_custom_verification_email_template/2]). + -export([get_identity_dkim_attributes/1, get_identity_dkim_attributes/2]). -export([get_identity_notification_attributes/1, get_identity_notification_attributes/2]). -export([get_identity_verification_attributes/1, get_identity_verification_attributes/2]). @@ -31,24 +36,24 @@ -export([get_send_quota/0, get_send_quota/1]). -export([get_send_statistics/0, get_send_statistics/1]). +-export([list_custom_verification_email_templates/0, list_custom_verification_email_templates/1]). + -export([list_identities/0, list_identities/1, list_identities/2]). +-export([send_custom_verification_email/2, send_custom_verification_email/3]). + -export([send_email/4, send_email/5, send_email/6]). -export([set_identity_dkim_enabled/2, set_identity_dkim_enabled/3]). -export([set_identity_feedback_forwarding_enabled/2, set_identity_feedback_forwarding_enabled/3]). -export([set_identity_notification_topic/3, set_identity_notification_topic/4]). +-export([update_custom_verification_email_template/2, update_custom_verification_email_template/3]). + -export([verify_domain_dkim/1, verify_domain_dkim/2]). -export([verify_email_identity/1, verify_email_identity/2]). -export([verify_domain_identity/1, verify_domain_identity/2]). --export([create_custom_verification_email_template/6, create_custom_verification_email_template/7]). --export([update_custom_verification_email_template/2, update_custom_verification_email_template/3]). --export([send_custom_verification_email/2, send_custom_verification_email/3]). --export([delete_custom_verification_email_template/1, delete_custom_verification_email_template/2]). --export([get_custom_verification_email_template/1, get_custom_verification_email_template/2]). --export([list_custom_verification_email_templates/0, list_custom_verification_email_templates/1]). -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). @@ -59,6 +64,9 @@ %%% Common types %%%------------------------------------------------------------------------------ +-type custom_template_attribute_names() :: template_name | from_email_address | template_subject | template_content | success_redirect_url | failure_redirect_url . +-type custom_template_attributes() :: [{custom_template_attribute_names(), string()}]. + -type identity() :: string() | binary(). %% identities (as input) can be a single identity or a list of them. @@ -73,16 +81,12 @@ -type verification_status() :: pending | success | failed | temporary_failure | not_started. --export_type([identity/0, identities/0, +-export_type([custom_template_attribute_names/0, custom_template_attributes/0, + identity/0, identities/0, email/0, emails/0, domain/0, verification_status/0]). --type custom_template_attribute_names() :: template_name | from_email_address | template_subject | template_content | success_redirect_url | failure_redirect_url . --type custom_template_attributes() :: [{custom_template_attribute_names(), string()}]. - --export_type([custom_template_attribute_names/0, custom_template_attributes/0]). - %%%------------------------------------------------------------------------------ %%% Library initialization %%%------------------------------------------------------------------------------ From 77471f1a6c5e9b9679a2a899f3df4c08aedf691f Mon Sep 17 00:00:00 2001 From: Sonic Gao Date: Tue, 6 Aug 2019 23:35:22 +0800 Subject: [PATCH 081/310] fix test case --- test/erlcloud_aws_tests.erl | 123 ++++++++++++++++++++++++------------ 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/test/erlcloud_aws_tests.erl b/test/erlcloud_aws_tests.erl index 59668263c..ba2058fba 100644 --- a/test/erlcloud_aws_tests.erl +++ b/test/erlcloud_aws_tests.erl @@ -147,7 +147,7 @@ get_service_status_test(_) -> OKStatusEmpty = erlcloud_aws:get_service_status(["sqs", "sns"]), meck:expect(erlcloud_httpc, request, fun(_,_,_,_,_,_) -> {ok, {{200, "OK"}, [], StatusJsonS3}} end), OKStatus = erlcloud_aws:get_service_status(["cloudformation", "sns", "vpc"]), - + [?_assertEqual(proplists:get_value(<<"status">>, S3Status), 0), ?_assertEqual(proplists:get_value(<<"service">>, S3Status), <<"s3-eu-central-1">>), ?_assertEqual(proplists:get_value(<<"status">>, EC2Status), 2), @@ -237,7 +237,7 @@ profile_indirect_test_() -> erlcloud_aws:profile( blah ) ) ) }. - + profile_indirect_role_test_() -> {setup, fun profiles_assume_setup/0, fun profiles_assume_cleanup/1, ?_test( @@ -309,14 +309,14 @@ profile_undefined_profile_test_() -> ?assertMatch( {error, _}, erlcloud_aws:profile( what ) ) ) }. - + profile_undefined_indirect_profile_test_() -> {setup, fun profiles_test_setup/0, fun profiles_test_cleanup/1, ?_test( ?assertMatch( {error, _}, erlcloud_aws:profile( whoa ) ) ) }. - + profiles_test_setup() -> Profile = <<" @@ -416,7 +416,8 @@ service_config_autoscaling_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["autoscaling.us-east-1.amazonaws.com", "autoscaling.us-west-1.amazonaws.com", "autoscaling.us-west-2.amazonaws.com", @@ -426,7 +427,9 @@ service_config_autoscaling_test() -> "autoscaling.ap-northeast-2.amazonaws.com", "autoscaling.ap-southeast-1.amazonaws.com", "autoscaling.ap-southeast-2.amazonaws.com", - "autoscaling.sa-east-1.amazonaws.com"], + "autoscaling.sa-east-1.amazonaws.com", + "autoscaling.cn-north-1.amazonaws.com.cn", + "autoscaling.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ as_host = H } <- [erlcloud_aws:service_config( @@ -440,7 +443,8 @@ service_config_cloudformation_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["cloudformation.us-east-1.amazonaws.com", "cloudformation.us-west-1.amazonaws.com", "cloudformation.us-west-2.amazonaws.com", @@ -450,7 +454,9 @@ service_config_cloudformation_test() -> "cloudformation.ap-northeast-2.amazonaws.com", "cloudformation.ap-southeast-1.amazonaws.com", "cloudformation.ap-southeast-2.amazonaws.com", - "cloudformation.sa-east-1.amazonaws.com"], + "cloudformation.sa-east-1.amazonaws.com", + "cloudformation.cn-north-1.amazonaws.com.cn", + "cloudformation.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ cloudformation_host = H } <- [erlcloud_aws:service_config( @@ -468,7 +474,8 @@ service_config_cloudtrail_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["cloudtrail.us-east-1.amazonaws.com", "cloudtrail.us-west-1.amazonaws.com", "cloudtrail.us-west-2.amazonaws.com", @@ -478,7 +485,9 @@ service_config_cloudtrail_test() -> "cloudtrail.ap-northeast-2.amazonaws.com", "cloudtrail.ap-southeast-1.amazonaws.com", "cloudtrail.ap-southeast-2.amazonaws.com", - "cloudtrail.sa-east-1.amazonaws.com"], + "cloudtrail.sa-east-1.amazonaws.com", + "cloudtrail.cn-north-1.amazonaws.com.cn", + "cloudtrail.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ cloudtrail_host = H } <- [erlcloud_aws:service_config( @@ -492,7 +501,8 @@ service_config_dynamodb_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["dynamodb.us-east-1.amazonaws.com", "dynamodb.us-west-1.amazonaws.com", "dynamodb.us-west-2.amazonaws.com", @@ -502,7 +512,9 @@ service_config_dynamodb_test() -> "dynamodb.ap-northeast-2.amazonaws.com", "dynamodb.ap-southeast-1.amazonaws.com", "dynamodb.ap-southeast-2.amazonaws.com", - "dynamodb.sa-east-1.amazonaws.com"], + "dynamodb.sa-east-1.amazonaws.com", + "dynamodb.cn-north-1.amazonaws.com.cn", + "dynamodb.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ ddb_host = H } <- [erlcloud_aws:service_config( @@ -520,7 +532,8 @@ service_config_dynamodb_streams_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["streams.dynamodb.us-east-1.amazonaws.com", "streams.dynamodb.us-west-1.amazonaws.com", "streams.dynamodb.us-west-2.amazonaws.com", @@ -530,7 +543,9 @@ service_config_dynamodb_streams_test() -> "streams.dynamodb.ap-northeast-2.amazonaws.com", "streams.dynamodb.ap-southeast-1.amazonaws.com", "streams.dynamodb.ap-southeast-2.amazonaws.com", - "streams.dynamodb.sa-east-1.amazonaws.com"], + "streams.dynamodb.sa-east-1.amazonaws.com", + "streams.dynamodb.cn-north-1.amazonaws.com.cn", + "streams.dynamodb.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ ddb_streams_host = H } <- [erlcloud_aws:service_config( @@ -543,7 +558,8 @@ service_config_ec2_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["ec2.us-east-1.amazonaws.com", "ec2.us-west-1.amazonaws.com", "ec2.us-west-2.amazonaws.com", @@ -553,7 +569,9 @@ service_config_ec2_test() -> "ec2.ap-northeast-2.amazonaws.com", "ec2.ap-southeast-1.amazonaws.com", "ec2.ap-southeast-2.amazonaws.com", - "ec2.sa-east-1.amazonaws.com"], + "ec2.sa-east-1.amazonaws.com", + "ec2.cn-north-1.amazonaws.com.cn", + "ec2.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ ec2_host = H } <- [erlcloud_aws:service_config( @@ -567,7 +585,8 @@ service_config_elasticloadbalancing_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["elasticloadbalancing.us-east-1.amazonaws.com", "elasticloadbalancing.us-west-1.amazonaws.com", "elasticloadbalancing.us-west-2.amazonaws.com", @@ -577,7 +596,9 @@ service_config_elasticloadbalancing_test() -> "elasticloadbalancing.ap-northeast-2.amazonaws.com", "elasticloadbalancing.ap-southeast-1.amazonaws.com", "elasticloadbalancing.ap-southeast-2.amazonaws.com", - "elasticloadbalancing.sa-east-1.amazonaws.com"], + "elasticloadbalancing.sa-east-1.amazonaws.com", + "elasticloadbalancing.cn-north-1.amazonaws.com.cn", + "elasticloadbalancing.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ elb_host = H } <- [erlcloud_aws:service_config( @@ -598,7 +619,8 @@ service_config_elasticmapreduce_test() -> <<"ca-central-1">>, <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["elasticmapreduce.us-east-1.amazonaws.com", "elasticmapreduce.us-east-2.amazonaws.com", "elasticmapreduce.us-west-1.amazonaws.com", @@ -610,7 +632,9 @@ service_config_elasticmapreduce_test() -> "elasticmapreduce.ap-northeast-2.amazonaws.com", "elasticmapreduce.ap-southeast-1.amazonaws.com", "elasticmapreduce.ap-southeast-2.amazonaws.com", - "elasticmapreduce.sa-east-1.amazonaws.com"], + "elasticmapreduce.sa-east-1.amazonaws.com", + "elasticmapreduce.cn-north-1.amazonaws.com.cn", + "elasticmapreduce.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ emr_host = H } <- [erlcloud_aws:service_config( @@ -637,13 +661,24 @@ service_config_iam_test() -> Service, Region, #aws_config{} ) || Region <- Regions]] ). +service_config_china_iam_test() -> + Service = <<"iam">>, + Regions = [<<"cn-north-1">>, <<"cn-northwest-1">>], + Expected = lists:duplicate( length(Regions), "iam.amazonaws.com.cn" ), + ?assertEqual( Expected, + [H || #aws_config{ iam_host = H } <- + [erlcloud_aws:service_config( + Service, Region, #aws_config{} ) + || Region <- Regions]] ). + service_config_kinesis_test() -> Service = <<"kinesis">>, Regions = [<<"us-east-1">>, <<"us-west-1">>, <<"us-west-2">>, <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["kinesis.us-east-1.amazonaws.com", "kinesis.us-west-1.amazonaws.com", "kinesis.us-west-2.amazonaws.com", @@ -653,7 +688,9 @@ service_config_kinesis_test() -> "kinesis.ap-northeast-2.amazonaws.com", "kinesis.ap-southeast-1.amazonaws.com", "kinesis.ap-southeast-2.amazonaws.com", - "kinesis.sa-east-1.amazonaws.com"], + "kinesis.sa-east-1.amazonaws.com", + "kinesis.cn-north-1.amazonaws.com.cn", + "kinesis.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ kinesis_host = H } <- [erlcloud_aws:service_config( @@ -681,7 +718,8 @@ service_config_rds_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["rds.us-east-1.amazonaws.com", "rds.us-west-1.amazonaws.com", "rds.us-west-2.amazonaws.com", @@ -691,7 +729,9 @@ service_config_rds_test() -> "rds.ap-northeast-2.amazonaws.com", "rds.ap-southeast-1.amazonaws.com", "rds.ap-southeast-2.amazonaws.com", - "rds.sa-east-1.amazonaws.com"], + "rds.sa-east-1.amazonaws.com", + "rds.cn-north-1.amazonaws.com.cn", + "rds.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ rds_host = H } <- [erlcloud_aws:service_config( @@ -704,8 +744,8 @@ service_config_s3_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"sa-east-1">>, - <<"us-gov-west-1">>, <<"cn-north-1">>], + <<"sa-east-1">>, <<"us-gov-west-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["s3-external-1.amazonaws.com", "s3-us-west-1.amazonaws.com", "s3-us-west-2.amazonaws.com", @@ -717,7 +757,8 @@ service_config_s3_test() -> "s3-ap-southeast-2.amazonaws.com", "s3-sa-east-1.amazonaws.com", "s3-fips-us-gov-west-1.amazonaws.com", - "s3.cn-north-1.amazonaws.com.cn"], + "s3.cn-north-1.amazonaws.com.cn", + "s3.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ s3_host = H } <- [erlcloud_aws:service_config( @@ -766,7 +807,8 @@ service_config_sns_test() -> <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, <<"sa-east-1">>, - <<"us-gov-west-1">>, <<"cn-north-1">>], + <<"us-gov-west-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["sns.us-east-1.amazonaws.com", "sns.us-west-1.amazonaws.com", "sns.us-west-2.amazonaws.com", @@ -778,7 +820,8 @@ service_config_sns_test() -> "sns.ap-southeast-2.amazonaws.com", "sns.sa-east-1.amazonaws.com", "sns.us-gov-west-1.amazonaws.com", - "sns.cn-north-1.amazonaws.com"], + "sns.cn-north-1.amazonaws.com.cn", + "sns.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ sns_host = H } <- [erlcloud_aws:service_config( @@ -792,8 +835,8 @@ service_config_sqs_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"cn-north-1">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["sqs.us-east-1.amazonaws.com", "sqs.us-west-1.amazonaws.com", "sqs.us-west-2.amazonaws.com", @@ -804,8 +847,9 @@ service_config_sqs_test() -> "sqs.ap-northeast-2.amazonaws.com", "sqs.ap-southeast-1.amazonaws.com", "sqs.ap-southeast-2.amazonaws.com", - "sqs.cn-north-1.amazonaws.com", - "sqs.sa-east-1.amazonaws.com"], + "sqs.sa-east-1.amazonaws.com", + "sqs.cn-north-1.amazonaws.com.cn", + "sqs.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ sqs_host = H } <- [erlcloud_aws:service_config( @@ -821,15 +865,15 @@ service_config_sts_test() -> <<"eu-west-1">>, <<"eu-central-1">>, <<"ap-northeast-1">>, <<"ap-northeast-2">>, <<"ap-southeast-1">>, <<"ap-southeast-2">>, - <<"cn-north-1">>, - <<"sa-east-1">>], + <<"sa-east-1">>, + <<"cn-north-1">>, <<"cn-northwest-1">>], RegionsAlt = ["us-east-1", "us-west-1", "us-west-2", "us-gov-west-1", "eu-west-1", "eu-central-1", "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", - "cn-north-1", - "sa-east-1"], + "sa-east-1", + "cn-north-1", "cn-northwest-1"], Expected = ["sts.us-east-1.amazonaws.com", "sts.us-west-1.amazonaws.com", "sts.us-west-2.amazonaws.com", @@ -840,8 +884,9 @@ service_config_sts_test() -> "sts.ap-northeast-2.amazonaws.com", "sts.ap-southeast-1.amazonaws.com", "sts.ap-southeast-2.amazonaws.com", - "sts.cn-north-1.amazonaws.com", - "sts.sa-east-1.amazonaws.com"], + "sts.sa-east-1.amazonaws.com", + "sts.cn-north-1.amazonaws.com.cn", + "sts.cn-northwest-1.amazonaws.com.cn"], ?assertEqual( Expected, [H || #aws_config{ sts_host = H } <- [erlcloud_aws:service_config( From 5143c0ecd46daa6ca882db9323768f88624bd32e Mon Sep 17 00:00:00 2001 From: MikeBenza Date: Wed, 7 Aug 2019 15:23:17 -0500 Subject: [PATCH 082/310] Remove execute bit from erlcloud_s3.erl --- src/erlcloud_s3.erl | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/erlcloud_s3.erl diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl old mode 100755 new mode 100644 From 63feb9797ad2dfd289794bfb866522138fd62e7e Mon Sep 17 00:00:00 2001 From: saherkar Date: Thu, 8 Aug 2019 23:50:18 +0530 Subject: [PATCH 083/310] Support for bucket tagging and object tagging APIs. --- src/erlcloud_s3.erl | 85 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 5fe5b50d9..eafe34521 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -44,7 +44,13 @@ delete_bucket_inventory/2, delete_bucket_inventory/3, put_bucket_encryption/2, put_bucket_encryption/3, put_bucket_encryption/4, get_bucket_encryption/1, get_bucket_encryption/2, - delete_bucket_encryption/1, delete_bucket_encryption/2 + delete_bucket_encryption/1, delete_bucket_encryption/2, + put_object_tagging/3, put_object_tagging/4, + delete_object_tagging/2, delete_object_tagging/3, + get_object_tagging/2, get_object_tagging/3, + put_bucket_tagging/2, put_bucket_tagging/3, + delete_bucket_tagging/1, delete_bucket_tagging/2, + get_bucket_tagging/1, get_bucket_tagging/2 ]). -ifdef(TEST). @@ -1278,6 +1284,83 @@ set_bucket_attribute(BucketName, AttributeName, Value, Config) Headers = [{"content-type", "application/xml"}], s3_simple_request(Config, put, BucketName, "/", Subresource, [], POSTData, Headers). +-spec put_object_tagging(string(), string(), list({string(), string()})) -> ok. +put_object_tagging(BucketName, Key, TagList) when is_list(BucketName) -> + put_object_tagging(BucketName, Key, TagList, default_config()). + +-spec put_object_tagging(string(), string(), list({string(), string()}), aws_config()) -> ok. +put_object_tagging(BucketName, Key, TagList, #aws_config{} = Config) when is_list(BucketName) -> + TaggingXML = {'Tagging', + [{'TagSet', encode_tags(TagList)}]}, + POSTData = unicode:characters_to_binary(xmerl:export_simple([TaggingXML], xmerl_xml)), + Md5 = base64:encode(crypto:hash(md5, POSTData)), + Headers = [{"content-md5", Md5}, {"content-type", "application/xml"}], + s3_simple_request(Config, put, BucketName, [$/|Key], "tagging", [], POSTData, Headers). + +-spec delete_object_tagging(string(), string()) -> ok. +delete_object_tagging(BucketName, Key) when is_list(BucketName) -> + delete_object_tagging(BucketName, Key, default_config()). + +-spec delete_object_tagging(string(), string(), aws_config()) -> ok. +delete_object_tagging(BucketName, Key, #aws_config{} = Config) when is_list(BucketName) -> + s3_simple_request(Config, delete, BucketName, [$/|Key], "tagging", [], <<>>, []). + +-spec get_object_tagging(string(), string()) -> {ok, list({string(), string()})}. +get_object_tagging(BucketName, Key) when is_list(BucketName) -> + get_object_tagging(BucketName, Key, default_config()). + +-spec get_object_tagging(string(), string(), aws_config()) -> {ok, list({string(), string()})}. +get_object_tagging(BucketName, Key, #aws_config{} = Config) when is_list(BucketName) -> + Doc = s3_xml_request(Config, get, BucketName, [$/|Key], "tagging", [], <<>>, []), + {ok, + [extract_tag(Node) || Node <- xmerl_xpath:string("/Tagging/TagSet/Tag", Doc)]}. + +-spec put_bucket_tagging(string(), list({string(), string()})) -> ok. +put_bucket_tagging(BucketName, TagList) when is_list(BucketName) -> + put_bucket_tagging(BucketName, TagList, default_config()). + +-spec put_bucket_tagging(string(), list({string(), string()}), aws_config()) -> ok. +put_bucket_tagging(BucketName, TagList, #aws_config{} = Config) when is_list(BucketName) -> + TaggingXML = {'Tagging', + [{'TagSet', encode_tags(TagList)}]}, + POSTData = list_to_binary(xmerl:export_simple([TaggingXML], xmerl_xml)), + Md5 = base64:encode(crypto:hash(md5, POSTData)), + Headers = [{"content-md5", Md5}, {"content-type", "application/xml"}], + s3_simple_request(Config, put, BucketName, "/", "tagging", [], POSTData, Headers). + +-spec delete_bucket_tagging(string()) -> ok. +delete_bucket_tagging(BucketName) when is_list(BucketName) -> + delete_bucket_tagging(BucketName, default_config()). + +-spec delete_bucket_tagging(string(), aws_config()) -> ok. +delete_bucket_tagging(BucketName, #aws_config{} = Config) when is_list(BucketName) -> + s3_simple_request(Config, delete, BucketName, "/", "tagging", [], <<>>, []). + +-spec get_bucket_tagging(string()) -> {ok, list({string(), string()})}. +get_bucket_tagging(BucketName) when is_list(BucketName) -> + get_bucket_tagging(BucketName, default_config()). + +-spec get_bucket_tagging(string(), aws_config()) -> {ok, list({string(), string()})}. +get_bucket_tagging(BucketName, #aws_config{} = Config) when is_list(BucketName) -> + Doc = s3_xml_request(Config, get, BucketName, "/", "tagging", [], <<>>, []), + {ok, + [extract_tag(Node) || Node <- xmerl_xpath:string("/Tagging/TagSet/Tag", Doc)]}. + +extract_tag(Node) -> + List = erlcloud_xml:decode([{key, "Key", text}, {value, "Value", text}], Node), + {proplists:get_value(key, List), proplists:get_value(value, List)}. + +encode_tags(Taglist) -> + lists:map(fun encode_one_tag/1, Taglist). + +encode_one_tag({Key, Value}) -> + {'Tag', + [ + {'Key', [Key]}, + {'Value', [Value]} + ] + }. + -spec list_bucket_inventory(string()) -> {ok, Result:: list(term())} | {error, Reason::term()}. list_bucket_inventory(BucketName) when is_list(BucketName) -> From 5b6046de7d75aa1da6f18fd607061a0edff9c7a5 Mon Sep 17 00:00:00 2001 From: saherkar Date: Thu, 8 Aug 2019 23:58:18 +0530 Subject: [PATCH 084/310] Support for HTTP/HTTPS S3 proxy and corresponding http client options tuple to store configuration. --- include/erlcloud_aws.hrl | 15 ++++++++++++++- src/erlcloud_httpc.erl | 22 ++++++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 097e0df73..5cd718533 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -10,6 +10,14 @@ -type(aws_assume_role() :: #aws_assume_role{}). +-record(http_client_options, { + insecure = true :: boolean() | undefined, + proxy = undefined :: binary() | {binary(), non_neg_integer()} | {socks5, binary(), binary()} | {connect, binary(), binary()} | undefined, + proxy_auth = undefined :: {binary(), binary()} | undefined +}). + +-type(http_client_options() :: #http_client_options{}). + -record(aws_config, { as_host="autoscaling.amazonaws.com"::string(), ec2_host="ec2.amazonaws.com"::string(), @@ -154,7 +162,12 @@ assume_role = #aws_assume_role{} :: aws_assume_role(), %% If a role to be assumed is given %% then we will try to assume the role during the update_config %% region override for API gateway type requests - aws_region=undefined::string()|undefined + aws_region=undefined::string()|undefined, + %% http proxy support + http_proxy=undefined::string()|undefined, + http_client_options = #http_client_options{} :: http_client_options() %% The http client options + %% are used to specify the proxy, proxy_auth and insecure which is + %% used to support proxy based requests to s3. }). -type(aws_config() :: #aws_config{}). diff --git a/src/erlcloud_httpc.erl b/src/erlcloud_httpc.erl index 73dd16d93..df2a38e71 100644 --- a/src/erlcloud_httpc.erl +++ b/src/erlcloud_httpc.erl @@ -43,10 +43,16 @@ request(URL, Method, Hdrs, Body, Timeout, when is_function(F, 6) -> F(URL, Method, Hdrs, Body, Timeout, Config). -request_lhttpc(URL, Method, Hdrs, Body, Timeout, #aws_config{lhttpc_pool = undefined}) -> +request_lhttpc(URL, Method, Hdrs, Body, Timeout, #aws_config{lhttpc_pool = undefined, http_proxy = undefined}) -> lhttpc:request(URL, Method, Hdrs, Body, Timeout, []); -request_lhttpc(URL, Method, Hdrs, Body, Timeout, #aws_config{lhttpc_pool = Pool}) -> +request_lhttpc(URL, Method, Hdrs, Body, Timeout, #aws_config{http_proxy = HttpProxy, lhttpc_pool = undefined}) -> + LHttpcOpts = [{proxy, HttpProxy}], + lhttpc:request(URL, Method, Hdrs, Body, Timeout, LHttpcOpts); +request_lhttpc(URL, Method, Hdrs, Body, Timeout, #aws_config{lhttpc_pool = Pool, http_proxy = undefined}) -> LHttpcOpts = [{pool, Pool}, {pool_ensure, true}], + lhttpc:request(URL, Method, Hdrs, Body, Timeout, LHttpcOpts); +request_lhttpc(URL, Method, Hdrs, Body, Timeout, #aws_config{lhttpc_pool = Pool, http_proxy = HttpProxy}) -> + LHttpcOpts = [{pool, Pool}, {pool_ensure, true}, {proxy, HttpProxy}], lhttpc:request(URL, Method, Hdrs, Body, Timeout, LHttpcOpts). %% Guard clause protects against empty bodied requests from being @@ -70,7 +76,13 @@ request_httpc(URL, Method, Hdrs, Body, Timeout, _Config) -> [{timeout, Timeout}], [{body_format, binary}])). -request_hackney(URL, Method, Hdrs, Body, Timeout, #aws_config{hackney_pool = Pool}) -> +request_hackney(URL, Method, Hdrs, Body, Timeout, + #aws_config{hackney_pool = Pool, + http_client_options = #http_client_options{ + insecure = Insecure, + proxy = Proxy, + proxy_auth = ProxyAuth}} + ) -> BinURL = to_binary(URL), BinHdrs = [{to_binary(K), to_binary(V)} || {K, V} <- Hdrs], PoolOpt = if Pool =:= undefined -> @@ -78,10 +90,12 @@ request_hackney(URL, Method, Hdrs, Body, Timeout, #aws_config{hackney_pool = Poo true -> [{pool, Pool}] end, + HttpProxyOpt = [{proxy, Proxy}, {proxy_auth, ProxyAuth}], response_hackney(hackney:request(Method, BinURL, BinHdrs, Body, - [{recv_timeout, Timeout}] ++ PoolOpt)). + [{recv_timeout, Timeout}, {insecure, Insecure}] ++ + PoolOpt ++ HttpProxyOpt)). response_httpc({ok, {{_HTTPVer, Status, StatusLine}, Headers, Body}}) -> {ok, {{Status, StatusLine}, Headers, Body}}; From 2ecee7b2ddf7a4dc702144d59a140bb0a594f2fd Mon Sep 17 00:00:00 2001 From: saherkar Date: Fri, 9 Aug 2019 00:01:49 +0530 Subject: [PATCH 085/310] Support for encoding type option added in list_objects. --- src/erlcloud_s3.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index eafe34521..15943b30c 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -517,6 +517,7 @@ list_objects(BucketName, Options, Config) Params = [{"delimiter", proplists:get_value(delimiter, Options)}, {"marker", proplists:get_value(marker, Options)}, {"max-keys", proplists:get_value(max_keys, Options)}, + {"encoding-type", proplists:get_value(encoding_type, Options)}, {"prefix", proplists:get_value(prefix, Options)}], Doc = s3_xml_request(Config, get, BucketName, "/", "", Params, <<>>, []), Attributes = [{name, "Name", text}, From 675c43cfa05be430b256b8bd80f597880db5b943 Mon Sep 17 00:00:00 2001 From: saherkar Date: Fri, 9 Aug 2019 00:03:20 +0530 Subject: [PATCH 086/310] Support for tagging in PUT, GET requests as per AWS S3 documentation. --- src/erlcloud_s3.erl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 15943b30c..d139b5a92 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -825,6 +825,7 @@ get_or_head(Method, BucketName, Key, Options, Config) -> {content_encoding, proplists:get_value("content-encoding", Headers)}, {delete_marker, list_to_existing_atom(proplists:get_value("x-amz-delete-marker", Headers, "false"))}, {version_id, proplists:get_value("x-amz-version-id", Headers, "null")}, + {tag_count, proplists:get_value("x-amz-tagging-count", Headers, "null")}, {content, Body}| extract_metadata(Headers)]. @@ -1008,7 +1009,9 @@ put_object(BucketName, Key, Value, Options, HTTPHeaders, Config) is_list(Options) -> RequestHeaders = [{"x-amz-acl", encode_acl(proplists:get_value(acl, Options))}|HTTPHeaders] ++ [{"x-amz-meta-" ++ string:to_lower(MKey), MValue} || - {MKey, MValue} <- proplists:get_value(meta, Options, [])], + {MKey, MValue} <- proplists:get_value(meta, Options, [])] + ++ [{"x-amz-tagging-" ++ MKey, MValue} || + {MKey, MValue} <- proplists:get_value(tags, Options, [])], POSTData = iolist_to_binary(Value), {Headers, _Body} = s3_request(Config, put, BucketName, [$/|Key], "", [], POSTData, RequestHeaders), @@ -1109,7 +1112,9 @@ start_multipart(BucketName, Key, Options, HTTPHeaders, Config) RequestHeaders = [{"x-amz-acl", encode_acl(proplists:get_value(acl, Options))}|HTTPHeaders] ++ [{"x-amz-meta-" ++ string:to_lower(MKey), MValue} || - {MKey, MValue} <- proplists:get_value(meta, Options, [])], + {MKey, MValue} <- proplists:get_value(meta, Options, [])] + ++ [{"x-amz-tagging-" ++ MKey, MValue} || + {MKey, MValue} <- proplists:get_value(tags, Options, [])], POSTData = <<>>, case s3_xml_request2(Config, post, BucketName, [$/|Key], "uploads", [], POSTData, RequestHeaders) of From af89ae7099b0d4e7c30bb5e90a34a18c76138047 Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Mon, 19 Aug 2019 11:24:20 +0200 Subject: [PATCH 087/310] handle future non-integer msg attr values --- src/erlcloud_sqs.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index d4ce66337..7ee02e4ae 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -393,7 +393,12 @@ decode_msg_attribute_name(Name) when is_list(Name) -> Name. decode_msg_attribute_value("SenderId", Value) -> Value; decode_msg_attribute_value("MessageGroupId", Value) -> Value; decode_msg_attribute_value("MessageDeduplicationId", Value) -> Value; -decode_msg_attribute_value(_Name, Value) -> list_to_integer(Value). +decode_msg_attribute_value(_Name, Value) -> + try list_to_integer(Value) + catch + _:_:_ -> + Value + end. decode_messages(Messages) -> [decode_message(Message) || Message <- Messages]. From 097718d48696fa2e94f62599bd4caaedc134fd83 Mon Sep 17 00:00:00 2001 From: Carl-Johan Kjellander Date: Mon, 19 Aug 2019 15:24:00 +0200 Subject: [PATCH 088/310] use old catch syntax --- src/erlcloud_sqs.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index 7ee02e4ae..5928934d9 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -396,7 +396,7 @@ decode_msg_attribute_value("MessageDeduplicationId", Value) -> Value; decode_msg_attribute_value(_Name, Value) -> try list_to_integer(Value) catch - _:_:_ -> + _:_ -> Value end. From 5e61bbf72ebdd29c82767496b03c72beb40a9c27 Mon Sep 17 00:00:00 2001 From: bruno vicente Date: Mon, 19 Aug 2019 15:13:46 -0300 Subject: [PATCH 089/310] upgrade meck version --- rebar.config | 2 +- rebar.config.script | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 1e3d0ef8e..19535e886 100644 --- a/rebar.config +++ b/rebar.config @@ -32,6 +32,6 @@ {profiles, [ - {test, [{deps, [{meck, "0.8.12"}]}]} + {test, [{deps, [{meck, "0.8.13"}]}]} ,{warnings, [{erl_opts, [warnings_as_errors]}]} ]}. diff --git a/rebar.config.script b/rebar.config.script index 19a5167d0..8a278af28 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -5,7 +5,7 @@ case erlang:function_exported(rebar3, main, 1) of false -> % rebar 2.x or older %% Use git-based deps %% profiles - [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.12"}}}, + [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.13"}}}, {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "2.9.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.6"}}}, From 96dbdf90a2d0a230c89bc257f9eba4a6a6d35633 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 4 Sep 2019 16:44:16 +0100 Subject: [PATCH 090/310] Add missing type to export section It's being referenced elsewhere --- src/erlcloud_ddb2.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 2b0af2d95..5f5dfab7a 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -220,6 +220,7 @@ sse_description/0, sse_specification/0, stream_specification/0, + stream_view_type/0, select/0, table_name/0, tag_key/0, From a9c271183dc59f6af2d6d8a14e4c5ec8459ad871 Mon Sep 17 00:00:00 2001 From: saherkar Date: Thu, 19 Sep 2019 00:07:46 +0530 Subject: [PATCH 091/310] Proxy record validation test added. --- include/erlcloud_aws.hrl | 6 +++--- src/erlcloud_httpc.erl | 2 +- test/erlcloud_s3_tests.erl | 39 ++++++++++++++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 5cd718533..050b12988 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -10,13 +10,13 @@ -type(aws_assume_role() :: #aws_assume_role{}). --record(http_client_options, { +-record(hackney_client_options, { insecure = true :: boolean() | undefined, proxy = undefined :: binary() | {binary(), non_neg_integer()} | {socks5, binary(), binary()} | {connect, binary(), binary()} | undefined, proxy_auth = undefined :: {binary(), binary()} | undefined }). --type(http_client_options() :: #http_client_options{}). +-type(hackney_client_options() :: #hackney_client_options{}). -record(aws_config, { as_host="autoscaling.amazonaws.com"::string(), @@ -165,7 +165,7 @@ aws_region=undefined::string()|undefined, %% http proxy support http_proxy=undefined::string()|undefined, - http_client_options = #http_client_options{} :: http_client_options() %% The http client options + hackney_client_options = #hackney_client_options{} :: hackney_client_options() %% The hackney client options %% are used to specify the proxy, proxy_auth and insecure which is %% used to support proxy based requests to s3. }). diff --git a/src/erlcloud_httpc.erl b/src/erlcloud_httpc.erl index df2a38e71..0132d3f87 100644 --- a/src/erlcloud_httpc.erl +++ b/src/erlcloud_httpc.erl @@ -78,7 +78,7 @@ request_httpc(URL, Method, Hdrs, Body, Timeout, _Config) -> request_hackney(URL, Method, Hdrs, Body, Timeout, #aws_config{hackney_pool = Pool, - http_client_options = #http_client_options{ + hackney_client_options = #hackney_client_options{ insecure = Insecure, proxy = Proxy, proxy_auth = ProxyAuth}} diff --git a/test/erlcloud_s3_tests.erl b/test/erlcloud_s3_tests.erl index df865666e..2d28982b3 100755 --- a/test/erlcloud_s3_tests.erl +++ b/test/erlcloud_s3_tests.erl @@ -40,7 +40,8 @@ operation_test_() -> fun put_bucket_encryption_test/1, fun get_bucket_encryption_test/1, fun get_bucket_encryption_not_found_test/1, - fun delete_bucket_encryption_test/1 + fun delete_bucket_encryption_test/1, + fun hackney_proxy_put_validation_test/1 ]}. start() -> @@ -62,9 +63,26 @@ httpc_expect(Response) -> httpc_expect(get, Response). httpc_expect(Method, Response) -> - fun(_Url, Method2, _Headers, _Body, _Timeout, _Config) -> - Method = Method2, - Response + fun(_Url, Method2, _Headers, _Body, _Timeout, _Config = #aws_config{hackney_client_options = #hackney_client_options{insecure = Insecure, + proxy = Proxy, + proxy_auth = Proxy_auth}, + http_client = Http_client}) -> + + case Http_client of + hackney -> + Method = Method2, + Insecure = false, + Proxy = "10.10.10.10", + Proxy_auth = {"AAAA", "BBBB"}; + + _else -> + Method = Method2, + Insecure = true, + Proxy = undefined, + Proxy_auth = undefined + end, + + Response end. get_bucket_lifecycle_tests(_) -> @@ -792,3 +810,16 @@ delete_bucket_encryption_test(_) -> meck:expect(erlcloud_httpc, request, httpc_expect(delete, Response)), Result = erlcloud_s3:delete_bucket_encryption("bucket", config()), ?_assertEqual(ok, Result). + +hackney_proxy_put_validation_test(_) -> + Response = {ok, {{200, "OK"}, [{"x-amz-version-id", "version_id"}], <<>>}}, + Config2 = #aws_config{hackney_client_options = #hackney_client_options{insecure = false, + proxy = "10.10.10.10", + proxy_auth = {"AAAA", "BBBB"}}, + http_client = hackney}, + meck:expect(erlcloud_httpc, request, httpc_expect(put, Response)), + Result = erlcloud_s3:put_object("BucketName", "Key", "Data", config(Config2)), + ?_assertEqual([{version_id, "version_id"} + ,{"x-amz-version-id", "version_id"} + ], Result). + From 0da1199cdc901d3938f0f0c5edec611d041cd94c Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Wed, 18 Sep 2019 14:54:29 +0100 Subject: [PATCH 092/310] Fixe external id maximum length. --- src/erlcloud_sts.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_sts.erl b/src/erlcloud_sts.erl index 1882b050a..bd6e8c508 100644 --- a/src/erlcloud_sts.erl +++ b/src/erlcloud_sts.erl @@ -15,6 +15,7 @@ -define(API_VERSION, "2011-06-15"). -define(UTC_TO_GREGORIAN, 62167219200). +-define(EXTERNAL_ID_MAX_LEN, 1224). assume_role(AwsConfig, RoleArn, RoleSessionName, DurationSeconds) -> @@ -37,7 +38,7 @@ assume_role(AwsConfig, RoleArn, RoleSessionName, DurationSeconds, ExternalId) ExternalIdPart = case ExternalId of undefined -> []; - _ when length(ExternalId) >= 2, length(ExternalId) =< 96 -> [{"ExternalId", ExternalId}] + _ when length(ExternalId) >= 2, length(ExternalId) =< ?EXTERNAL_ID_MAX_LEN -> [{"ExternalId", ExternalId}] end, Xml = sts_query(AwsConfig, "AssumeRole", Params ++ ExternalIdPart), From 64414bfd0f2d98862def30f6472797c3f5856c5b Mon Sep 17 00:00:00 2001 From: Aaron O'Hagan Date: Fri, 22 Nov 2019 14:04:44 +0000 Subject: [PATCH 093/310] add paris, stockholm, bahrain, hongkong, osaka to s3 --- src/erlcloud_s3.erl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index d139b5a92..96336f6ae 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -143,14 +143,21 @@ configure(AccessKeyID, SecretAccessKey, Host, Port, Scheme) -> | 'us-east-1' | 'us-east-2' | 'us-west-1' + | 'us-west-2' + | 'ca-central-1' | 'eu-west-1' | 'eu-west-2' + | 'eu-west-3' + | 'eu-north-1' | 'eu-central-1' | 'ap-south-1' | 'ap-southeast-1' | 'ap-southeast-2' | 'ap-northeast-1' | 'ap-northeast-2' + | 'ap-northeast-3' + | 'ap-east-1' + | 'me-south-1' | 'sa-east-1'. @@ -237,14 +244,20 @@ encode_location_constraint('us-east-1') -> undefined; encode_location_constraint('us-east-2') -> "us-east-2"; encode_location_constraint('us-west-1') -> "us-west-1"; encode_location_constraint('us-west-2') -> "us-west-2"; +encode_location_constraint('ca-central-1') -> "ca-central-1"; encode_location_constraint('eu-west-1') -> "eu-west-1"; encode_location_constraint('eu-west-2') -> "eu-west-2"; +encode_location_constraint('eu-west-3') -> "eu-west-3"; +encode_location_constraint('eu-north-1') -> "eu-north-1"; encode_location_constraint('eu-central-1') -> "eu-central-1"; encode_location_constraint('ap-south-1') -> "ap-south-1"; encode_location_constraint('ap-southeast-1') -> "ap-southeast-1"; encode_location_constraint('ap-southeast-2') -> "ap-southeast-2"; encode_location_constraint('ap-northeast-1') -> "ap-northeast-1"; encode_location_constraint('ap-northeast-2') -> "ap-northeast-2"; +encode_location_constraint('ap-northeast-3') -> "ap-northeast-3"; +encode_location_constraint('ap-east-1') -> "ap-east-1"; +encode_location_constraint('me-south-1') -> "me-south-1"; encode_location_constraint('sa-east-1') -> "sa-east-1"; encode_location_constraint(_) -> undefined. From b8d35328873ce4621e1d65998e2cdb3c1547e534 Mon Sep 17 00:00:00 2001 From: daniel-greening <36239549+daniel-greening@users.noreply.github.com> Date: Mon, 9 Dec 2019 13:32:58 +0000 Subject: [PATCH 094/310] add cloudformation create stack function (#625) * add cloudformation create stack function * fix cloudformation_hrl def * move aws query encoding logic to erlcloud_util * set relevant empty list defaults for create stack input * add cloudformation delete stack function * add cloudformation update stack function * fix delete stack return --- include/erlcloud_cloudformation.hrl | 82 +++++++ src/erlcloud_cloudformation.erl | 201 ++++++++++++++++- src/erlcloud_util.erl | 50 ++++- test/erlcloud_cloudformation_tests.erl | 289 +++++++++++++++++++++++++ 4 files changed, 616 insertions(+), 6 deletions(-) create mode 100644 include/erlcloud_cloudformation.hrl diff --git a/include/erlcloud_cloudformation.hrl b/include/erlcloud_cloudformation.hrl new file mode 100644 index 000000000..1e82060a3 --- /dev/null +++ b/include/erlcloud_cloudformation.hrl @@ -0,0 +1,82 @@ +-ifndef(erlcloud_cloudformation_hrl). +-define(erlcloud_cloudformation_hrl, 0). + +-include("erlcloud.hrl"). + +-record(cloudformation_create_stack_input, { + capabilities = [] :: [string()], %% list containing CAPABILITY_IAM | CAPABILITY_NAMED_IAM | CAPABILITY_AUTO_EXPAND + client_request_token :: string(), + disable_rollback :: boolean(), + enable_termination_protection :: boolean(), + notification_arns = [] :: [string()], + on_failure = "ROLLBACK" :: string(), %% DO_NOTHING | ROLLBACK | DELETE + parameters = [] :: [cloudformation_parameter()], + resource_types = [] :: [string()], + role_arn :: string(), + rollback_configuration :: cloudformation_rollback_configuration(), + stack_name :: string(), + stack_policy_body :: string(), + stack_policy_url :: string(), + tags = []:: [cloudformation_tag()], + template_body :: string(), + template_url :: string(), + timeout_in_minutes :: integer() +}). + +-record(cloudformation_update_stack_input, { + capabilities = [] :: [string()], %% list containing CAPABILITY_IAM | CAPABILITY_NAMED_IAM | CAPABILITY_AUTO_EXPAND + client_request_token :: string(), + notification_arns = [] :: [string()], + parameters = [] :: [cloudformation_parameter()], + resource_types = [] :: [string()], + role_arn :: string(), + rollback_configuration :: cloudformation_rollback_configuration(), + stack_name :: string(), + stack_policy_body :: string(), + stack_policy_during_update_body :: string(), + stack_policy_during_update_url :: string(), + stack_policy_url :: string(), + tags = []:: [cloudformation_tag()], + template_body :: string(), + template_url :: string(), + use_previous_template :: boolean() +}). + +-record(cloudformation_delete_stack_input, { + client_request_token :: string(), + retain_resources = [] :: [string()], + role_arn :: string(), + stack_name :: string() +}). + +-record(cloudformation_parameter, { + parameter_key :: string(), + parameter_value :: string(), + resolved_value :: string(), + use_previous_value :: boolean() +}). + +-record(cloudformation_rollback_configuration, { + monitoring_time_in_minutes :: integer(), + rollback_triggers:: [cloudformation_rollback_trigger()] +}). + +-record(cloudformation_rollback_trigger, { + arn :: string(), + type:: string() +}). + +-record(cloudformation_tag, { + key :: string(), + value :: string() +}). + +-type(cloudformation_create_stack_input() :: #cloudformation_create_stack_input{}). +-type(cloudformation_update_stack_input() :: #cloudformation_update_stack_input{}). +-type(cloudformation_delete_stack_input() :: #cloudformation_delete_stack_input{}). +-type(cloudformation_parameter() :: #cloudformation_parameter{}). +-type(cloudformation_rollback_configuration() :: #cloudformation_rollback_configuration{}). +-type(cloudformation_rollback_trigger() :: #cloudformation_rollback_trigger{}). +-type(cloudformation_tag() :: #cloudformation_tag{}). + +-endif. diff --git a/src/erlcloud_cloudformation.erl b/src/erlcloud_cloudformation.erl index 87366a1f0..24592c733 100644 --- a/src/erlcloud_cloudformation.erl +++ b/src/erlcloud_cloudformation.erl @@ -2,6 +2,7 @@ -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). +-include("erlcloud_cloudformation.hrl"). -define(API_VERSION, "2010-05-15"). @@ -23,10 +24,16 @@ %% Cloud Formation API Functions -export ([ + create_stack/2, + create_stack/1, list_stacks_all/1, list_stacks_all/2, + update_stack/2, + update_stack/1, list_stacks/2, list_stacks/1, + delete_stack/2, + delete_stack/1, list_stack_resources_all/2, list_stack_resources_all/3, list_stack_resources/2, @@ -74,6 +81,27 @@ new(AccessKeyID, SecretAccessKey) -> %%============================================================================== %% Cloud Formation API Functions %%============================================================================== +-spec create_stack(cloudformation_create_stack_input()) -> + {ok, string()} | {error, error_reason()}. +create_stack(Spec = #cloudformation_create_stack_input{}) -> + create_stack(Spec, default_config()). + +-spec create_stack(cloudformation_create_stack_input(), aws_config()) -> + {ok, string()} | {error, error_reason()}. +create_stack(Spec = #cloudformation_create_stack_input{}, Config = #aws_config{}) -> + + Params = create_stack_input_to_params(Spec), + case cloudformation_request(Config, "CreateStack", Params) of + {ok, XmlNode} -> + StackId = erlcloud_xml:get_text( + "/CreateStackResponse/CreateStackResult/StackId", + XmlNode, + undefined), + {ok, StackId}; + {error, Error} -> + {error, Error} + end. + -spec list_stacks_all(params()) -> {ok, cloudformation_list()} | {error, error_reason()}. list_stacks_all(Params) -> list_stacks_all(Params, default_config()). @@ -115,6 +143,44 @@ list_stacks(Params, Config = #aws_config{}) -> {error, Error} end. +-spec update_stack(cloudformation_update_stack_input()) -> + {ok, string()} | {error, error_reason()}. +update_stack(Spec = #cloudformation_update_stack_input{}) -> + update_stack(Spec, default_config()). + +-spec update_stack(cloudformation_update_stack_input(), aws_config()) -> + {ok, string()} | {error, error_reason()}. +update_stack(Spec = #cloudformation_update_stack_input{}, Config = #aws_config{}) -> + + Params = update_stack_input_to_params(Spec), + case cloudformation_request(Config, "UpdateStack", Params) of + {ok, XmlNode} -> + StackId = erlcloud_xml:get_text( + "/UpdateStackResponse/UpdateStackResult/StackId", + XmlNode, + undefined), + {ok, StackId}; + {error, Error} -> + {error, Error} + end. + +-spec delete_stack(cloudformation_delete_stack_input()) -> + {ok, string()} | {error, error_reason()}. +delete_stack(Spec = #cloudformation_delete_stack_input{}) -> + delete_stack(Spec, default_config()). + +-spec delete_stack(cloudformation_delete_stack_input(), aws_config()) -> + ok | {error, error_reason()}. +delete_stack(Spec = #cloudformation_delete_stack_input{}, Config = #aws_config{}) -> + + Params = delete_stack_input_to_params(Spec), + case cloudformation_request(Config, "DeleteStack", Params) of + {ok, _XmlNode} -> + ok; + {error, Error} -> + {error, Error} + end. + -spec list_stack_resources_all(params(), string()) -> {ok, cloudformation_list()} | {error, error_reason()}. list_stack_resources_all(Params, StackName) -> @@ -659,6 +725,137 @@ extract_account_limit(XmlNode) -> {value, "Value", optional_text} ], XmlNode). - - +create_stack_input_to_params(Spec) -> + lists:flatten([ + base_create_stack_input_params(Spec), + erlcloud_util:encode_object_list( + "Parameters", + cloudformation_parameters_fields(Spec#cloudformation_create_stack_input.parameters)), + erlcloud_util:encode_list("Capabilities", Spec#cloudformation_create_stack_input.capabilities), + erlcloud_util:encode_list("NotificationARNs", Spec#cloudformation_create_stack_input.notification_arns), + erlcloud_util:encode_list("ResourceTypes", Spec#cloudformation_create_stack_input.resource_types), + erlcloud_util:encode_object( + "RollbackConfiguration", + cloudformation_rollback_configuration_fields(Spec#cloudformation_create_stack_input.rollback_configuration) + ), + erlcloud_util:encode_object_list( + "Tags", + cloudformation_tags_fields(Spec#cloudformation_create_stack_input.tags) + ) + ]). + +base_create_stack_input_params(Spec) -> + [ + { "ClientRequestToken", Spec#cloudformation_create_stack_input.client_request_token }, + { "DisableRollback", Spec#cloudformation_create_stack_input.disable_rollback }, + { "EnableTerminationProtection", Spec#cloudformation_create_stack_input.enable_termination_protection }, + { "OnFailure", Spec#cloudformation_create_stack_input.on_failure }, + { "RoleARN", Spec#cloudformation_create_stack_input.role_arn }, + { "StackName", Spec#cloudformation_create_stack_input.stack_name }, + { "StackPolicyBody", Spec#cloudformation_create_stack_input.stack_policy_body }, + { "StackPolicyURL", Spec#cloudformation_create_stack_input.stack_policy_url }, + { "TemplateBody", Spec#cloudformation_create_stack_input.template_body }, + { "TemplateURL", Spec#cloudformation_create_stack_input.template_url }, + { "TimeoutInMinutes", Spec#cloudformation_create_stack_input.timeout_in_minutes } + ]. + +update_stack_input_to_params(Spec) -> + lists:flatten([ + update_create_stack_input_params(Spec), + erlcloud_util:encode_list("Capabilities", Spec#cloudformation_update_stack_input.capabilities), + erlcloud_util:encode_list("NotificationARNs", Spec#cloudformation_update_stack_input.notification_arns), + erlcloud_util:encode_object_list( + "Parameters", + cloudformation_parameters_fields(Spec#cloudformation_update_stack_input.parameters) + ), + erlcloud_util:encode_list("ResourceTypes", Spec#cloudformation_update_stack_input.resource_types), + erlcloud_util:encode_object( + "RollbackConfiguration", + cloudformation_rollback_configuration_fields(Spec#cloudformation_update_stack_input.rollback_configuration) + ), + erlcloud_util:encode_object_list( + "Tags", + cloudformation_tags_fields(Spec#cloudformation_update_stack_input.tags) + ) + ]). + +update_create_stack_input_params(Spec) -> + [ + { "ClientRequestToken", Spec#cloudformation_update_stack_input.client_request_token }, + { "RoleARN", Spec#cloudformation_update_stack_input.role_arn }, + { "StackName", Spec#cloudformation_update_stack_input.stack_name }, + { "StackPolicyBody", Spec#cloudformation_update_stack_input.stack_policy_body }, + { "StackPolicyDuringUpdateBody", Spec#cloudformation_update_stack_input.stack_policy_during_update_body }, + { "StackPolicyDuringUpdateURL", Spec#cloudformation_update_stack_input.stack_policy_during_update_url}, + { "StackPolicyURL", Spec#cloudformation_update_stack_input.stack_policy_url }, + { "TemplateBody", Spec#cloudformation_update_stack_input.template_body }, + { "TemplateURL", Spec#cloudformation_update_stack_input.template_url }, + { "UsePreviousTemplate", Spec#cloudformation_update_stack_input.use_previous_template } + ]. + +delete_stack_input_to_params(#cloudformation_delete_stack_input{} = Spec) -> + lists:flatten([ + base_delete_stack_input_params(Spec), + erlcloud_util:encode_list("RetainResources", Spec#cloudformation_delete_stack_input.retain_resources) + ]). + +base_delete_stack_input_params(Spec) -> + [ + { "ClientRequestToken", Spec#cloudformation_delete_stack_input.client_request_token }, + { "RoleARN", Spec#cloudformation_delete_stack_input.role_arn }, + { "StackName", Spec#cloudformation_delete_stack_input.stack_name } + ]. + +cloudformation_parameters_fields(CloudformationParameters) -> + lists:map(fun cloudformation_parameter_fields/1, CloudformationParameters). + +cloudformation_parameter_fields(#cloudformation_parameter{} = Parameters) -> + Params = [ + { "ParameterKey", Parameters#cloudformation_parameter.parameter_key}, + { "ParameterValue", Parameters#cloudformation_parameter.parameter_value}, + { "ResolvedValue", Parameters#cloudformation_parameter.resolved_value}, + { "UsePreviousValue", Parameters#cloudformation_parameter.use_previous_value} + ], + filter_undefined(Params); +cloudformation_parameter_fields(_) -> + []. + +cloudformation_tags_fields(Tags) -> + lists:map(fun cloudformation_tag_fields/1, Tags). + +cloudformation_tag_fields(#cloudformation_tag{} = Tag) -> + Params = [ + { "Key", Tag#cloudformation_tag.key}, + { "Value", Tag#cloudformation_tag.value} + ], + filter_undefined(Params); +cloudformation_tag_fields(_) -> + []. + +cloudformation_rollback_configuration_fields(#cloudformation_rollback_configuration{} = RollbackConfig) -> + lists:flatten([ + [{"MonitoringTimeInMinutes", RollbackConfig#cloudformation_rollback_configuration.monitoring_time_in_minutes}], + erlcloud_util:encode_object_list( + "RollbackTriggers", + cloudformation_rollback_triggers_fields( + RollbackConfig#cloudformation_rollback_configuration.rollback_triggers + ) + ) + ]); +cloudformation_rollback_configuration_fields(_) -> + []. + +cloudformation_rollback_triggers_fields(RollbackTriggers) -> + lists:map(fun cloudformation_rollback_trigger_fields/1, RollbackTriggers). + +cloudformation_rollback_trigger_fields(#cloudformation_rollback_trigger{} = RollbackTrigger) -> + filter_undefined([ + {"Arn", RollbackTrigger#cloudformation_rollback_trigger.arn}, + {"Type", RollbackTrigger#cloudformation_rollback_trigger.type} + ]); +cloudformation_rollback_trigger_fields(_) -> + []. + +filter_undefined(Params) -> + [{K, V} || {K, V} <- Params, V =/= undefined]. diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index 9d08c7f6c..d8836a1d0 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -1,9 +1,23 @@ -module(erlcloud_util). --export([sha_mac/2, sha256_mac/2, md5/1, sha256/1, rand_uniform/1, + +-export([ + sha_mac/2, + sha256_mac/2, + md5/1, + sha256/1, + rand_uniform/1, is_dns_compliant_name/1, - query_all/4, query_all/5, query_all_token/4, make_response/2, - get_items/2, to_string/1, encode_list/2, next_token/2, - filter_undef/1]). + query_all/4, query_all/5, + query_all_token/4, + make_response/2, + get_items/2, + to_string/1, + encode_list/2, + encode_object/2, + encode_object_list/2, + next_token/2, + filter_undef/1 +]). -define(MAX_ITEMS, 1000). @@ -95,11 +109,39 @@ query_all(QueryFun, Config, Action, Params, MaxItems, Marker, Acc) -> Error end. +-spec encode_list(string(), [term()]) -> + {ok, proplists:proplist()}. encode_list(ElementName, Elements) -> Numbered = lists:zip(lists:seq(1, length(Elements)), Elements), [{ElementName ++ ".member." ++ integer_to_list(N), Element} || {N, Element} <- Numbered]. +-spec encode_object(string(), proplists:proplist()) -> + {ok, proplists:proplist()}. +encode_object(ElementName, ElementParameters) -> + lists:map( + fun({Key, Value}) -> + {ElementName ++ "." ++ Key, Value} + end, + ElementParameters + ). + +-spec encode_object_list(string(), [proplists:proplist()]) -> + {ok, proplists:proplist()}. +encode_object_list(Prefix, ElementParameterList) -> + lists:flatten(lists:foldl( + fun(ElementMap, Acc) -> + [lists:map( + fun({Key, Value}) -> + {Prefix ++ ".member." ++ integer_to_list(length(Acc)+1) ++ "." ++ Key, Value} + end, + ElementMap + ) | Acc] + end, + [], + ElementParameterList + )). + make_response(Xml, Result) -> IsTruncated = erlcloud_xml:get_bool("/*/*/IsTruncated", Xml), Marker = erlcloud_xml:get_text("/*/*/Marker", Xml), diff --git a/test/erlcloud_cloudformation_tests.erl b/test/erlcloud_cloudformation_tests.erl index 2338387d5..ebed26a99 100644 --- a/test/erlcloud_cloudformation_tests.erl +++ b/test/erlcloud_cloudformation_tests.erl @@ -5,6 +5,7 @@ -include_lib("eunit/include/eunit.hrl"). -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). +-include("erlcloud_cloudformation.hrl"). -define(_cloudformation_test(T), {?LINE, T}). @@ -16,7 +17,18 @@ describe_cloudformation_test_() -> {foreach, fun start/0, fun stop/1, [ + fun create_stack_input_tests/1, + fun create_stack_input_with_parameters_tests/1, + fun create_stack_input_with_tags_tests/1, + fun create_stack_input_with_rollback_config_tests/1, fun list_stacks_all_output_tests/1, + fun update_stack_input_tests/1, + fun update_stack_input_with_parameters_tests/1, + fun update_stack_input_with_tags_tests/1, + fun update_stack_input_with_rollback_config_tests/1, + fun delete_stack_input_tests/1, + fun delete_stack_input_with_retain_resources_tests/1, + fun delete_stack_input_with_client_request_token_tests/1, fun list_stack_resources_output_tests/1, fun get_template_summary_output_tests/1, fun get_template_output_tests/1, @@ -96,6 +108,113 @@ input_test(Response, {Line, {Description, Fun, Params}}) %% Input Tests %%============================================================================== +create_stack_input_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","CreateStack"}, + {"Version","2010-05-15"}, + {"OnFailure","ROLLBACK"}, + {"StackName","TestStack"} + ], + + input_test(Response, ?_cloudformation_test({"Test create stack input", + ?_f(erlcloud_cloudformation:create_stack(#cloudformation_create_stack_input{stack_name="TestStack"}, #aws_config{})), + ExpectedParams})). + +create_stack_input_with_parameters_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","CreateStack"}, + {"Version","2010-05-15"}, + {"OnFailure","ROLLBACK"}, + {"StackName","TestStack"}, + {"Parameters.member.1.ParameterKey", "TestParamKey1"}, + {"Parameters.member.1.ParameterValue", "TestParamVal1"}, + {"Parameters.member.2.ParameterKey", "TestParamKey2"}, + {"Parameters.member.2.ParameterValue", "TestParamVal2"}, + {"Parameters.member.3.ParameterKey", "TestParamKey3"}, + {"Parameters.member.3.ParameterValue", "TestParamVal3"} + ], + + input_test(Response, ?_cloudformation_test({"Test create stack input", + ?_f(erlcloud_cloudformation:create_stack( + #cloudformation_create_stack_input{ + stack_name="TestStack", + parameters=[ + #cloudformation_parameter{parameter_key="TestParamKey1", parameter_value="TestParamVal1"}, + #cloudformation_parameter{parameter_key="TestParamKey2", parameter_value="TestParamVal2"}, + #cloudformation_parameter{parameter_key="TestParamKey3", parameter_value="TestParamVal3"} + ] + }, + #aws_config{} + )), + ExpectedParams})). + +create_stack_input_with_tags_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","CreateStack"}, + {"Version","2010-05-15"}, + {"OnFailure","ROLLBACK"}, + {"StackName","TestStack"}, + {"Tags.member.1.Key", "TagKey1"}, + {"Tags.member.1.Value", "TagVal1"}, + {"Tags.member.2.Key", "TagKey2"}, + {"Tags.member.2.Value", "TagVal2"} + ], + + input_test(Response, ?_cloudformation_test({"Test create stack input", + ?_f(erlcloud_cloudformation:create_stack( + #cloudformation_create_stack_input{ + stack_name="TestStack", + tags=[ + #cloudformation_tag{key="TagKey0", value="TagVal0"}, + #cloudformation_tag{key="TagKey1", value="TagVal1"} + ] + }, + #aws_config{} + )), + ExpectedParams})). + +create_stack_input_with_rollback_config_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","CreateStack"}, + {"Version","2010-05-15"}, + {"OnFailure","ROLLBACK"}, + {"StackName","TestStack"}, + {"RollbackConfiguration.MonitoringTimeInMinutes", 10}, + {"RollbackConfiguration.RollbackTriggers.member.1.Arn", "TestARN"}, + {"RollbackConfiguration.RollbackTriggers.member.1.Type", "TestType"} + ], + + input_test(Response, ?_cloudformation_test({"Test create stack input", + ?_f(erlcloud_cloudformation:create_stack( + #cloudformation_create_stack_input{ + stack_name="TestStack", + rollback_configuration = #cloudformation_rollback_configuration{ + monitoring_time_in_minutes = 10, + rollback_triggers = [ + #cloudformation_rollback_trigger{arn="TestARN", type="TestType"} + ] + } + }, + #aws_config{} + )), + ExpectedParams})). + list_stacks_all_input_tests(_) -> Response = {ok, element(1, xmerl_scan:string( binary_to_list(<<"null">>) @@ -107,6 +226,176 @@ list_stacks_all_input_tests(_) -> ?_f(erlcloud_cloudformation:list_stacks_all([], #aws_config{})), ExpectedParams})). +update_stack_input_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","UpdateStack"}, + {"Version","2010-05-15"}, + {"StackName","TestStack"} + ], + + input_test(Response, ?_cloudformation_test({"Test update stack input", + ?_f(erlcloud_cloudformation:update_stack(#cloudformation_update_stack_input{stack_name="TestStack"}, #aws_config{})), + ExpectedParams})). + + +update_stack_input_with_parameters_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","UpdateStack"}, + {"Version","2010-05-15"}, + {"StackName","TestStack"}, + {"Parameters.member.1.ParameterKey", "TestParamKey1"}, + {"Parameters.member.1.ParameterValue", "TestParamVal1"}, + {"Parameters.member.2.ParameterKey", "TestParamKey2"}, + {"Parameters.member.2.ParameterValue", "TestParamVal2"}, + {"Parameters.member.3.ParameterKey", "TestParamKey3"}, + {"Parameters.member.3.ParameterValue", "TestParamVal3"} + ], + + input_test(Response, ?_cloudformation_test({"Test update stack input with parameters", + ?_f(erlcloud_cloudformation:update_stack( + #cloudformation_update_stack_input{ + stack_name="TestStack", + parameters=[ + #cloudformation_parameter{parameter_key="TestParamKey1", parameter_value="TestParamVal1"}, + #cloudformation_parameter{parameter_key="TestParamKey2", parameter_value="TestParamVal2"}, + #cloudformation_parameter{parameter_key="TestParamKey3", parameter_value="TestParamVal3"} + ] + }, + #aws_config{} + )), + ExpectedParams})). + +update_stack_input_with_tags_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","UpdateStack"}, + {"Version","2010-05-15"}, + {"StackName","TestStack"}, + {"Tags.member.1.Key", "TagKey1"}, + {"Tags.member.1.Value", "TagVal1"}, + {"Tags.member.2.Key", "TagKey2"}, + {"Tags.member.2.Value", "TagVal2"} + ], + + input_test(Response, ?_cloudformation_test({"Test update stack input with tags", + ?_f(erlcloud_cloudformation:update_stack( + #cloudformation_update_stack_input{ + stack_name="TestStack", + tags=[ + #cloudformation_tag{key="TagKey0", value="TagVal0"}, + #cloudformation_tag{key="TagKey1", value="TagVal1"} + ] + }, + #aws_config{} + )), + ExpectedParams})). + +update_stack_input_with_rollback_config_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","UpdateStack"}, + {"Version","2010-05-15"}, + {"StackName","TestStack"}, + {"RollbackConfiguration.MonitoringTimeInMinutes", 10}, + {"RollbackConfiguration.RollbackTriggers.member.1.Arn", "TestARN"}, + {"RollbackConfiguration.RollbackTriggers.member.1.Type", "TestType"} + ], + + input_test(Response, ?_cloudformation_test({"Test update stack input with rollback configuration", + ?_f(erlcloud_cloudformation:update_stack( + #cloudformation_update_stack_input{ + stack_name="TestStack", + rollback_configuration = #cloudformation_rollback_configuration{ + monitoring_time_in_minutes = 10, + rollback_triggers = [ + #cloudformation_rollback_trigger{arn="TestARN", type="TestType"} + ] + } + }, + #aws_config{} + )), + ExpectedParams})). + +delete_stack_input_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action", "DeleteStack"}, + {"Version", "2010-05-15"}, + {"StackName", "TestStack"} + ], + + input_test(Response, ?_cloudformation_test({"Test Delete Stack input", + ?_f(erlcloud_cloudformation:delete_stack( + #cloudformation_delete_stack_input{stack_name="TestStack"}, + #aws_config{} + )), + ExpectedParams})). + +delete_stack_input_with_retain_resources_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action", "DeleteStack"}, + {"Version", "2010-05-15"}, + {"StackName", "TestStack"}, + {"RetainResources.member.1", "arn::ec2::MyTestInstance1"}, + {"RetainResources.member.2", "arn::ec2::MyTestInstance2"} + ], + + input_test(Response, ?_cloudformation_test({"Test Delete Stack input", + ?_f(erlcloud_cloudformation:delete_stack( + #cloudformation_delete_stack_input{ + stack_name="TestStack", + retain_resources = [ + "arn::ec2::MyTestInstance1", + "arn::ec2::MyTestInstance2" + ] + }, + #aws_config{} + )), + ExpectedParams})). + +delete_stack_input_with_client_request_token_tests(_) -> + Response = {ok, element(1, xmerl_scan:string( + binary_to_list(<<"null">>) + ))}, + + ExpectedParams = [ + {"Action","DeleteStack"}, + {"Version","2010-05-15"}, + {"StackName","TestStack"}, + {"ClientRequestToken","TestClientRequestToken"} + ], + + input_test(Response, ?_cloudformation_test({"Test Delete Stack input", + ?_f(erlcloud_cloudformation:delete_stack( + #cloudformation_delete_stack_input{ + stack_name="TestStack", + client_request_token="TestClientRequestToken" + }, + #aws_config{} + )), + ExpectedParams})). + list_stack_resources_input_tests(_) -> Response = {ok, element(1, xmerl_scan:string( binary_to_list(<<"null" From 77c94878506de61f59d76fefddd16770ae934822 Mon Sep 17 00:00:00 2001 From: Danila Fediashchin Date: Mon, 30 Dec 2019 16:14:00 +0000 Subject: [PATCH 095/310] Handle empty maps in ddb streams --- src/erlcloud_ddb_streams.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/erlcloud_ddb_streams.erl b/src/erlcloud_ddb_streams.erl index aa17fc371..d7cdd73a8 100644 --- a/src/erlcloud_ddb_streams.erl +++ b/src/erlcloud_ddb_streams.erl @@ -217,6 +217,9 @@ undynamize_value_untyped({<<"BS">>, Values}, _) -> [base64:decode(Value) || Value <- Values]; undynamize_value_untyped({<<"L">>, List}, Opts) -> [undynamize_value_untyped(Value, Opts) || [Value] <- List]; +undynamize_value_untyped({<<"M">>, [{}]}, _Opts) -> + %% jsx returns [{}] for empty objects + []; undynamize_value_untyped({<<"M">>, Map}, Opts) -> [undynamize_attr_untyped(Attr, Opts) || Attr <- Map]. @@ -269,6 +272,8 @@ undynamize_value_typed({<<"BS">>, Values}, _) -> {bs, [base64:decode(Value) || Value <- Values]}; undynamize_value_typed({<<"L">>, List}, Opts) -> {l, [undynamize_value_typed(Value, Opts) || [Value] <- List]}; +undynamize_value_typed({<<"M">>, [{}]}, _Opts) -> + {m, []}; undynamize_value_typed({<<"M">>, Map}, Opts) -> {m, [undynamize_attr_typed(Attr, Opts) || Attr <- Map]}. From 0e089d55eaf781608feb2a991cdb79f826947dda Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Sun, 5 Jan 2020 23:08:09 -0600 Subject: [PATCH 096/310] erlcloud_ddb2: Update support for version 2019.11.21 global tables * Add DescribeGlobalTableSettings/UpdateGlobalTableSettings support for version 2017.11.29 global tables * Add DescribeTableReplicaAutoScaling/UpdateTableReplicaAutoScaling support for version 2019.11.21 global tables * Update UpdateTable serialization to serialize ReplicaUpdates property * Update DescribeTable serialization deserialize GlobalTableVersion and Replicas properties --- include/erlcloud_ddb2.hrl | 85 ++- src/erlcloud_ddb2.erl | 649 ++++++++++++++++++- test/erlcloud_ddb2_tests.erl | 1138 ++++++++++++++++++++++++++++++++++ 3 files changed, 1870 insertions(+), 2 deletions(-) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index d211c1153..320ee4e28 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -16,16 +16,78 @@ -type date_time() :: number(). -type global_table_status() :: creating | active | deleting | updating. +-type replica_status() :: creating | active | deleting | active. -type table_status() :: creating | updating | deleting | active. -type backup_status() :: creating | deleted | available. -type index_status() :: creating | updating | deleting | active. -type continuous_backups_status() :: enabled | disabled. -type point_in_time_recovery_status() :: enabling | enabled | disabled. +-record(ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description, + {target_value :: undefined | number(), + disable_scale_in :: undefined | boolean(), + scale_in_cooldown :: undefined | number(), + scale_out_cooldown :: undefined | number()}). + +-record(ddb2_auto_scaling_policy_description, + {policy_name :: undefined | binary(), + target_tracking_scaling_policy_configuration :: undefined | #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{}}). + +-record(ddb2_auto_scaling_settings_description, + {auto_scaling_disabled :: undefined | boolean(), + auto_scaling_role_arn :: undefined | binary(), + maximum_units :: undefined | pos_integer(), + minimum_units :: undefined | pos_integer(), + scaling_policies :: undefined | [#ddb2_auto_scaling_policy_description{}]}). + +-record(ddb2_provisioned_throughput_override, + {read_capacity_units :: undefined | pos_integer()}). + +-record(ddb2_replica_global_secondary_index_description, + {index_name :: undefined | binary(), + provisioned_throughput_override :: undefined | #ddb2_provisioned_throughput_override{}}). + +-record(ddb2_replica_auto_scaling_description, + {global_secondary_indexes :: undefined | [#ddb2_replica_global_secondary_index_description{}], + region_name :: undefined | binary(), + replica_provisioned_read_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, + replica_provisioned_write_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, + replica_status :: undefined | replica_status()}). + -record(ddb2_replica_description, - {region_name :: undefined | binary() + {global_secondary_indexes :: undefined | [#ddb2_replica_global_secondary_index_description{}], + kms_master_key_id :: undefined | binary(), + provisioned_throughput_override :: undefined | #ddb2_provisioned_throughput_override{}, + region_name :: undefined | binary(), + replica_status :: undefined | replica_status(), + replica_status_description :: undefined | binary(), + replica_status_percent_progress :: undefined | binary() }). +-record(ddb2_replica_global_secondary_index_settings_description, + {index_name :: undefined | binary(), + index_status :: undefined | index_status(), + provisioned_read_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, + provisioned_read_capacity_units :: undefined | pos_integer(), + provisioned_write_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, + provisioned_write_capacity_units :: undefined | pos_integer()}). + +-record(ddb2_replica_global_secondary_index_auto_scaling_description, + {index_name :: undefined | binary(), + index_status :: undefined | binary(), + provisioned_read_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, + provisioned_write_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}}). + +-record(ddb2_replica_settings_description, + {region_name :: undefined | binary(), + replica_billing_mode_summary :: undefined | binary(), + replica_global_secondary_index_settings :: undefined | [#ddb2_replica_global_secondary_index_settings_description{}], + replica_provisioned_read_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, + replica_provisioned_read_capacity_units :: undefined | number(), + replica_provisioned_write_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, + replica_provisioned_write_capacity_units :: undefined | number(), + replica_status :: undefined | replica_status()}). + -record(ddb2_global_table_description, {creation_date_time :: undefined | number(), global_table_arn :: undefined | binary(), @@ -88,12 +150,14 @@ billing_mode_summary :: undefined | #ddb2_billing_mode_summary{}, creation_date_time :: undefined | number(), global_secondary_indexes :: undefined | [#ddb2_global_secondary_index_description{}], + global_table_version :: undefined | binary(), item_count :: undefined | integer(), key_schema :: undefined | erlcloud_ddb2:key_schema(), latest_stream_arn :: undefined | binary(), latest_stream_label :: undefined | binary(), local_secondary_indexes :: undefined | [#ddb2_local_secondary_index_description{}], provisioned_throughput :: undefined | #ddb2_provisioned_throughput_description{}, + replicas :: undefined | [#ddb2_replica_description{}], restore_summary :: undefined | #ddb2_restore_summary{}, sse_description :: undefined | erlcloud_ddb2:sse_description(), stream_specification :: undefined | erlcloud_ddb2:stream_specification(), @@ -103,6 +167,11 @@ table_status :: undefined | table_status() }). +-record(ddb2_table_auto_scaling_description, + {replicas :: undefined, + table_name :: undefined | binary(), + table_status :: undefined | table_status()}). + -record(ddb2_consumed_capacity, {capacity_units :: undefined | number(), global_secondary_indexes :: undefined | [{erlcloud_ddb2:index_name(), number()}], @@ -155,6 +224,10 @@ {global_table_description :: undefined | #ddb2_global_table_description{} }). +-record(ddb2_describe_global_table_settings, + {global_table_name :: undefined | binary(), + replica_settings :: undefined | [#ddb2_replica_settings_description{}]}). + -record(ddb2_describe_limits, {account_max_read_capacity_units :: undefined | pos_integer(), account_max_write_capacity_units :: undefined | pos_integer(), @@ -166,6 +239,9 @@ {table :: undefined | #ddb2_table_description{} }). +-record(ddb2_describe_table_replica_auto_scaling, + {table_auto_scaling_description :: undefined}). + -record(ddb2_time_to_live_description, {attribute_name :: undefined | erlcloud_ddb2:attr_name(), time_to_live_status :: undefined | erlcloud_ddb2:time_to_live_status() @@ -226,10 +302,17 @@ {global_table_description :: undefined | #ddb2_global_table_description{} }). +-record(ddb2_update_global_table_settings, + {global_table_name :: undefined | binary(), + replica_settings :: undefined | [#ddb2_replica_settings_description{}]}). + -record(ddb2_update_table, {table_description :: undefined | #ddb2_table_description{} }). +-record(ddb2_update_table_replica_auto_scaling, + {table_auto_scaling_description :: undefined | #ddb2_table_auto_scaling_description{}}). + -record(ddb2_time_to_live_specification, {attribute_name :: undefined | erlcloud_ddb2:attr_name(), enabled :: undefined | boolean() diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 5f5dfab7a..f96d1c1c8 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -104,8 +104,10 @@ describe_backup/1, describe_backup/2, describe_backup/3, describe_continuous_backups/1, describe_continuous_backups/2, describe_continuous_backups/3, describe_global_table/1, describe_global_table/2, describe_global_table/3, + describe_global_table_settings/1, describe_global_table_settings/2, describe_global_table_settings/3, describe_limits/0, describe_limits/1, describe_limits/2, describe_table/1, describe_table/2, describe_table/3, + describe_table_replica_auto_scaling/1, describe_table_replica_auto_scaling/2, describe_table_replica_auto_scaling/3, describe_time_to_live/1, describe_time_to_live/2, describe_time_to_live/3, get_item/2, get_item/3, get_item/4, list_backups/0, list_backups/1, list_backups/2, @@ -125,7 +127,9 @@ update_continuous_backups/2, update_continuous_backups/3, update_continuous_backups/4, update_item/3, update_item/4, update_item/5, update_global_table/2, update_global_table/3, update_global_table/4, + update_global_table_settings/2, update_global_table_settings/3, update_table/2, update_table/3, update_table/4, update_table/5, + update_table_replica_auto_scaling/2, update_table_replica_auto_scaling/3, update_time_to_live/2, update_time_to_live/3, update_time_to_live/4 ]). @@ -164,7 +168,10 @@ delete_item_opts/0, delete_item_return/0, delete_table_return/0, + describe_global_table_return/0, + describe_global_table_settings_return/0, describe_table_return/0, + describe_table_replica_auto_scaling_return/0, describe_time_to_live_return/0, expected_opt/0, expression/0, @@ -232,7 +239,10 @@ update_item_opt/0, update_item_opts/0, update_item_return/0, + update_global_table_return/0, + update_global_table_settings_return/0, update_table_return/0, + update_table_replica_auto_scaling_return/0, update_time_to_live_return/0, write_units/0 ]). @@ -389,6 +399,24 @@ default_config() -> erlcloud_aws:default_config(). -type ok_return(T) :: {ok, T} | {error, term()}. -type client_request_token() :: binary(). +-type auto_scaling_target_tracking_scaling_policy_configuration_update_opt() :: {target_value, number()}| + {disable_scale_in, boolean()}| + {scale_in_cooldown, pos_integer()}| + {scale_out_cooldown, pos_integer()}. +-type auto_scaling_target_tracking_scaling_policy_configuration_update_opts() :: [auto_scaling_target_tracking_scaling_policy_configuration_update_opt()]. + +-type auto_scaling_policy_opt() :: {policy_name, binary()}| + {target_tracking_scaling_policy_configuration, auto_scaling_target_tracking_scaling_policy_configuration_update_opts()}. + +-type auto_scaling_policy_opts() :: [auto_scaling_policy_opt()]. + +-type auto_scaling_settings_update_opt() :: {auto_scaling_disabled, boolean()}| + {auto_scaling_role_arn, binary()}| + {maximum_units, non_neg_integer()}| + {minimum_units, non_neg_integer()}| + {scaling_policy_update, auto_scaling_policy_opts()}. +-type auto_scaling_settings_update_opts() :: [auto_scaling_settings_update_opt()]. + %%%------------------------------------------------------------------------------ %%% Shared Dynamizers %%%------------------------------------------------------------------------------ @@ -643,6 +671,46 @@ dynamize_return_item_collection_metrics(none) -> dynamize_return_item_collection_metrics(size) -> <<"SIZE">>. +-spec dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opt(auto_scaling_target_tracking_scaling_policy_configuration_update_opt()) -> json_pair(). +dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opt({target_value, TargetValue}) -> + {<<"TargetValue">>, TargetValue}; +dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opt({disable_scale_in, DisableScaleIn}) -> + {<<"DisableScaleIn">>, DisableScaleIn}; +dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opt({scale_in_cooldown, ScaleInCooldown}) -> + {<<"ScaleInCooldown">>, ScaleInCooldown}; +dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opt({scale_out_cooldown, ScaleOutCooldown}) -> + {<<"ScaleOutCooldown">>, ScaleOutCooldown}. + +-spec dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opts(auto_scaling_policy_opts()) -> jsx:json_term(). +dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opts(Opts) -> + dynamize_maybe_list(fun dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opt/1, Opts). + +-spec dynamize_auto_scaling_policy_update_opt(auto_scaling_policy_opt()) -> json_pair(). +dynamize_auto_scaling_policy_update_opt({policy_name, PolicyName}) -> + {<<"PolicyName">>, PolicyName}; +dynamize_auto_scaling_policy_update_opt({target_tracking_scaling_policy_configuration, TargetTrackingScalingPolicyConfiguration}) -> + {<<"TargetTrackingScalingPolicyConfiguration">>, dynamize_auto_scaling_target_tracking_scaling_policy_configuration_update_opts(TargetTrackingScalingPolicyConfiguration)}. + +-spec dynamize_auto_scaling_policy_update_opts(auto_scaling_policy_opts()) -> jsx:json_term(). +dynamize_auto_scaling_policy_update_opts(Opts) -> + dynamize_maybe_list(fun dynamize_auto_scaling_policy_update_opt/1, Opts). + +-spec dynamize_auto_scaling_settings_update_opt(auto_scaling_settings_update_opt()) -> json_pair(). +dynamize_auto_scaling_settings_update_opt({auto_scaling_disabled, AutoScalingDisabled}) -> + {<<"AutoScalingDisabled">>, AutoScalingDisabled}; +dynamize_auto_scaling_settings_update_opt({auto_scaling_role_arn, AutoScalingRoleArn}) -> + {<<"AutoScalingRoleArn">>, AutoScalingRoleArn}; +dynamize_auto_scaling_settings_update_opt({maximum_units, MaximumUnits}) -> + {<<"MaximumUnits">>, MaximumUnits}; +dynamize_auto_scaling_settings_update_opt({minimum_units, MinimumUnits}) -> + {<<"MinimumUnits">>, MinimumUnits}; +dynamize_auto_scaling_settings_update_opt({scaling_policy_update, ScalingPolicyUpdate}) -> + {<<"ScalingPolicyUpdate">>, dynamize_auto_scaling_policy_update_opts(ScalingPolicyUpdate)}. + +-spec dynamize_auto_scaling_settings_update_opts(auto_scaling_settings_update_opts()) -> jsx:json_term(). +dynamize_auto_scaling_settings_update_opts(Opts) -> + dynamize_maybe_list(fun dynamize_auto_scaling_settings_update_opt/1, Opts). + %%%------------------------------------------------------------------------------ %%% Shared Undynamizers %%%------------------------------------------------------------------------------ @@ -817,6 +885,12 @@ undynamize_global_table_status(<<"UPDATING">>, _) -> updating; undynamize_global_table_status(<<"DELETING">>, _) -> deleting; undynamize_global_table_status(<<"ACTIVE">>, _) -> active. +-spec undynamize_replica_status(binary(), undynamize_opts()) -> replica_status(). +undynamize_replica_status(<<"CREATING">>, _) -> creating; +undynamize_replica_status(<<"UPDATING">>, _) -> updating; +undynamize_replica_status(<<"DELETING">>, _) -> deleting; +undynamize_replica_status(<<"ACTIVE">>, _) -> active. + -spec undynamize_replica_description(jsx:json_term(), undynamize_opts()) -> replica_description(). undynamize_replica_description(ReplicaDescription, _) -> #ddb2_replica_description{region_name = proplists:get_value(<<"RegionName">>, ReplicaDescription)}. @@ -1239,6 +1313,115 @@ restore_summary_record() -> {<<"SourceTableArn">>, #ddb2_restore_summary.source_table_arn, fun id/2} ]}. +-spec auto_scaling_target_tracking_scaling_policy_configuration_description_record() -> record_desc(). +auto_scaling_target_tracking_scaling_policy_configuration_description_record() -> + {#ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{}, + [{<<"TargetValue">>, #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description.target_value, fun id/2}, + {<<"DisableScaleIn">>, #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description.disable_scale_in, fun id/2}, + {<<"ScaleInCooldown">>, #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description.scale_in_cooldown, fun id/2}, + {<<"ScaleOutCooldown">>, #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description.scale_out_cooldown, fun id/2}]}. + +-spec auto_scaling_policy_description_record() -> record_desc(). +auto_scaling_policy_description_record() -> + {#ddb2_auto_scaling_policy_description{}, + [{<<"PolicyName">>, #ddb2_auto_scaling_policy_description.policy_name, fun id/2}, + {<<"TargetTrackingScalingPolicyConfiguration">>, #ddb2_auto_scaling_policy_description.target_tracking_scaling_policy_configuration, + fun(V, Opts) -> undynamize_record(auto_scaling_target_tracking_scaling_policy_configuration_description_record(), V, Opts) end}]}. + +-spec auto_scaling_settings_description_record() -> record_desc(). +auto_scaling_settings_description_record() -> + {#ddb2_auto_scaling_settings_description{}, + [{<<"AutoScalingDisabled">>, #ddb2_auto_scaling_settings_description.auto_scaling_disabled, fun id/2}, + {<<"AutoScalingRoleArn">>, #ddb2_auto_scaling_settings_description.auto_scaling_role_arn, fun id/2}, + {<<"MaximumUnits">>, #ddb2_auto_scaling_settings_description.maximum_units, fun id/2}, + {<<"MinimumUnits">>, #ddb2_auto_scaling_settings_description.minimum_units, fun id/2}, + {<<"ScalingPolicies">>, #ddb2_auto_scaling_settings_description.scaling_policies, + fun(V, Opts) -> [undynamize_record(auto_scaling_policy_description_record(), I, Opts) || I <- V] end}]}. + +-spec replica_global_secondary_index_settings_description_record() -> record_desc(). +replica_global_secondary_index_settings_description_record() -> + {#ddb2_replica_global_secondary_index_settings_description{}, + [{<<"IndexName">>, #ddb2_replica_global_secondary_index_settings_description.index_name, fun id/2}, + {<<"IndexStatus">>, #ddb2_replica_global_secondary_index_settings_description.index_status, fun undynamize_index_status/2}, + {<<"ProvisionedReadCapacityAutoScalingSettings">>, #ddb2_replica_global_secondary_index_settings_description.provisioned_read_capacity_auto_scaling_settings, + fun(V, Opts) -> undynamize_record(auto_scaling_settings_description_record(), V, Opts) end}, + {<<"ProvisionedReadCapacityUnits">>, #ddb2_replica_global_secondary_index_settings_description.provisioned_read_capacity_units, fun id/2}, + {<<"ProvisionedWriteCapacityAutoScalingSettings">>, #ddb2_replica_global_secondary_index_settings_description.provisioned_write_capacity_auto_scaling_settings, + fun(V, Opts) -> undynamize_record(auto_scaling_settings_description_record(), V, Opts) end}, + {<<"ProvisionedWriteCapacityUnits">>, #ddb2_replica_global_secondary_index_settings_description.provisioned_write_capacity_units, fun id/2}]}. + +-spec replica_global_secondary_index_auto_scaling_description_record() -> record_desc(). +replica_global_secondary_index_auto_scaling_description_record() -> + {#ddb2_replica_global_secondary_index_auto_scaling_description{}, + [{<<"IndexName">>, #ddb2_replica_global_secondary_index_auto_scaling_description.index_name, fun id/2}, + {<<"IndexStatus">>, #ddb2_replica_global_secondary_index_auto_scaling_description.index_status, fun undynamize_index_status/2}, + {<<"ProvisionedReadCapacityAutoScalingSettings">>, #ddb2_replica_global_secondary_index_auto_scaling_description.provisioned_read_capacity_auto_scaling_settings, + fun(V, Opts) -> undynamize_record(auto_scaling_settings_description_record(), V, Opts) end}, + {<<"ProvisionedWriteCapacityAutoScalingSettings">>, #ddb2_replica_global_secondary_index_auto_scaling_description.provisioned_write_capacity_auto_scaling_settings, + fun(V, Opts) -> undynamize_record(auto_scaling_settings_description_record(), V, Opts) end}]}. + +-spec replica_settings_description_record() -> record_desc(). +replica_settings_description_record() -> + {#ddb2_replica_settings_description{}, + [{<<"RegionName">>, #ddb2_replica_settings_description.region_name, fun id/2}, + {<<"ReplicaBillingModeSummary">>, #ddb2_replica_settings_description.replica_billing_mode_summary, fun undynamize_billing_mode_summary/2}, + {<<"ReplicaGlobalSecondaryIndexSettings">>, #ddb2_replica_settings_description.replica_global_secondary_index_settings, + fun(V, Opts) -> [undynamize_record(replica_global_secondary_index_settings_description_record(), I, Opts) || I <- V] end}, + {<<"ReplicaProvisionedReadCapacityAutoScalingSettings">>, #ddb2_replica_settings_description.replica_provisioned_read_capacity_auto_scaling_settings, + fun(V, Opts) -> undynamize_record(auto_scaling_settings_description_record(), V, Opts) end}, + {<<"ReplicaProvisionedReadCapacityUnits">>, #ddb2_replica_settings_description.replica_provisioned_read_capacity_units, fun id/2}, + {<<"ReplicaProvisionedWriteCapacityAutoScalingSettings">>, #ddb2_replica_settings_description.replica_provisioned_write_capacity_auto_scaling_settings, + fun(V, Opts) -> undynamize_record(auto_scaling_settings_description_record(), V, Opts) end}, + {<<"ReplicaProvisionedWriteCapacityUnits">>, #ddb2_replica_settings_description.replica_provisioned_write_capacity_units, fun id/2}, + {<<"ReplicaStatus">>, #ddb2_replica_settings_description.replica_status, fun undynamize_replica_status/2}]}. + +-spec provisioned_throughput_override_record() -> record_desc(). +provisioned_throughput_override_record() -> + {#ddb2_provisioned_throughput_override{}, + [{<<"ReadCapacityUnits">>, #ddb2_provisioned_throughput_override.read_capacity_units, fun id/2}]}. + +undynamize_provisioned_throughput_override(V, Opts) -> + undynamize_record(provisioned_throughput_override_record(), V, Opts). + +-spec replica_global_secondary_index_description_record() -> record_desc(). +replica_global_secondary_index_description_record() -> + {#ddb2_replica_global_secondary_index_description{}, + [{<<"IndexName">>, #ddb2_replica_global_secondary_index_description.index_name, fun id/2}, + {<<"ProvisionedThroughputOverride">>, #ddb2_replica_global_secondary_index_description.provisioned_throughput_override, + fun undynamize_provisioned_throughput_override/2}]}. + +-spec replica_description_record() -> record_desc(). +replica_description_record() -> + {#ddb2_replica_description{}, + [{<<"GlobalSecondaryIndexes">>, #ddb2_replica_description.global_secondary_indexes, + fun(V, Opts) -> [undynamize_record(replica_global_secondary_index_description_record(), I, Opts) || I <- V] end}, + {<<"KMSMasterKeyId">>, #ddb2_replica_description.kms_master_key_id, fun id/2}, + {<<"ProvisionedThroughputOverride">>, #ddb2_replica_description.provisioned_throughput_override, fun undynamize_provisioned_throughput_override/2}, + {<<"RegionName">>, #ddb2_replica_description.region_name, fun id/2}, + {<<"ReplicaStatus">>, #ddb2_replica_description.replica_status, fun undynamize_replica_status/2}, + {<<"ReplicaStatusDescription">>, #ddb2_replica_description.replica_status_description, fun id/2}, + {<<"ReplicaStatusPercentProgress">>, #ddb2_replica_description.replica_status_percent_progress, fun id/2}]}. + +-spec replica_auto_scaling_description_record() -> record_desc(). +replica_auto_scaling_description_record() -> + {#ddb2_replica_auto_scaling_description{}, + [{<<"GlobalSecondaryIndexes">>, #ddb2_replica_auto_scaling_description.global_secondary_indexes, + fun(V, Opts) -> [undynamize_record(replica_global_secondary_index_auto_scaling_description_record(), I, Opts) || I <- V] end}, + {<<"RegionName">>, #ddb2_replica_auto_scaling_description.region_name, fun id/2}, + {<<"ReplicaProvisionedReadCapacityAutoScalingSettings">>, #ddb2_replica_auto_scaling_description.replica_provisioned_read_capacity_auto_scaling_settings, + fun(V, Opts) -> undynamize_record(auto_scaling_settings_description_record(), V, Opts) end}, + {<<"ReplicaProvisionedWriteCapacityAutoScalingSettings">>, #ddb2_replica_auto_scaling_description.replica_provisioned_write_capacity_auto_scaling_settings, + fun(V, Opts) -> undynamize_record(auto_scaling_settings_description_record(), V, Opts) end}, + {<<"ReplicaStatus">>, #ddb2_replica_auto_scaling_description.replica_status, fun undynamize_replica_status/2}]}. + +-spec table_auto_scaling_description_record() -> record_desc(). +table_auto_scaling_description_record() -> + {#ddb2_table_auto_scaling_description{}, + [{<<"Replicas">>, #ddb2_table_auto_scaling_description.replicas, + fun(V, Opts) -> [undynamize_record(replica_auto_scaling_description_record(), I, Opts) || I <- V] end}, + {<<"TableName">>, #ddb2_table_auto_scaling_description.table_name, fun id/2}, + {<<"TableStatus">>, #ddb2_table_auto_scaling_description.table_status, fun undynamize_table_status/2}]}. + -spec table_description_record() -> record_desc(). table_description_record() -> {#ddb2_table_description{}, @@ -1247,6 +1430,7 @@ table_description_record() -> {<<"CreationDateTime">>, #ddb2_table_description.creation_date_time, fun id/2}, {<<"GlobalSecondaryIndexes">>, #ddb2_table_description.global_secondary_indexes, fun(V, Opts) -> [undynamize_record(global_secondary_index_description_record(), I, Opts) || I <- V] end}, + {<<"GlobalTableVersion">>, #ddb2_table_description.global_table_version, fun id/2}, {<<"ItemCount">>, #ddb2_table_description.item_count, fun id/2}, {<<"KeySchema">>, #ddb2_table_description.key_schema, fun undynamize_key_schema/2}, {<<"LatestStreamArn">>, #ddb2_table_description.latest_stream_arn, fun id/2}, @@ -1255,6 +1439,8 @@ table_description_record() -> fun(V, Opts) -> [undynamize_record(local_secondary_index_description_record(), I, Opts) || I <- V] end}, {<<"ProvisionedThroughput">>, #ddb2_table_description.provisioned_throughput, fun(V, Opts) -> undynamize_record(provisioned_throughput_description_record(), V, Opts) end}, + {<<"Replicas">>, #ddb2_table_description.replicas, + fun(V, Opts) -> [undynamize_record(replica_description_record(), I, Opts) || I <- V] end}, {<<"RestoreSummary">>, #ddb2_table_description.restore_summary, fun(V, Opts) -> undynamize_record(restore_summary_record(), V, Opts) end}, {<<"SSEDescription">>, #ddb2_table_description.sse_description, fun undynamize_sse_description/2}, @@ -2195,6 +2381,54 @@ describe_global_table(GlobalTableName, Opts, Config) -> out(Return, fun(Json, UOpts) -> undynamize_record(describe_global_table_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_table.table). +%%%------------------------------------------------------------------------------ +%%% DescribeGlobalTableSettings +%%%------------------------------------------------------------------------------ + +-type describe_global_table_settings_return() :: ddb_return(#ddb2_describe_global_table_settings{}, #ddb2_replica_settings_description{}). + +-spec describe_global_table_settings_record() -> record_desc(). +describe_global_table_settings_record() -> + {#ddb2_describe_global_table_settings{}, + [{<<"GlobalTableName">>, #ddb2_describe_global_table_settings.global_table_name, fun id/2}, + {<<"ReplicaSettings">>, #ddb2_describe_global_table_settings.replica_settings, + fun(V, Opts) -> [undynamize_record(replica_settings_description_record(), I, Opts) || I <- V] end}]}. + +-spec describe_global_table_settings(table_name()) -> describe_global_table_settings_return(). +describe_global_table_settings(GlobalTableName) -> + describe_global_table_settings(GlobalTableName, []). + +-spec describe_global_table_settings(table_name(), ddb_opts()) -> describe_global_table_settings_return(). +describe_global_table_settings(GlobalTableName, Opts) -> + describe_global_table_settings(GlobalTableName, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% DynamoDB API: +%% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DescribeGlobalTableSettings.html] +%% +%% Note: This method only applies to Version 2017.11.29 of global tables. +%% +%% ===Example=== +%% +%% Describes Region-specific settings for "Thread" global table. +%% +%% ` +%% {ok, GlobalTableSettings} = +%% erlcloud_ddb2:describe_global_table_settings(<<"Thread">>), +%% ' +%% @end +%%------------------------------------------------------------------------------ +-spec describe_global_table_settings(table_name(), ddb_opts(), aws_config()) -> describe_global_table_settings_return(). +describe_global_table_settings(GlobalTableName, Opts, Config) -> + {[], DdbOpts} = opts([], Opts), + Return = erlcloud_ddb_impl:request( + Config, + "DynamoDB_20120810.DescribeGlobalTableSettings", + [{<<"GlobalTableName">>, GlobalTableName}]), + out(Return, fun(Json, UOpts) -> undynamize_record(describe_global_table_settings_record(), Json, UOpts) end, + DdbOpts, #ddb2_describe_global_table_settings.replica_settings). + %%%------------------------------------------------------------------------------ %%% DescribeLimits %%%------------------------------------------------------------------------------ @@ -2293,6 +2527,53 @@ describe_table(Table, Opts, Config) -> out(Return, fun(Json, UOpts) -> undynamize_record(describe_table_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_table.table). +%%%------------------------------------------------------------------------------ +%%% DescribeTableReplicaAutoScaling +%%%------------------------------------------------------------------------------ + +-type describe_table_replica_auto_scaling_return() :: ddb_return(#ddb2_describe_table_replica_auto_scaling{}, #ddb2_table_auto_scaling_description{}). + +-spec describe_table_replica_auto_scaling_record() -> record_desc(). +describe_table_replica_auto_scaling_record() -> + {#ddb2_describe_table_replica_auto_scaling{}, + [{<<"TableAutoScalingDescription">>, #ddb2_describe_table_replica_auto_scaling.table_auto_scaling_description, + fun(V, Opts) -> undynamize_record(table_auto_scaling_description_record(), V, Opts) end} + ]}. + +-spec describe_table_replica_auto_scaling(table_name()) -> describe_table_replica_auto_scaling_return(). +describe_table_replica_auto_scaling(Table) -> + describe_table_replica_auto_scaling(Table, [], default_config()). + +-spec describe_table_replica_auto_scaling(table_name(), ddb_opts()) -> describe_table_replica_auto_scaling_return(). +describe_table_replica_auto_scaling(Table, Opts) -> + describe_table_replica_auto_scaling(Table, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% DynamoDB API: +%% [http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DescribeTableReplicaAutoScaling.html] +%% Note: This method only applies to Version 2019.11.21 of global tables. +%% +%% ===Example=== +%% +%% Describe "Thread" table. +%% +%% ` +%% {ok, Description} = +%% erlcloud_ddb2:describe_table_replica_auto_scaling(<<"Thread">>), +%% ' +%% @end +%%------------------------------------------------------------------------------ +-spec describe_table_replica_auto_scaling(table_name(), ddb_opts(), aws_config()) -> describe_table_replica_auto_scaling_return(). +describe_table_replica_auto_scaling(Table, Opts, Config) -> + {[], DdbOpts} = opts([], Opts), + Return = erlcloud_ddb_impl:request( + Config, + "DynamoDB_20120810.DescribeTableReplicaAutoScaling", + [{<<"TableName">>, Table}]), + out(Return, fun(Json, UOpts) -> undynamize_record(describe_table_replica_auto_scaling_record(), Json, UOpts) end, + DdbOpts, #ddb2_describe_table_replica_auto_scaling.table_auto_scaling_description). + %%%------------------------------------------------------------------------------ %%% DescribeTimeToLive %%%------------------------------------------------------------------------------ @@ -3668,6 +3949,154 @@ update_global_table(GlobalTableName, ReplicaUpdates, Opts, Config) -> out(Return, fun(Json, UOpts) -> undynamize_record(update_global_table_record(), Json, UOpts) end, DdbOpts, #ddb2_update_global_table.global_table_description). +%%%------------------------------------------------------------------------------ +%%% UpdateGlobalTableSettings +%%%------------------------------------------------------------------------------ + +-type update_global_table_settings_return() :: ddb_return(#ddb2_update_global_table_settings{}, [#ddb2_replica_settings_description{}]). + +-type global_table_global_secondary_index_settings_update_opt() :: {index_name, binary()}| + {provisioned_write_capacity_auto_scaling_settings_update, auto_scaling_settings_update_opts()}| + {provisioned_read_capacity_units, read_units()}| + {provisioned_write_capacity_units, write_units()}. + +-type global_table_global_secondary_index_settings_update_opts() :: [global_table_global_secondary_index_settings_update_opt()]. + +-type replica_global_secondary_index_settings_update_opt() :: {index_name, binary()}| + {provisioned_read_capacity_auto_scaling_settings_update, auto_scaling_settings_update_opts()}| + {provisioned_read_capacity_units, read_units()}. + +-type replica_global_secondary_index_settings_update_opts() :: [replica_global_secondary_index_settings_update_opt()]. + +-type replica_settings_update_opt() :: {region_name, binary()}| + {replica_global_secondary_index_settings_update, [global_table_global_secondary_index_settings_update_opts()]}| + {replica_provisioned_read_capacity_auto_scaling_settings_update, auto_scaling_settings_update_opts()}| + {replica_provisioned_read_capacity_units, read_units()}. + +-type replica_settings_update_opts() :: [replica_settings_update_opts()]. + +-spec dynamize_global_table_global_secondary_index_settings_update_opt(global_table_global_secondary_index_settings_update_opt()) -> json_pair(). +dynamize_global_table_global_secondary_index_settings_update_opt({index_name, IndexName}) -> + {<<"IndexName">>, IndexName}; +dynamize_global_table_global_secondary_index_settings_update_opt({provisioned_write_capacity_auto_scaling_settings_update, Update}) -> + {<<"ProvisionedWriteCapacityAutoScalingSettingsUpdate">>, dynamize_auto_scaling_settings_update_opts(Update)}; +dynamize_global_table_global_secondary_index_settings_update_opt({provisioned_read_capacity_units, ProvisionedReadCapacityUnits}) -> + {<<"ProvisionedReadCapacityUnits">>, ProvisionedReadCapacityUnits}; +dynamize_global_table_global_secondary_index_settings_update_opt({provisioned_write_capacity_units, ProvisionedWriteCapacityUnits}) -> + {<<"ProvisionedWriteCapacityUnits">>, ProvisionedWriteCapacityUnits}. + +-spec dynamize_global_table_global_secondary_index_settings_update_opts(global_table_global_secondary_index_settings_update_opts()) -> jsx:json_term(). +dynamize_global_table_global_secondary_index_settings_update_opts(Opts) -> + dynamize_maybe_list(fun dynamize_global_table_global_secondary_index_settings_update_opt/1, Opts). + +-spec dynamize_global_table_global_secondary_index_settings_update([global_table_global_secondary_index_settings_update_opts()]) -> jsx:json_term(). +dynamize_global_table_global_secondary_index_settings_update(Updates) -> + [dynamize_global_table_global_secondary_index_settings_update_opts(Update) || Update <- Updates]. + +-spec dynamize_replica_global_secondary_index_settings_update_opt(replica_global_secondary_index_settings_update_opt()) -> json_pair(). +dynamize_replica_global_secondary_index_settings_update_opt({index_name, IndexName}) -> + {<<"IndexName">>, IndexName}; +dynamize_replica_global_secondary_index_settings_update_opt({provisioned_read_capacity_auto_scaling_settings_update, Update}) -> + {<<"ProvisionedReadCapacityAutoScalingSettingsUpdate">>, dynamize_auto_scaling_settings_update_opts(Update)}; +dynamize_replica_global_secondary_index_settings_update_opt({provisioned_read_capacity_units, Units}) -> + {<<"ProvisionedReadCapacityUnits">>, Units}. + +-spec dynamize_replica_global_secondary_index_settings_update_opts(replica_global_secondary_index_settings_update_opts()) -> jsx:json_term(). +dynamize_replica_global_secondary_index_settings_update_opts(Opts) -> + dynamize_maybe_list(fun dynamize_replica_global_secondary_index_settings_update_opt/1, Opts). + +-spec dynamize_replica_global_secondary_index_settings_update([replica_global_secondary_index_settings_update_opts()]) -> jsx:json_term(). +dynamize_replica_global_secondary_index_settings_update(Updates) -> + [dynamize_replica_global_secondary_index_settings_update_opts(Update) || Update <- Updates]. + +-spec dynamize_replica_settings_update_opt(replica_settings_update_opt()) -> json_pair(). +dynamize_replica_settings_update_opt({region_name, RegionName}) -> + {<<"RegionName">>, RegionName}; +dynamize_replica_settings_update_opt({replica_global_secondary_index_settings_update, Update}) -> + {<<"ReplicaGlobalSecondaryIndexSettingsUpdate">>, dynamize_replica_global_secondary_index_settings_update(Update)}; +dynamize_replica_settings_update_opt({replica_provisioned_read_capacity_auto_scaling_settings_update, Update}) -> + {<<"ReplicaProvisionedReadCapacityAutoScalingSettingsUpdate">>, dynamize_auto_scaling_settings_update_opts(Update)}; +dynamize_replica_settings_update_opt({replica_provisioned_read_capacity_units, Units}) -> + {<<"ReplicaProvisionedReadCapacityUnits">>, Units}. + +-spec dynamize_replica_settings_update_opts(replica_settings_update_opts()) -> jsx:json_term(). +dynamize_replica_settings_update_opts(Opts) -> + dynamize_maybe_list(fun dynamize_replica_settings_update_opt/1, Opts). + +-spec dynamize_replica_settings_updates([replica_settings_update_opts()]) -> jsx:json_term(). +dynamize_replica_settings_updates(Updates) -> + [dynamize_replica_settings_update_opts(Update) || Update <- Updates]. + +-type update_global_table_settings_opt() :: {global_table_billing_mode, billing_mode()}| + {global_table_global_secondary_index_settings_update, [global_table_global_secondary_index_settings_update_opts()]}| + {global_table_provisioned_write_capacity_auto_scaling_settings_update, auto_scaling_settings_update_opts()}| + {global_table_provisioned_write_capacity_units, write_units()}| + {replica_settings_update, [replica_settings_update_opts()]}| + out_opt(). + +-type update_global_table_settings_opts() :: [update_global_table_settings_opt()]. + +-spec update_global_table_settings_opts() -> opt_table(). +update_global_table_settings_opts() -> + [{global_table_billing_mode, <<"GlobalTableBillingMode">>, fun dynamize_billing_mode/1}, + {global_table_global_secondary_index_settings_update, <<"GlobalTableGlobalSecondaryIndexSettingsUpdate">>, fun dynamize_global_table_global_secondary_index_settings_update/1}, + {global_table_provisioned_write_capacity_auto_scaling_settings_update, <<"GlobalTableProvisionedWriteCapacityAutoScalingSettingsUpdate">>, fun dynamize_auto_scaling_settings_update_opts/1}, + {global_table_provisioned_write_capacity_units, <<"GlobalTableProvisionedWriteCapacityUnits">>, fun id/1}, + {replica_settings_update, <<"ReplicaSettingsUpdate">>, fun dynamize_replica_settings_updates/1}]. + +-spec update_global_table_settings_record() -> record_desc(). +update_global_table_settings_record() -> + {#ddb2_update_global_table_settings{}, + [{<<"GlobalTableName">>, #ddb2_update_global_table_settings.global_table_name, fun id/2}, + {<<"ReplicaSettings">>, #ddb2_update_global_table_settings.replica_settings, + fun(V, Opts) -> [undynamize_record(replica_settings_description_record(), I, Opts) || I <- V] end}]}. + + +-spec update_global_table_settings(table_name(), update_global_table_settings_opts()) -> update_global_table_settings_return(). +update_global_table_settings(GlobalTableName, Opts) -> + update_global_table_settings(GlobalTableName, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% DynamoDB API: +%% [https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateGlobalTableSettings.html] +%% +%% ===Example=== +%% +%% Update global table settings for a table called "Thread" in us-west-2 and eu-west-2. +%% +%% ``` +%% ReadUnits = 10, +%% WriteUnits = 10, +%% erlcloud_ddb2:update_global_table_settings( +%% <<"Thread">>, +%% [{global_table_billing_mode, provisioned}, +%% {global_table_global_secondary_index_settings_update, [[{index_name, <<"id-index">>}, +%% {provisioned_write_capacity_units, WriteUnits}]]}, +%% {global_table_provisioned_write_capacity_units, WriteUnits}, +%% {replica_settings_update, [[{region_name, <<"us-west-2">>}, +%% {replica_global_secondary_index_settings_update, [[{index_name, <<"id-index">>}, +%% {provisioned_read_capacity_units, ReadUnits}]]}, +%% {replica_provisioned_read_capacity_units, ReadUnits}], +%% [{region_name, <<"eu-west-2">>}, +%% {replica_global_secondary_index_settings_update, [[{index_name, <<"id-index">>}, +%% {provisioned_read_capacity_units, ReadUnits}]]}, +%% {replica_provisioned_read_capacity_units, ReadUnits}]]}]) +%% ''' +%% @end +%%------------------------------------------------------------------------------ + +-spec update_global_table_settings(table_name(), update_global_table_settings_opts(), aws_config()) -> update_global_table_settings_return(). +update_global_table_settings(GlobalTableName, Opts, Config) -> + {AwsOpts, DdbOpts} = opts(update_global_table_settings_opts(), Opts), + Return = erlcloud_ddb_impl:request( + Config, + "DynamoDB_20120810.UpdateGlobalTableSettings", + [{<<"GlobalTableName">>, GlobalTableName} | AwsOpts]), + out(Return, fun(Json, UOpts) -> undynamize_record(update_global_table_settings_record(), Json, UOpts) end, + DdbOpts, #ddb2_update_global_table_settings.replica_settings). + + %%%------------------------------------------------------------------------------ %%% UpdateTable %%%------------------------------------------------------------------------------ @@ -3679,6 +4108,32 @@ update_global_table(GlobalTableName, ReplicaUpdates, Opts, Config) -> global_secondary_index_def(). -type global_secondary_index_updates() :: maybe_list(global_secondary_index_update()). +-type provisioned_throughput_override_opt() :: {read_capacity_units, non_neg_integer()}. +-type provisioned_throughput_override_opts() :: [provisioned_throughput_override_opt()]. + +-type create_replication_group_member_action_opt() :: {global_secondary_indexes, binary()}| + {kms_master_key_id, binary()}| + {provisioned_throughput_override, provisioned_throughput_override_opts()}| + {region_name, binary()}. + +-type create_replication_group_member_action_opts() :: [create_replication_group_member_action_opt()]. + +-type delete_replication_group_member_action_opt() :: {region_name, binary()}. + +-type delete_replication_group_member_action_opts() :: [create_replication_group_member_action_opt()]. + +-type update_replication_group_member_action_opt() :: {global_secondary_indexes, binary()}| + {kms_master_key_id, binary()}| + {provisioned_throughput_override, provisioned_throughput_override_opts()}| + {region_name, binary()}. + +-type update_replication_group_member_action_opts() :: [update_replication_group_member_action_opt()]. + +-type replication_group_update_opt() :: {create, create_replication_group_member_action_opts()}| + {delete, delete_replication_group_member_action_opts()}| + {update, update_replication_group_member_action_opts()}. +-type replication_group_update_opts() :: [replication_group_update_opt()]. + -spec dynamize_global_secondary_index_update(global_secondary_index_update()) -> jsx:json_term(). dynamize_global_secondary_index_update({IndexName, ReadUnits, WriteUnits}) when is_integer(ReadUnits), is_integer(WriteUnits) -> @@ -3697,6 +4152,62 @@ dynamize_global_secondary_index_update(Index) -> dynamize_global_secondary_index_updates(Updates) -> dynamize_maybe_list(fun dynamize_global_secondary_index_update/1, Updates). +-spec dynamize_provisioned_throughput_override_opt(provisioned_throughput_override_opt()) -> json_pair(). +dynamize_provisioned_throughput_override_opt({read_capacity_units, ReadCapacityUnits}) -> + {<<"ReadCapacityUnits">>, ReadCapacityUnits}. + +-spec dynamize_provisioned_throughput_override_opts(provisioned_throughput_override_opts()) -> jsx:json_term(). +dynamize_provisioned_throughput_override_opts(Opts) -> + dynamize_maybe_list(fun dynamize_provisioned_throughput_override_opt/1, Opts). + +-spec dynamize_create_replication_group_member_action_opt(create_replication_group_member_action_opt()) -> json_pair(). +dynamize_create_replication_group_member_action_opt({global_secondary_indexes, GlobalSecondaryIndexes}) -> + {<<"GlobalSecondaryIndexes">>, GlobalSecondaryIndexes}; +dynamize_create_replication_group_member_action_opt({kms_master_key_id, KMSMasterKeyId}) -> + {<<"KMSMasterKeyId">>, KMSMasterKeyId}; +dynamize_create_replication_group_member_action_opt({provisioned_throughput_override, ProvisionedThroughputOverride}) -> + {<<"ProvisionedThroughputOverride">>, dynamize_provisioned_throughput_override_opts(ProvisionedThroughputOverride)}; +dynamize_create_replication_group_member_action_opt({region_name, RegionName}) -> + {<<"RegionName">>, RegionName}. + +-spec dynamize_create_replication_group_member_action_opts(create_replication_group_member_action_opts()) -> jsx:json_term(). +dynamize_create_replication_group_member_action_opts(Opts) -> + dynamize_maybe_list(fun dynamize_create_replication_group_member_action_opt/1, Opts). + +-spec dynamize_delete_replication_group_member_action_opt(delete_replication_group_member_action_opt()) -> json_pair(). +dynamize_delete_replication_group_member_action_opt({region_name, RegionName}) -> + {<<"RegionName">>, RegionName}. + +-spec dynamize_delete_replication_group_member_action_opts(delete_replication_group_member_action_opts()) -> jsx:json_term(). +dynamize_delete_replication_group_member_action_opts(Opts) -> + dynamize_maybe_list(fun dynamize_delete_replication_group_member_action_opt/1, Opts). + +-spec dynamize_update_replication_group_member_action_opt(update_replication_group_member_action_opt()) -> json_pair(). +dynamize_update_replication_group_member_action_opt({global_secondary_indexes, GlobalSecondaryIndexes}) -> + {<<"GlobalSecondaryIndexes">>, GlobalSecondaryIndexes}; +dynamize_update_replication_group_member_action_opt({kms_master_key_id, KMSMasterKeyId}) -> + {<<"KMSMasterKeyId">>, KMSMasterKeyId}; +dynamize_update_replication_group_member_action_opt({provisioned_throughput_override, ProvisionedThroughputOverride}) -> + {<<"ProvisionedThroughputOverride">>, ProvisionedThroughputOverride}; +dynamize_update_replication_group_member_action_opt({region_name, RegionName}) -> + {<<"RegionName">>, RegionName}. + +-spec dynamize_update_replication_group_member_action_opts(update_replication_group_member_action_opts()) -> jsx:json_term(). +dynamize_update_replication_group_member_action_opts(Opts) -> + dynamize_maybe_list(fun dynamize_update_replication_group_member_action_opt/1, Opts). + +-spec dynamize_replication_group_update(replication_group_update_opt()) -> json_pair(). +dynamize_replication_group_update({create, Create}) -> + {<<"Create">>, dynamize_create_replication_group_member_action_opts(Create)}; +dynamize_replication_group_update({delete, Delete}) -> + {<<"Delete">>, dynamize_delete_replication_group_member_action_opts(Delete)}; +dynamize_replication_group_update({update, Update}) -> + {<<"Update">>, dynamize_update_replication_group_member_action_opts(Update)}. + +-spec dynamize_replication_group_updates(replication_group_update_opts()) -> jsx:json_term(). +dynamize_replication_group_updates(Updates) -> + [dynamize_maybe_list(fun dynamize_replication_group_update/1, Update) || Update <- Updates]. + -type update_table_opt() :: {billing_mode, billing_mode()} | {provisioned_throughput, {read_units(), write_units()}} | {attribute_definitions, attr_defs()} | @@ -3712,7 +4223,8 @@ update_table_opts() -> {attribute_definitions, <<"AttributeDefinitions">>, fun dynamize_attr_defs/1}, {global_secondary_index_updates, <<"GlobalSecondaryIndexUpdates">>, fun dynamize_global_secondary_index_updates/1}, - {stream_specification, <<"StreamSpecification">>, fun dynamize_stream_specification/1}]. + {stream_specification, <<"StreamSpecification">>, fun dynamize_stream_specification/1}, + {replica_updates, <<"ReplicaUpdates">>, fun dynamize_replication_group_updates/1}]. -spec update_table_record() -> record_desc(). update_table_record() -> @@ -3767,6 +4279,141 @@ update_table(Table, ReadUnits, WriteUnits, Opts, Config) -> update_table(Table, [{provisioned_throughput, {ReadUnits, WriteUnits}} | Opts], Config). +%%%------------------------------------------------------------------------------ +%%% UpdateTableReplicaAutoScaling +%%%------------------------------------------------------------------------------ + +-type update_table_replica_auto_scaling_return() :: ddb_return(#ddb2_update_table_replica_auto_scaling{}, #ddb2_table_auto_scaling_description{}). + +-type global_secondary_index_auto_scaling_update_opt() :: {index_name, binary()}| + {provisioned_read_capacity_auto_scaling_update, auto_scaling_settings_update_opts()}. + +-type global_secondary_index_auto_scaling_update_opts() :: [global_secondary_index_auto_scaling_update_opt()]. + +-type replica_global_secondary_index_auto_scaling_opt() :: {index_name, binary()}| + {provisioned_read_capacity_auto_scaling_update, auto_scaling_settings_update_opts()}. + +-type replica_global_secondary_index_auto_scaling_opts() :: [replica_global_secondary_index_auto_scaling_opt()]. + +-type replica_auto_scaling_update_opt() :: {region_name, binary()}| + {replica_global_secondary_index_updates, [replica_global_secondary_index_auto_scaling_opts()]}| + {replica_provisioned_read_capacity_auto_scaling_update, auto_scaling_settings_update_opts()}. +-type replica_auto_scaling_update_opts() :: [replica_auto_scaling_update_opt()]. + +-spec dynamize_global_secondary_index_auto_scaling_update_opt(global_secondary_index_auto_scaling_update_opt()) -> json_pair(). +dynamize_global_secondary_index_auto_scaling_update_opt({index_name, IndexName}) -> + {<<"IndexName">>, IndexName}; +dynamize_global_secondary_index_auto_scaling_update_opt({provisioned_write_capacity_auto_scaling_update, ProvisionedWriteCapacityAutoScalingUpdate}) -> + {<<"ProvisionedWriteCapacityAutoScalingUpdate">>, dynamize_maybe_list(fun dynamize_auto_scaling_settings_update_opt/1, + ProvisionedWriteCapacityAutoScalingUpdate)}. + +-spec dynamize_global_secondary_index_auto_scaling_update_opts([replica_global_secondary_index_auto_scaling_opts()]) -> jsx:json_term(). +dynamize_global_secondary_index_auto_scaling_update_opts(GlobalSecondaryIndexUpdates) -> + [dynamize_maybe_list(fun dynamize_global_secondary_index_auto_scaling_update_opt/1, + Update) || Update <- GlobalSecondaryIndexUpdates]. + +-spec dynamize_replica_global_secondary_index_auto_scaling_update_opt(replica_global_secondary_index_auto_scaling_opt()) -> json_pair(). +dynamize_replica_global_secondary_index_auto_scaling_update_opt({index_name, IndexName}) -> + {<<"IndexName">>, IndexName}; +dynamize_replica_global_secondary_index_auto_scaling_update_opt({provisioned_read_capacity_auto_scaling_update, ProvisionedReadCapacityAutoScalingUpdate}) -> + {<<"ProvisionedReadCapacityAutoScalingUpdate">>, dynamize_auto_scaling_settings_update_opts(ProvisionedReadCapacityAutoScalingUpdate)}. + +-spec dynamize_replica_global_secondary_index_auto_scaling_update_opts(replica_global_secondary_index_auto_scaling_opts()) -> jsx:json_term(). +dynamize_replica_global_secondary_index_auto_scaling_update_opts(Opts) -> + dynamize_maybe_list(fun dynamize_replica_global_secondary_index_auto_scaling_update_opt/1, + Opts). + +-spec dynamize_replica_auto_scaling_update(replica_auto_scaling_update_opt()) -> json_pair(). +dynamize_replica_auto_scaling_update({region_name, RegionName}) -> + {<<"RegionName">>, RegionName}; +dynamize_replica_auto_scaling_update({replica_global_secondary_index_updates, ReplicaGlobalSecondaryIndexUpdates}) -> + {<<"ReplicaGlobalSecondaryIndexUpdates">>, [dynamize_replica_global_secondary_index_auto_scaling_update_opts(Update) || Update <- ReplicaGlobalSecondaryIndexUpdates]}; +dynamize_replica_auto_scaling_update({replica_provisioned_read_capacity_auto_scaling_update, ReplicaProvisionedReadCapacityAutoScalingUpdate}) -> + {<<"ReplicaProvisionedReadCapacityAutoScalingUpdate">>, dynamize_auto_scaling_settings_update_opts(ReplicaProvisionedReadCapacityAutoScalingUpdate)}. + +-spec dynamize_replica_auto_scaling_updates([replica_auto_scaling_update_opts()]) -> jsx:json_term(). +dynamize_replica_auto_scaling_updates(ReplicaUpdates) -> + [dynamize_maybe_list(fun dynamize_replica_auto_scaling_update/1, + Update) || Update <- ReplicaUpdates]. + +-spec maybe_update_config_timeout(aws_config(), MinDesiredTimeout :: pos_integer()) -> aws_config(). +maybe_update_config_timeout(Config, MinDesiredTimeout) -> + UpdatedTimeout = case erlcloud_aws:get_timeout(Config) of + undefined -> MinDesiredTimeout; + ProvidedTimeout -> ProvidedTimeout + end, + Config#aws_config{timeout = UpdatedTimeout}. + +-type update_table_replica_auto_scaling_opt() :: {global_secondary_index_updates, [global_secondary_index_auto_scaling_update_opts()]}| + {provisioned_write_capacity_auto_scaling_update, auto_scaling_settings_update_opts()}| + {replica_updates, [replica_auto_scaling_update_opts()]}| + out_opt(). +-type update_table_replica_auto_scaling_opts() :: [update_table_replica_auto_scaling_opt()]. + +-spec update_table_replica_auto_scaling_opts() -> opt_table(). +update_table_replica_auto_scaling_opts() -> + [{global_secondary_index_updates, <<"GlobalSecondaryIndexUpdates">>, + fun dynamize_global_secondary_index_auto_scaling_update_opts/1}, + {provisioned_write_capacity_auto_scaling_update, <<"ProvisionedWriteCapacityAutoScalingUpdate">>, + fun dynamize_auto_scaling_settings_update_opts/1}, + {replica_updates, <<"ReplicaUpdates">>, + fun dynamize_replica_auto_scaling_updates/1}]. + +-spec update_table_replica_auto_scaling_record() -> record_desc(). +update_table_replica_auto_scaling_record() -> + {#ddb2_update_table_replica_auto_scaling{}, + [{<<"TableAutoScalingDescription">>, #ddb2_update_table_replica_auto_scaling.table_auto_scaling_description, + fun(V, Opts) -> undynamize_record(table_auto_scaling_description_record(), V, Opts) end}]}. + +-spec update_table_replica_auto_scaling(table_name(), update_table_replica_auto_scaling_opts()) -> update_table_replica_auto_scaling_return(). +update_table_replica_auto_scaling(Table, Opts) -> + update_table_replica_auto_scaling(Table, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% DynamoDB API: +%% [http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateTableReplicaAutoScaling.html] +%% +%% Note: This method only applies to Version 2019.11.21 of global tables. +%% Note: +%% +%% ===Example=== +%% +%% Update table "Thread" to scale between 10 and 20 provisioned read and write units uniformly across replica regions. +%% ``` +%% AutoScalingSettingsUpdate = [{maximum_units, 20}, +%% {minimum_units, 10}, +%% {scaling_policy_update, [{target_tracking_scaling_policy_configuration, [{target_value, 60.0}]}]}], +%% erlcloud_ddb2:update_table_replica_auto_scaling( +%% <<"Thread">>, +%% [{global_secondary_index_updates, [[{index_name, <<"id-index">>}, +%% {provisioned_write_capacity_auto_scaling_update, AutoScalingSettingsUpdate}]]}, +%% {provisioned_write_capacity_auto_scaling_update, AutoScalingSettingsUpdate}, +%% {replica_updates, [[{region_name, <<"us-west-2">>}, +%% {replica_global_secondary_index_updates, [[{index_name, <<"id-index">>}, +%% {provisioned_read_capacity_auto_scaling_update, AutoScalingSettingsUpdate}]]}, +%% {replica_provisioned_read_capacity_auto_scaling_update, AutoScalingSettingsUpdate}], +%% [{region_name, <<"eu-west-2">>}, +%% {replica_global_secondary_index_updates, [[{index_name, <<"id-index">>}, +%% {provisioned_read_capacity_auto_scaling_update, AutoScalingSettingsUpdate}]]}, +%% {replica_provisioned_read_capacity_auto_scaling_update, AutoScalingSettingsUpdate}]]}]). +%% ''' +%% @end +%%------------------------------------------------------------------------------ +-spec update_table_replica_auto_scaling(table_name(), update_table_replica_auto_scaling_opts(), aws_config()) -> update_table_replica_auto_scaling_return(). +update_table_replica_auto_scaling(Table, Opts, Config) when is_list(Opts) -> + % The default timeout for this endpoint is updated to 25 seconds below due to the endpoint + % regularly taking around 20 seconds during testing. + % Any non-default timeout already written to the config will not be overridden. + UpdatedConfig = maybe_update_config_timeout(Config, 25000), + {AwsOpts, DdbOpts} = opts(update_table_replica_auto_scaling_opts(), Opts), + Return = erlcloud_ddb_impl:request( + UpdatedConfig, + "DynamoDB_20120810.UpdateTableReplicaAutoScaling", + [{<<"TableName">>, Table} | AwsOpts]), + out(Return, fun(Json, UOpts) -> undynamize_record(update_table_replica_auto_scaling_record(), Json, UOpts) end, + DdbOpts, #ddb2_update_table_replica_auto_scaling.table_auto_scaling_description). + %%%------------------------------------------------------------------------------ %%% UpdateTimeToLive %%%------------------------------------------------------------------------------ diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index d96c4d18f..cf4ffcb80 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -53,8 +53,12 @@ operation_test_() -> fun describe_limits_output_tests/1, fun describe_global_table_input_tests/1, fun describe_global_table_output_tests/1, + fun describe_global_table_settings_input_tests/1, + fun describe_global_table_settings_output_tests/1, fun describe_table_input_tests/1, fun describe_table_output_tests/1, + fun describe_table_replica_auto_scaling_input_tests/1, + fun describe_table_replica_auto_scaling_output_tests/1, fun describe_time_to_live_input_tests/1, fun describe_time_to_live_output_tests/1, fun get_item_input_tests/1, @@ -92,8 +96,12 @@ operation_test_() -> fun update_item_output_tests/1, fun update_global_table_input_tests/1, fun update_global_table_output_tests/1, + fun update_global_table_settings_input_tests/1, + fun update_global_table_settings_output_tests/1, fun update_table_input_tests/1, fun update_table_output_tests/1, + fun update_table_replica_auto_scaling_input_tests/1, + fun update_table_replica_auto_scaling_output_tests/1, fun update_time_to_live_input_tests/1, fun update_time_to_live_output_tests/1 ]}. @@ -2641,6 +2649,254 @@ describe_global_table_output_tests(_) -> #ddb2_replica_description{region_name = <<"eu-west-1">>}]}}})], output_tests(?_f(erlcloud_ddb2:describe_global_table(<<"Thread">>)), Tests). +%% DescribeGlobalTableSettings tests based on the API request/response syntax: +%% http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DescribeGlobalTableSettings.html +describe_global_table_settings_input_tests(_) -> + Tests = + [?_ddb_test( + {"DescribeGlobalTableSettings example request", + ?_f(erlcloud_ddb2:describe_global_table_settings(<<"Thread">>)), " +{ + \"GlobalTableName\":\"Thread\" +}" + }) + ], + Response = " +{ + \"GlobalTableName\": \"Thread\", + \"ReplicaSettings\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaBillingModeSummary\": { + \"BillingMode\": \"PROVISIONED\", + \"LastUpdateToPayPerRequestDateTime\": 1578092745.455 + }, + \"ReplicaGlobalSecondaryIndexSettings\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedReadCapacityUnits\": 10, + \"ProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": true, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedWriteCapacityUnits\": 10 + } + ], + \"ReplicaProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": true, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"policy\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedReadCapacityUnits\": 10, + \"ReplicaProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"policy\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedWriteCapacityUnits\": 10, + \"ReplicaStatus\": \"ACTIVE\" + } + ] +}", + input_tests(Response, Tests). + +describe_global_table_settings_output_tests(_) -> + Tests = + [?_ddb_test( + {"DescribeGlobalTableSettings example response", " +{ + \"GlobalTableName\": \"Thread\", + \"ReplicaSettings\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaBillingModeSummary\": { + \"BillingMode\": \"PROVISIONED\", + \"LastUpdateToPayPerRequestDateTime\": 1578092745.455 + }, + \"ReplicaGlobalSecondaryIndexSettings\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedReadCapacityUnits\": 10, + \"ProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedWriteCapacityUnits\": 10 + } + ], + \"ReplicaProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedReadCapacityUnits\": 10, + \"ReplicaProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedWriteCapacityUnits\": 10, + \"ReplicaStatus\": \"ACTIVE\" + } + ] +}", {ok, [#ddb2_replica_settings_description{region_name = <<"us-west-2">>, + replica_billing_mode_summary = #ddb2_billing_mode_summary{billing_mode = provisioned, + last_update_to_pay_per_request_date_time = 1578092745.455}, + replica_global_secondary_index_settings = [#ddb2_replica_global_secondary_index_settings_description{index_name = <<"id-index">>, + index_status = active, + provisioned_read_capacity_auto_scaling_settings = #ddb2_auto_scaling_settings_description{auto_scaling_disabled = false, + auto_scaling_role_arn = <<"arn:test">>, + maximum_units = 20, + minimum_units = 10, + scaling_policies = [#ddb2_auto_scaling_policy_description{policy_name = <<"PolicyName">>, + target_tracking_scaling_policy_configuration = #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{disable_scale_in = false, + scale_in_cooldown = 600, + scale_out_cooldown = 600, + target_value = 70.0}}]}, + provisioned_read_capacity_units = 10, + provisioned_write_capacity_auto_scaling_settings = #ddb2_auto_scaling_settings_description{auto_scaling_disabled = false, + auto_scaling_role_arn = <<"arn:test">>, + maximum_units = 20, + minimum_units = 10, + scaling_policies = [#ddb2_auto_scaling_policy_description{policy_name = <<"PolicyName">>, + target_tracking_scaling_policy_configuration = #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{disable_scale_in = false, + scale_in_cooldown = 600, + scale_out_cooldown = 600, + target_value = 70.0}}]}, + provisioned_write_capacity_units = 10}], + replica_provisioned_read_capacity_auto_scaling_settings = #ddb2_auto_scaling_settings_description{auto_scaling_disabled = false, + auto_scaling_role_arn = <<"arn:test">>, + maximum_units = 20, + minimum_units = 10, + scaling_policies = [#ddb2_auto_scaling_policy_description{policy_name = <<"PolicyName">>, + target_tracking_scaling_policy_configuration = #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{disable_scale_in = false, + scale_in_cooldown = 600, + scale_out_cooldown = 600, + target_value = 70.0}}]}, + replica_provisioned_read_capacity_units = 10, + replica_provisioned_write_capacity_auto_scaling_settings = #ddb2_auto_scaling_settings_description{auto_scaling_disabled = false, + auto_scaling_role_arn = <<"arn:test">>, + maximum_units = 20, + minimum_units = 10, + scaling_policies = [#ddb2_auto_scaling_policy_description{policy_name = <<"PolicyName">>, + target_tracking_scaling_policy_configuration = #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{disable_scale_in = false, + scale_in_cooldown = 600, + scale_out_cooldown = 600, + target_value = 70.0}}]}, + replica_provisioned_write_capacity_units = 10, + replica_status = active}]}})], + output_tests(?_f(erlcloud_ddb2:describe_global_table_settings(<<"Thread">>)), Tests). + %% DescribeTable test based on the API examples: %% http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_DescribeTable.html describe_table_input_tests(_) -> @@ -2852,6 +3108,216 @@ describe_table_output_tests(_) -> output_tests(?_f(erlcloud_ddb2:describe_table(<<"name">>)), Tests). +%% DescribeTableReplicaAutoScaling test based on API request/response syntax +describe_table_replica_auto_scaling_input_tests(_) -> + Tests = + [?_ddb_test( + {"DescribeTableReplicaAutoScaling example request", + ?_f(erlcloud_ddb2:describe_table_replica_auto_scaling(<<"Thread">>)), " +{ + \"TableName\":\"Thread\" +}" + })], + Response = " +{ + \"TableAutoScalingDescription\": { + \"Replicas\": [ + { + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + } + } + ], + \"RegionName\": \"us-west-2\", + \"ReplicaProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaStatus\": \"ACTIVE\" + } + ], + \"TableName\": \"Thread\", + \"TableStatus\": \"ACTIVE\" + } +}", + input_tests(Response, Tests). + +describe_table_replica_auto_scaling_output_tests(_) -> + AutoScalingSettings = #ddb2_auto_scaling_settings_description{auto_scaling_disabled = false, + auto_scaling_role_arn = <<"arn:test">>, + maximum_units = 20, + minimum_units = 10, + scaling_policies = [#ddb2_auto_scaling_policy_description{policy_name = <<"PolicyName">>, + target_tracking_scaling_policy_configuration = #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{disable_scale_in = false, + scale_in_cooldown = 600, + scale_out_cooldown = 600, + target_value = 70.0}}]}, + Tests = + [?_ddb_test( + {"DescribeGlobalTableSettings example response", " +{ + \"TableAutoScalingDescription\": { + \"Replicas\": [ + { + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + } + } + ], + \"RegionName\": \"us-west-2\", + \"ReplicaProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaStatus\": \"ACTIVE\" + } + ], + \"TableName\": \"Thread\", + \"TableStatus\": \"ACTIVE\" + } +}", + {ok, #ddb2_table_auto_scaling_description{replicas = [#ddb2_replica_auto_scaling_description{global_secondary_indexes = [#ddb2_replica_global_secondary_index_auto_scaling_description{index_name = <<"id-index">>, + index_status = active, + provisioned_read_capacity_auto_scaling_settings = AutoScalingSettings, + provisioned_write_capacity_auto_scaling_settings = AutoScalingSettings}], + region_name = <<"us-west-2">>, + replica_provisioned_read_capacity_auto_scaling_settings = AutoScalingSettings, + replica_provisioned_write_capacity_auto_scaling_settings = AutoScalingSettings, + replica_status = active}], + table_name = <<"Thread">>, + table_status = active}}})], + output_tests(?_f(erlcloud_ddb2:describe_table_replica_auto_scaling(<<"Thread">>)), Tests). + %% DescribeTimeToLive test describe_time_to_live_input_tests(_) -> Tests = @@ -5598,6 +6064,375 @@ update_global_table_output_tests(_) -> replication_group = [#ddb2_replica_description{region_name = <<"eu-west-1">>}]}}})], output_tests(?_f(erlcloud_ddb2:update_global_table(<<"Thread">>, {create, {region_name, <<"us-east-1">>}})), Tests). +update_global_table_settings_input_tests(_) -> + ReadUnits = 10, + WriteUnits = 10, + StaticProvisionOpts = [{global_table_billing_mode, provisioned}, + {global_table_global_secondary_index_settings_update, [[{index_name, <<"id-index">>}, + {provisioned_write_capacity_units, WriteUnits}]]}, + {global_table_provisioned_write_capacity_units, WriteUnits}, + {replica_settings_update, [[{region_name, <<"us-west-2">>}, + {replica_global_secondary_index_settings_update, [[{index_name, <<"id-index">>}, + {provisioned_read_capacity_units, ReadUnits}]]}, + {replica_provisioned_read_capacity_units, ReadUnits}]]}], + AutoScalingSettingsUpdate = [{maximum_units, 20}, + {minimum_units, 10}, + {scaling_policy_update, [{target_tracking_scaling_policy_configuration, [{disable_scale_in, false}, + {scale_in_cooldown, 600}, + {scale_out_cooldown, 600}, + {target_value, 70.0}]}]}], + AutoScalingProvisionOpts = [{global_table_billing_mode, provisioned}, + {global_table_global_secondary_index_settings_update, [[{index_name, <<"id-index">>}, + {provisioned_write_capacity_auto_scaling_settings_update, AutoScalingSettingsUpdate}]]}, + {global_table_provisioned_write_capacity_auto_scaling_settings_update, AutoScalingSettingsUpdate}, + {replica_settings_update, [[{region_name, <<"us-west-2">>}, + {replica_global_secondary_index_settings_update, [[{index_name, <<"id-index">>}, + {provisioned_read_capacity_auto_scaling_settings_update, AutoScalingSettingsUpdate}]]}, + {replica_provisioned_read_capacity_auto_scaling_settings_update, AutoScalingSettingsUpdate}]]}], + Tests = + [?_ddb_test( + {"UpdateGlobalTableSettings example request: update all read/write capacity to static values", + ?_f(erlcloud_ddb2:update_global_table_settings(<<"Thread">>, StaticProvisionOpts)), " +{ + \"GlobalTableBillingMode\": \"PROVISIONED\", + \"GlobalTableGlobalSecondaryIndexSettingsUpdate\": [ + { + \"IndexName\": \"id-index\", + \"ProvisionedWriteCapacityUnits\": 10 + } + ], + \"GlobalTableName\": \"Thread\", + \"GlobalTableProvisionedWriteCapacityUnits\": 10, + \"ReplicaSettingsUpdate\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaGlobalSecondaryIndexSettingsUpdate\": [ + { + \"IndexName\": \"id-index\", + \"ProvisionedReadCapacityUnits\": 10 + } + ], + \"ReplicaProvisionedReadCapacityUnits\": 10 + } + ] +}" + }), + ?_ddb_test( + {"UpdateGlobalTableSettings example request: update all read/write capacity to autoscale", + ?_f(erlcloud_ddb2:update_global_table_settings(<<"Thread">>, AutoScalingProvisionOpts)), " +{ + \"GlobalTableBillingMode\": \"PROVISIONED\", + \"GlobalTableGlobalSecondaryIndexSettingsUpdate\": [ + { + \"IndexName\": \"id-index\", + \"ProvisionedWriteCapacityAutoScalingSettingsUpdate\": { + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicyUpdate\": { + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + } + } + ], + \"GlobalTableName\": \"Thread\", + \"GlobalTableProvisionedWriteCapacityAutoScalingSettingsUpdate\": { + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicyUpdate\": { + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + }, + \"ReplicaSettingsUpdate\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaGlobalSecondaryIndexSettingsUpdate\": [ + { + \"IndexName\": \"id-index\", + \"ProvisionedReadCapacityAutoScalingSettingsUpdate\": { + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicyUpdate\": { + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + } + } + ], + \"ReplicaProvisionedReadCapacityAutoScalingSettingsUpdate\": { + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicyUpdate\": { + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + } + } + ] +}" + })], + Response = " +{ + \"GlobalTableName\": \"Thread\", + \"ReplicaSettings\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaBillingModeSummary\": { + \"BillingMode\": \"PROVISIONED\", + \"LastUpdateToPayPerRequestDateTime\": 1578092745.455 + }, + \"ReplicaGlobalSecondaryIndexSettings\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedReadCapacityUnits\": 10, + \"ProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"string\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedWriteCapacityUnits\": 10 + } + ], + \"ReplicaProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"string\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedReadCapacityUnits\": 10, + \"ReplicaProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"string\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedWriteCapacityUnits\": 10, + \"ReplicaStatus\": \"ACTIVE\" + } + ] +}", + input_tests(Response, Tests). + +update_global_table_settings_output_tests(_) -> + AutoScalingSettings = #ddb2_auto_scaling_settings_description{auto_scaling_disabled = false, + auto_scaling_role_arn = <<"arn:test">>, + maximum_units = 20, + minimum_units = 10, + scaling_policies = [#ddb2_auto_scaling_policy_description{policy_name = <<"PolicyName">>, + target_tracking_scaling_policy_configuration = #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{disable_scale_in = false, + scale_in_cooldown = 600, + scale_out_cooldown = 600, + target_value = 70.0}}]}, + Tests = + [?_ddb_test( + {"UpdateGlobalTableSettings example response: update all read/write capacity to static values", " +{ + \"GlobalTableName\": \"Thread\", + \"ReplicaSettings\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaBillingModeSummary\": { + \"BillingMode\": \"PROVISIONED\", + \"LastUpdateToPayPerRequestDateTime\": 1578092745.455 + }, + \"ReplicaGlobalSecondaryIndexSettings\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityUnits\": 10, + \"ProvisionedWriteCapacityUnits\": 10 + } + ], + \"ReplicaProvisionedReadCapacityUnits\": 10, + \"ReplicaProvisionedWriteCapacityUnits\": 10, + \"ReplicaStatus\": \"ACTIVE\" + } + ] +}", + {ok, [#ddb2_replica_settings_description{region_name = <<"us-west-2">>, + replica_billing_mode_summary = #ddb2_billing_mode_summary{billing_mode = provisioned, + last_update_to_pay_per_request_date_time = 1578092745.455}, + replica_global_secondary_index_settings = [#ddb2_replica_global_secondary_index_settings_description{index_name = <<"id-index">>, + index_status = active, + provisioned_read_capacity_units = 10, + provisioned_write_capacity_units = 10}], + replica_provisioned_read_capacity_units = 10, + replica_provisioned_write_capacity_units = 10, + replica_status = active}]}} + ), + ?_ddb_test( + {"UpdateGlobalTableSettings example response: update all read/write capacity to autoscale", " +{ + \"GlobalTableName\": \"Thread\", + \"ReplicaSettings\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaBillingModeSummary\": { + \"BillingMode\": \"PROVISIONED\", + \"LastUpdateToPayPerRequestDateTime\": 1578092745.455 + }, + \"ReplicaGlobalSecondaryIndexSettings\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + } + ], + \"ReplicaProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaStatus\": \"ACTIVE\" + } + ] +} + +", + {ok, [#ddb2_replica_settings_description{region_name = <<"us-west-2">>, + replica_billing_mode_summary = #ddb2_billing_mode_summary{billing_mode = provisioned, + last_update_to_pay_per_request_date_time = 1578092745.455}, + replica_global_secondary_index_settings = [#ddb2_replica_global_secondary_index_settings_description{index_name = <<"id-index">>, + index_status = active, + provisioned_read_capacity_auto_scaling_settings = AutoScalingSettings, + provisioned_write_capacity_auto_scaling_settings = AutoScalingSettings}], + replica_provisioned_read_capacity_auto_scaling_settings = AutoScalingSettings, + replica_provisioned_write_capacity_auto_scaling_settings = AutoScalingSettings, + replica_status = active}]}} + )], + output_tests(?_f(erlcloud_ddb2:update_global_table_settings(<<"Thread">>, [])), Tests). + %% UpdateTable test based on the API examples: %% http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateTable.html update_table_input_tests(_) -> @@ -6026,6 +6861,309 @@ update_table_output_tests(_) -> output_tests(?_f(erlcloud_ddb2:update_table(<<"name">>, 5, 15)), Tests). +update_table_replica_auto_scaling_input_tests(_) -> + AutoScalingSettingsUpdate = [{auto_scaling_disabled, false}, + {auto_scaling_role_arn, <<"arn:test">>}, + {maximum_units, 20}, + {minimum_units, 10}, + {scaling_policy_update, [{policy_name, <<"PolicyName">>}, + {target_tracking_scaling_policy_configuration, [{disable_scale_in, false}, + {scale_in_cooldown, 600}, + {scale_out_cooldown, 600}, + {target_value, 70.0}]}]}], + Opts = [{global_secondary_index_updates, [[{index_name, <<"id-index">>}, + {provisioned_write_capacity_auto_scaling_update, AutoScalingSettingsUpdate}]]}, + {provisioned_write_capacity_auto_scaling_update, AutoScalingSettingsUpdate}, + {replica_updates, [[{region_name, <<"us-west-2">>}, + {replica_global_secondary_index_updates, [[{index_name, <<"id-index">>}, + {provisioned_read_capacity_auto_scaling_update, AutoScalingSettingsUpdate}]]}, + {replica_provisioned_read_capacity_auto_scaling_update, AutoScalingSettingsUpdate}]]}], + Tests = + [?_ddb_test({"UpdateTableReplicaAutoScaling example request", + ?_f(erlcloud_ddb2:update_table_replica_auto_scaling(<<"Thread">>, Opts)), + " +{ + \"GlobalSecondaryIndexUpdates\": [ + { + \"IndexName\": \"id-index\", + \"ProvisionedWriteCapacityAutoScalingUpdate\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicyUpdate\": { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + } + } + ], + \"ProvisionedWriteCapacityAutoScalingUpdate\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicyUpdate\": { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + }, + \"ReplicaUpdates\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaGlobalSecondaryIndexUpdates\": [ + { + \"IndexName\": \"id-index\", + \"ProvisionedReadCapacityAutoScalingUpdate\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicyUpdate\": { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + } + } + ], + \"ReplicaProvisionedReadCapacityAutoScalingUpdate\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicyUpdate\": { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + } + } + ], + \"TableName\": \"Thread\" +} + +"})], + + Response = " +{ + \"TableAutoScalingDescription\": { + \"Replicas\": [ + { + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + } + } + ], + \"RegionName\": \"us-west-2\", + \"ReplicaProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaStatus\": \"ACTIVE\" + } + ], + \"TableName\": \"Thread\", + \"TableStatus\": \"ACTIVE\" + } +}", + input_tests(Response, Tests). + +update_table_replica_auto_scaling_output_tests(_) -> + AutoScalingSettings = #ddb2_auto_scaling_settings_description{auto_scaling_disabled = false, + auto_scaling_role_arn = <<"arn:test">>, + maximum_units = 20, + minimum_units = 10, + scaling_policies = [#ddb2_auto_scaling_policy_description{policy_name = <<"PolicyName">>, + target_tracking_scaling_policy_configuration = #ddb2_auto_scaling_target_tracking_scaling_policy_configuration_description{disable_scale_in = false, + scale_in_cooldown = 600, + scale_out_cooldown = 600, + target_value = 70.0}}]}, + Tests = + [?_ddb_test( + {"UpdateTableReplicaAutoScaling example", " +{ + \"TableAutoScalingDescription\": { + \"Replicas\": [ + { + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"id-index\", + \"IndexStatus\": \"ACTIVE\", + \"ProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + } + } + ], + \"RegionName\": \"us-west-2\", + \"ReplicaProvisionedReadCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaProvisionedWriteCapacityAutoScalingSettings\": { + \"AutoScalingDisabled\": false, + \"AutoScalingRoleArn\": \"arn:test\", + \"MaximumUnits\": 20, + \"MinimumUnits\": 10, + \"ScalingPolicies\": [ + { + \"PolicyName\": \"PolicyName\", + \"TargetTrackingScalingPolicyConfiguration\": { + \"DisableScaleIn\": false, + \"ScaleInCooldown\": 600, + \"ScaleOutCooldown\": 600, + \"TargetValue\": 70.0 + } + } + ] + }, + \"ReplicaStatus\": \"ACTIVE\" + } + ], + \"TableName\": \"Thread\", + \"TableStatus\": \"ACTIVE\" + } +}", + {ok, #ddb2_table_auto_scaling_description{replicas = [#ddb2_replica_auto_scaling_description{global_secondary_indexes = [#ddb2_replica_global_secondary_index_auto_scaling_description{index_name = <<"id-index">>, + index_status = active, + provisioned_read_capacity_auto_scaling_settings = AutoScalingSettings, + provisioned_write_capacity_auto_scaling_settings = AutoScalingSettings}], + region_name = <<"us-west-2">>, + replica_provisioned_read_capacity_auto_scaling_settings = AutoScalingSettings, + replica_provisioned_write_capacity_auto_scaling_settings = AutoScalingSettings, + replica_status = active}], + table_name = <<"Thread">>, + table_status = active}} +})], + output_tests(?_f(erlcloud_ddb2:update_table_replica_auto_scaling(<<"Thread">>, [])), Tests). + %% UpdateTimeToLive test: update_time_to_live_input_tests(_) -> Tests = From 21cc8338754460fb951e3c2e6941a510f5011970 Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Mon, 6 Jan 2020 13:28:59 -0600 Subject: [PATCH 097/310] Fix new global table record typing, add support for CREATION_FAILED to replica_status undynamizer Address first round of feedback: * Fix missing types in new global table record typing * Add CREATION_FAILED to replica_status undynamizer and fix replica_status type * Fix new function comments --- include/erlcloud_ddb2.hrl | 6 +++--- src/erlcloud_ddb2.erl | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index 320ee4e28..1dbf84b16 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -16,7 +16,7 @@ -type date_time() :: number(). -type global_table_status() :: creating | active | deleting | updating. --type replica_status() :: creating | active | deleting | active. +-type replica_status() :: creating | creation_failed | updating | deleting | active. -type table_status() :: creating | updating | deleting | active. -type backup_status() :: creating | deleted | available. -type index_status() :: creating | updating | deleting | active. @@ -168,7 +168,7 @@ }). -record(ddb2_table_auto_scaling_description, - {replicas :: undefined, + {replicas :: undefined | [#ddb2_replica_auto_scaling_description{}], table_name :: undefined | binary(), table_status :: undefined | table_status()}). @@ -240,7 +240,7 @@ }). -record(ddb2_describe_table_replica_auto_scaling, - {table_auto_scaling_description :: undefined}). + {table_auto_scaling_description :: undefined | #ddb2_table_auto_scaling_description{}}). -record(ddb2_time_to_live_description, {attribute_name :: undefined | erlcloud_ddb2:attr_name(), diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index f96d1c1c8..cf72bcc2f 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -399,10 +399,10 @@ default_config() -> erlcloud_aws:default_config(). -type ok_return(T) :: {ok, T} | {error, term()}. -type client_request_token() :: binary(). --type auto_scaling_target_tracking_scaling_policy_configuration_update_opt() :: {target_value, number()}| - {disable_scale_in, boolean()}| +-type auto_scaling_target_tracking_scaling_policy_configuration_update_opt() :: {disable_scale_in, boolean()}| {scale_in_cooldown, pos_integer()}| - {scale_out_cooldown, pos_integer()}. + {scale_out_cooldown, pos_integer()}| + {target_value, number()}. -type auto_scaling_target_tracking_scaling_policy_configuration_update_opts() :: [auto_scaling_target_tracking_scaling_policy_configuration_update_opt()]. -type auto_scaling_policy_opt() :: {policy_name, binary()}| @@ -887,6 +887,7 @@ undynamize_global_table_status(<<"ACTIVE">>, _) -> active. -spec undynamize_replica_status(binary(), undynamize_opts()) -> replica_status(). undynamize_replica_status(<<"CREATING">>, _) -> creating; +undynamize_replica_status(<<"CREATION_FAILED">>, _) -> creation_failed; undynamize_replica_status(<<"UPDATING">>, _) -> updating; undynamize_replica_status(<<"DELETING">>, _) -> deleting; undynamize_replica_status(<<"ACTIVE">>, _) -> active. @@ -2556,7 +2557,7 @@ describe_table_replica_auto_scaling(Table, Opts) -> %% %% ===Example=== %% -%% Describe "Thread" table. +%% Describes auto scaling settings across replicas of the global table called "Thread" at once. %% %% ` %% {ok, Description} = @@ -4065,7 +4066,7 @@ update_global_table_settings(GlobalTableName, Opts) -> %% %% Update global table settings for a table called "Thread" in us-west-2 and eu-west-2. %% -%% ``` +%% ` %% ReadUnits = 10, %% WriteUnits = 10, %% erlcloud_ddb2:update_global_table_settings( @@ -4082,7 +4083,7 @@ update_global_table_settings(GlobalTableName, Opts) -> %% {replica_global_secondary_index_settings_update, [[{index_name, <<"id-index">>}, %% {provisioned_read_capacity_units, ReadUnits}]]}, %% {replica_provisioned_read_capacity_units, ReadUnits}]]}]) -%% ''' +%% ' %% @end %%------------------------------------------------------------------------------ From 4bf276f6382884feefaa906ea5023a3b24850644 Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Thu, 9 Jan 2020 16:01:07 -0600 Subject: [PATCH 098/310] Decode GlobalTableVersion and Replicas in DescribeTable unit tests --- test/erlcloud_ddb2_tests.erl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index cf4ffcb80..acd39097f 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -2928,6 +2928,7 @@ describe_table_input_tests(_) -> } ], \"CreationDateTime\": 1.363729002358E9, + \"GlobalTableVersion\": \"2019.11.21\", \"ItemCount\": 0, \"KeySchema\": [ { @@ -2959,6 +2960,16 @@ describe_table_input_tests(_) -> } } ], + \"Replicas\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaStatus\": \"ACTIVE\", + }, + { + \"RegionName\": \"eu-west-2\", + \"ReplicaStatus\": \"ACTIVE\" + } + ], \"ProvisionedThroughput\": { \"NumberOfDecreasesToday\": 0, \"ReadCapacityUnits\": 5, @@ -3023,6 +3034,7 @@ describe_table_output_tests(_) -> } ], \"CreationDateTime\": 1.363729002358E9, + \"GlobalTableVersion\": \"2019.11.21\", \"ItemCount\": 0, \"KeySchema\": [ { @@ -3054,6 +3066,16 @@ describe_table_output_tests(_) -> } } ], + \"Replicas\": [ + { + \"RegionName\": \"us-west-2\", + \"ReplicaStatus\": \"ACTIVE\", + }, + { + \"RegionName\": \"eu-west-2\", + \"ReplicaStatus\": \"ACTIVE\" + } + ], \"ProvisionedThroughput\": { \"NumberOfDecreasesToday\": 0, \"ReadCapacityUnits\": 5, @@ -3069,6 +3091,7 @@ describe_table_output_tests(_) -> {<<"LastPostDateTime">>, s}, {<<"Subject">>, s}], creation_date_time = 1363729002.358, + global_table_version = <<"2019.11.21">>, item_count = 0, key_schema = {<<"ForumName">>, <<"Subject">>}, local_secondary_indexes = @@ -3100,6 +3123,10 @@ describe_table_output_tests(_) -> number_of_decreases_today = 0, read_capacity_units = 5, write_capacity_units = 5}, + replicas = [#ddb2_replica_description{region_name = <<"us-west-2">>, + replica_status = active}, + #ddb2_replica_description{region_name = <<"eu-west-2">>, + replica_status = active}], table_name = <<"Thread">>, table_size_bytes = 0, table_status = active}}}) From 50b8b1b10373be90a2d73a043cbbc6f90b47042e Mon Sep 17 00:00:00 2001 From: k32 <10274441+k32@users.noreply.github.com> Date: Thu, 30 Jan 2020 14:14:04 +0100 Subject: [PATCH 099/310] Get rid of "term() | no_return()" specs Fixes #559 --- src/erlcloud_elb.erl | 114 ++++++++++++------------ src/erlcloud_mon.erl | 28 +++--- src/erlcloud_mturk.erl | 194 ++++++++++++++++++++--------------------- src/erlcloud_s3.erl | 120 ++++++++++++------------- src/erlcloud_sdb.erl | 66 +++++++------- src/erlcloud_sns.erl | 136 ++++++++++++++--------------- src/erlcloud_sqs.erl | 94 ++++++++++---------- src/erlcloud_sts.erl | 6 +- 8 files changed, 379 insertions(+), 379 deletions(-) diff --git a/src/erlcloud_elb.erl b/src/erlcloud_elb.erl index 5c8f6ebc8..3f65ff540 100644 --- a/src/erlcloud_elb.erl +++ b/src/erlcloud_elb.erl @@ -40,13 +40,13 @@ -define(DEFAULT_MAX_RECORDS, 400). % xpath for elb descriptions used in describe_groups functions: --define(DESCRIBE_ELBS_PATH, +-define(DESCRIBE_ELBS_PATH, "/DescribeLoadBalancersResponse/DescribeLoadBalancersResult/LoadBalancerDescriptions/member"). --define(DESCRIBE_ELBS_NEXT_TOKEN, +-define(DESCRIBE_ELBS_NEXT_TOKEN, "/DescribeLoadBalancersResponse/NextMarker"). --define(DESCRIBE_ELB_POLICIES_PATH, +-define(DESCRIBE_ELB_POLICIES_PATH, "/DescribeLoadBalancerPoliciesResponse/DescribeLoadBalancerPoliciesResult/PolicyDescriptions/member"). --define(DESCRIBE_ELB_POLICY_TYPE_PATH, +-define(DESCRIBE_ELB_POLICY_TYPE_PATH, "/DescribeLoadBalancerPolicyTypesResponse/DescribeLoadBalancerPolicyTypesResult/PolicyTypeDescriptions/member"). -define(DESCRIBE_ELBS_TAGS_PATH, "/DescribeTagsResponse/DescribeTagsResult/TagDescriptions/member"). @@ -114,11 +114,11 @@ delete_load_balancer(LB, Config) when is_list(LB) -> [{"LoadBalancerName", LB}]). --spec register_instance(string(), string()) -> ok | no_return(). +-spec register_instance(string(), string()) -> ok. register_instance(LB, InstanceId) -> register_instance(LB, InstanceId, default_config()). --spec register_instance(string(), string(), aws_config()) -> ok | no_return(). +-spec register_instance(string(), string(), aws_config()) -> ok. register_instance(LB, InstanceId, Config) when is_list(LB) -> elb_simple_request(Config, "RegisterInstancesWithLoadBalancer", @@ -126,11 +126,11 @@ register_instance(LB, InstanceId, Config) when is_list(LB) -> erlcloud_aws:param_list([[{"InstanceId", InstanceId}]], "Instances.member")]). --spec deregister_instance(string(), string()) -> ok | no_return(). +-spec deregister_instance(string(), string()) -> ok. deregister_instance(LB, InstanceId) -> deregister_instance(LB, InstanceId, default_config()). --spec deregister_instance(string(), string(), aws_config()) -> ok | no_return(). +-spec deregister_instance(string(), string(), aws_config()) -> ok. deregister_instance(LB, InstanceId, Config) when is_list(LB) -> elb_simple_request(Config, "DeregisterInstancesFromLoadBalancer", @@ -139,13 +139,13 @@ deregister_instance(LB, InstanceId, Config) when is_list(LB) -> --spec configure_health_check(string(), string()) -> ok | no_return(). +-spec configure_health_check(string(), string()) -> ok. configure_health_check(LB, Target) when is_list(LB), is_list(Target) -> configure_health_check(LB, Target, default_config()). --spec configure_health_check(string(), string(), aws_config()) -> ok | no_return(). +-spec configure_health_check(string(), string(), aws_config()) -> ok. configure_health_check(LB, Target, Config) when is_list(LB) -> elb_simple_request(Config, "ConfigureHealthCheck", @@ -153,8 +153,8 @@ configure_health_check(LB, Target, Config) when is_list(LB) -> {"HealthCheck.Target", Target}]). %% -------------------------------------------------------------------- -%% @doc describe_load_balancer with a specific balancer name or with a -%% specific configuration and specific balancer name. +%% @doc describe_load_balancer with a specific balancer name or with a +%% specific configuration and specific balancer name. %% @end %% -------------------------------------------------------------------- -spec describe_load_balancer(string()) -> {ok, term()} | {{paged, string()}, term()} | {error, metadata_not_available | container_credentials_unavailable | erlcloud_aws:httpc_result_error()}. @@ -171,7 +171,7 @@ describe_load_balancers() -> describe_load_balancers([], default_config()). %% -------------------------------------------------------------------- -%% @doc describe_load_balancers with specific balancer names or with a +%% @doc describe_load_balancers with specific balancer names or with a %% specific configuration. %% @end %% -------------------------------------------------------------------- @@ -184,13 +184,13 @@ describe_load_balancers(Config = #aws_config{}) -> %% @doc Get descriptions of the given load balancers. %% The account calling this function needs permission for the %% elasticloadbalancing:DescribeLoadBalancers action. -%% +%% %% Returns {{paged, NextPageId}, Results} if there are more than %% the current maximum count of results, {ok, Results} if everything %% fits and {error, Reason} if there was a problem. %% @end %% -------------------------------------------------------------------- --spec describe_load_balancers(list(string()), aws_config()) -> +-spec describe_load_balancers(list(string()), aws_config()) -> {ok, term()} | {{paged, string()}, term()} | {error, metadata_not_available | container_credentials_unavailable | erlcloud_aws:httpc_result_error()}. describe_load_balancers(Names, Config) -> describe_load_balancers(Names, ?DEFAULT_MAX_RECORDS, none, Config). @@ -200,20 +200,20 @@ describe_load_balancers(Names, Config) -> %% maximum number of results and optional paging offset. %% @end %% -------------------------------------------------------------------- --spec describe_load_balancers(list(string()), integer(), string() | none, aws_config()) -> +-spec describe_load_balancers(list(string()), integer(), string() | none, aws_config()) -> {ok, term()} | {{paged, string()}, term()} | {error, metadata_not_available | container_credentials_unavailable | erlcloud_aws:httpc_result_error()}. describe_load_balancers(Names, PageSize, none, Config) -> describe_load_balancers(Names, [{"PageSize", PageSize}], Config); describe_load_balancers(Names, PageSize, Marker, Config) -> describe_load_balancers(Names, [{"Marker", Marker}, {"PageSize", PageSize}], Config). --spec describe_load_balancers(list(string()), list({string(), term()}), aws_config()) -> +-spec describe_load_balancers(list(string()), list({string(), term()}), aws_config()) -> {ok, term()} | {{paged, string()}, term()} | {error, metadata_not_available | container_credentials_unavailable | erlcloud_aws:httpc_result_error()}. describe_load_balancers(Names, Params, Config) -> P = member_params("LoadBalancerNames.member.", Names) ++ Params, case elb_query(Config, "DescribeLoadBalancers", P) of {ok, Doc} -> - Elbs = xmerl_xpath:string(?DESCRIBE_ELBS_PATH, Doc), + Elbs = xmerl_xpath:string(?DESCRIBE_ELBS_PATH, Doc), {erlcloud_util:next_token(?DESCRIBE_ELBS_NEXT_TOKEN, Doc), [extract_elb(Elb) || Elb <- Elbs]}; {error, Reason} -> {error, Reason} @@ -258,8 +258,8 @@ describe_load_balancers_all(Names, Config) -> Names, ?DEFAULT_MAX_RECORDS, Marker, Cfg ) end, Config, none, []). - - + + extract_elb(Item) -> [ {load_balancer_name, get_text("LoadBalancerName", Item)}, @@ -296,11 +296,11 @@ extract_listener(Item) -> ]. %% -------------------------------------------------------------------- -%% @doc Calls describe_load_balancer_policies([], +%% @doc Calls describe_load_balancer_policies([], %% default_configuration()) %% @end %% -------------------------------------------------------------------- --spec describe_load_balancer_policies() -> +-spec describe_load_balancer_policies() -> {ok, term()} | {error, term()}. describe_load_balancer_policies() -> describe_load_balancer_policies([], [], default_config()). @@ -309,23 +309,23 @@ describe_load_balancer_policies() -> %% @doc describe_load_balancer_policies with specific config. %% @end %% -------------------------------------------------------------------- --spec describe_load_balancer_policies(aws_config()) -> +-spec describe_load_balancer_policies(aws_config()) -> {ok, term()} | {error, term()}. describe_load_balancer_policies(Config = #aws_config{}) -> describe_load_balancer_policies([], [], Config). %% -------------------------------------------------------------------- -%% @doc describe_load_balancer_policies for specified ELB +%% @doc describe_load_balancer_policies for specified ELB %% with specificied policy names using default config. %% @end %% -------------------------------------------------------------------- --spec describe_load_balancer_policies(string(), list() | aws_config()) -> +-spec describe_load_balancer_policies(string(), list() | aws_config()) -> {ok, term()} | {error, term()}. -describe_load_balancer_policies(ElbName, PolicyNames) +describe_load_balancer_policies(ElbName, PolicyNames) when is_list(ElbName), is_list(PolicyNames) -> describe_load_balancer_policies(ElbName, PolicyNames, default_config()); -describe_load_balancer_policies(PolicyNames, Config = #aws_config{}) +describe_load_balancer_policies(PolicyNames, Config = #aws_config{}) when is_list(PolicyNames) -> describe_load_balancer_policies([], PolicyNames, Config). @@ -335,7 +335,7 @@ describe_load_balancer_policies(PolicyNames, Config = #aws_config{}) %% with specified config. %% @end %% -------------------------------------------------------------------- --spec describe_load_balancer_policies(string(), list(), aws_config()) -> +-spec describe_load_balancer_policies(string(), list(), aws_config()) -> {ok, term()} | {error, term()}. describe_load_balancer_policies(ElbName, PolicyNames, Config) when is_list(ElbName), @@ -349,7 +349,7 @@ describe_load_balancer_policies(ElbName, PolicyNames, Config) Params = ElbNameParam ++ member_params("PolicyNames.member.", PolicyNames), case elb_query(Config, "DescribeLoadBalancerPolicies", Params) of {ok, Doc} -> - ElbPolicies = xmerl_xpath:string(?DESCRIBE_ELB_POLICIES_PATH, Doc), + ElbPolicies = xmerl_xpath:string(?DESCRIBE_ELB_POLICIES_PATH, Doc), {ok, [extract_elb_policy(ElbPolicy) || ElbPolicy <- ElbPolicies]}; {error, Reason} -> {error, Reason} @@ -359,8 +359,8 @@ extract_elb_policy(Item) -> [ {policy_name, get_text("PolicyName", Item)}, {policy_type_name, get_text("PolicyTypeName", Item)}, - {policy_attributes, - [extract_policy_attribute(A) || + {policy_attributes, + [extract_policy_attribute(A) || A <- xmerl_xpath:string("PolicyAttributeDescriptions/member", Item)]} ]. @@ -371,7 +371,7 @@ extract_policy_attribute(Item) -> ]. %% -------------------------------------------------------------------- -%% @doc Calls describe_load_balancer_policy_types([], +%% @doc Calls describe_load_balancer_policy_types([], %% default_configuration()) %% @end %% -------------------------------------------------------------------- @@ -379,7 +379,7 @@ describe_load_balancer_policy_types() -> describe_load_balancer_policy_types([], default_config()). %% -------------------------------------------------------------------- -%% @doc describe_load_balancer_policy_types() with specific +%% @doc describe_load_balancer_policy_types() with specific %% policy type names. %% @end %% -------------------------------------------------------------------- @@ -392,13 +392,13 @@ describe_load_balancer_policy_types(Config = #aws_config{}) -> %% @doc Get descriptions of the given load balancer policy types. %% @end %% -------------------------------------------------------------------- --spec describe_load_balancer_policy_types(list(string()), aws_config()) -> +-spec describe_load_balancer_policy_types(list(string()), aws_config()) -> {ok, term()} | {error, term()}. describe_load_balancer_policy_types(PolicyTypeNames, Config) -> P = member_params("PolicyTypeNames.member.", PolicyTypeNames), case elb_query(Config, "DescribeLoadBalancerPolicyTypes", P) of {ok, Doc} -> - ElbPolicyTypes = xmerl_xpath:string(?DESCRIBE_ELB_POLICY_TYPE_PATH, Doc), + ElbPolicyTypes = xmerl_xpath:string(?DESCRIBE_ELB_POLICY_TYPE_PATH, Doc), {ok, [extract_elb_policy_type(ElbPolicyType) || ElbPolicyType <- ElbPolicyTypes]}; {error, Reason} -> {error, Reason} @@ -408,8 +408,8 @@ extract_elb_policy_type(Item) -> [ {policy_type_name, get_text("PolicyTypeName", Item)}, {policy_type_description, get_text("Description", Item)}, - {policy_type_attributes, - [extract_policy_type_attribute(A) || + {policy_type_attributes, + [extract_policy_type_attribute(A) || A <- xmerl_xpath:string("PolicyAttributeTypeDescriptions/member", Item)]} ]. @@ -422,13 +422,13 @@ extract_policy_type_attribute(Item) -> {default_value, get_text("DefaultValue", Item)} ]. %% -------------------------------------------------------------------- -%% @doc Calls create_load_balancer_policy() without attributes and +%% @doc Calls create_load_balancer_policy() without attributes and %% with default aws config. %% @end %% -------------------------------------------------------------------- --spec create_load_balancer_policy(string(), string(), string()) -> - ok | {error, term()} | no_return(). -create_load_balancer_policy(LB, PolicyName, PolicyTypeName) +-spec create_load_balancer_policy(string(), string(), string()) -> + ok | {error, term()}. +create_load_balancer_policy(LB, PolicyName, PolicyTypeName) when is_list(LB), is_list(PolicyName), is_list(PolicyTypeName) -> @@ -438,9 +438,9 @@ create_load_balancer_policy(LB, PolicyName, PolicyTypeName) %% @doc Calls create_load_balancer_policy() with default aws config. %% @end %% -------------------------------------------------------------------- --spec create_load_balancer_policy(string(), string(), string(), list({string(), string()})) -> - ok | {error, term()} | no_return(). -create_load_balancer_policy(LB, PolicyName, PolicyTypeName, Attrs) +-spec create_load_balancer_policy(string(), string(), string(), list({string(), string()})) -> + ok | {error, term()}. +create_load_balancer_policy(LB, PolicyName, PolicyTypeName, Attrs) when is_list(LB), is_list(PolicyName), is_list(PolicyTypeName), @@ -452,9 +452,9 @@ create_load_balancer_policy(LB, PolicyName, PolicyTypeName, Attrs) %% http://docs.aws.amazon.com/ElasticLoadBalancing/latest/APIReference/API_CreateLoadBalancerPolicy.html %% @end %% -------------------------------------------------------------------- --spec create_load_balancer_policy(string(), string(), string(), list({string(), string()}), aws_config()) -> - ok | {error, term()} | no_return(). -create_load_balancer_policy(LB, PolicyName, PolicyTypeName, AttrList, Config) +-spec create_load_balancer_policy(string(), string(), string(), list({string(), string()}), aws_config()) -> + ok | {error, term()}. +create_load_balancer_policy(LB, PolicyName, PolicyTypeName, AttrList, Config) when is_list(LB), is_list(PolicyName), is_list(PolicyTypeName), @@ -465,17 +465,17 @@ create_load_balancer_policy(LB, PolicyName, PolicyTypeName, AttrList, Config) {"PolicyName", PolicyName}, {"PolicyTypeName", PolicyTypeName} | erlcloud_aws:param_list([[{"AttributeName", AttrName}, - {"AttributeValue", AttrValue}] || + {"AttributeValue", AttrValue}] || {AttrName, AttrValue} <- AttrList], "PolicyAttributes.member")]), ok. --spec describe_load_balancer_attributes(string()) -> proplist() | no_return(). +-spec describe_load_balancer_attributes(string()) -> proplist(). describe_load_balancer_attributes(Name) -> describe_load_balancer_attributes(Name, default_config()). --spec describe_load_balancer_attributes(string(), aws_config()) -> proplist() | no_return(). +-spec describe_load_balancer_attributes(string(), aws_config()) -> proplist(). describe_load_balancer_attributes(Name, Config) -> Node = elb_request(Config, "DescribeLoadBalancerAttributes", @@ -508,17 +508,17 @@ extract_elb_attribs(Node) -> %% @doc Calls delete_load_balancer_policy() with default aws config. %% @end %% -------------------------------------------------------------------- --spec delete_load_balancer_policy(string(), string()) -> ok | no_return(). -delete_load_balancer_policy(LB, PolicyName) when is_list(LB), +-spec delete_load_balancer_policy(string(), string()) -> ok. +delete_load_balancer_policy(LB, PolicyName) when is_list(LB), is_list(PolicyName) -> delete_load_balancer_policy(LB, PolicyName, default_config()). %% -------------------------------------------------------------------- -%% @doc Deletes the specified policy from the specified load balancer. +%% @doc Deletes the specified policy from the specified load balancer. %% This policy must not be enabled for any listeners. %% @end %% -------------------------------------------------------------------- --spec delete_load_balancer_policy(string(), string(), aws_config()) -> ok | no_return(). +-spec delete_load_balancer_policy(string(), string(), aws_config()) -> ok. delete_load_balancer_policy(LB, PolicyName, Config) when is_list(LB), is_list(PolicyName)-> elb_simple_request(Config, @@ -526,15 +526,15 @@ delete_load_balancer_policy(LB, PolicyName, Config) when is_list(LB), [{"LoadBalancerName", LB}, {"PolicyName", PolicyName}]). -%% given a list of member identifiers, return a list of +%% given a list of member identifiers, return a list of %% {key with prefix, member identifier} for use in elb calls. -%% Example pair that could be returned in a list is +%% Example pair that could be returned in a list is %% {"LoadBalancerNames.member.1", "my-elb}. -spec member_params(string(), list(string())) -> list({string(), string()}). member_params(Prefix, MemberIdentifiers) -> MemberKeys = [Prefix ++ integer_to_list(I) || I <- lists:seq(1, length(MemberIdentifiers))], [{K, V} || {K, V} <- lists:zip(MemberKeys, MemberIdentifiers)]. - + describe_all(Fun, AwsConfig, Marker, Acc) -> case Fun(Marker, AwsConfig) of diff --git a/src/erlcloud_mon.erl b/src/erlcloud_mon.erl index 9939613d9..93392d303 100644 --- a/src/erlcloud_mon.erl +++ b/src/erlcloud_mon.erl @@ -59,7 +59,7 @@ MetricName ::string(), DimensionFilter ::[{string(),string()}], NextToken ::string() - ) -> term() | no_return(). + ) -> term(). list_metrics( Namespace, @@ -75,7 +75,7 @@ list_metrics( DimensionFilter ::[{string(),string()}], NextToken ::string(), Config ::aws_config() - ) -> term() | no_return(). + ) -> term(). list_metrics( Namespace, @@ -321,7 +321,7 @@ extract_actions_dafm(Node) -> -spec put_metric_data( Namespace ::string(), MetricData ::[metric_datum()] - ) -> term() | no_return(). + ) -> term(). put_metric_data(Namespace, MetricData) -> put_metric_data(Namespace, MetricData, default_config()). @@ -330,7 +330,7 @@ put_metric_data(Namespace, MetricData) -> Namespace ::string(), MetricData ::[metric_datum()], Config ::aws_config() - ) -> term() | no_return(). + ) -> term(). put_metric_data(Namespace, MetricData, #aws_config{} = Config) -> @@ -407,7 +407,7 @@ params_stat(Prefix, StatisticValues) -> Value ::string(), Unit ::unit(), Timestamp ::datetime()|string() - ) -> term() | no_return(). + ) -> term(). put_metric_data(Namespace, MetricName, Value, Unit, Timestamp) -> put_metric_data(Namespace, MetricName, Value, Unit, Timestamp, default_config()). @@ -419,7 +419,7 @@ put_metric_data(Namespace, MetricName, Value, Unit, Timestamp) -> Unit ::unit(), Timestamp ::datetime()|string(), Config ::aws_config() - ) -> term() | no_return(). + ) -> term(). put_metric_data(Namespace, MetricName, Value, Unit, Timestamp, #aws_config{} = Config) -> Params = @@ -437,7 +437,7 @@ put_metric_data(Namespace, MetricName, Value, Unit, Timestamp, #aws_config{} = C %%------------------------------------------------------------------------------ %% @doc CloudWatch API - GetMetricStatistics - Easy average version -%% Gets average and max stats at 60 second intervals for +%% Gets average and max stats at 60 second intervals for %% the given metric on the given instance for the given interval %% @end %%------------------------------------------------------------------------------ @@ -446,7 +446,7 @@ put_metric_data(Namespace, MetricName, Value, Unit, Timestamp, #aws_config{} = C StartTime ::datetime() | string(), EndTime ::datetime() | string(), InstanceId ::string() - ) -> term() | no_return(). + ) -> term(). get_metric_statistics( MetricName, @@ -478,8 +478,8 @@ get_metric_statistics( %% "Percent", %% ["Average", "Maximum"], %% [{"InstanceType", "t2.micro"}]). -%% -%% @end +%% +%% @end %%------------------------------------------------------------------------------ -spec get_metric_statistics( Namespace ::string(), @@ -490,7 +490,7 @@ get_metric_statistics( Unit ::string(), Statistics ::[string()], Dimensions ::[{string(), string()}] - ) -> term() | no_return(). + ) -> term(). get_metric_statistics( Namespace, @@ -524,7 +524,7 @@ get_metric_statistics( Statistics ::[string()], Dimensions ::[{string(), string()}], Config ::aws_config() - ) -> term() | no_return(). + ) -> term(). get_metric_statistics( Namespace, @@ -584,7 +584,7 @@ extract_metrics(Node, Statistics) -> %%------------------------------------------------------------------------------ -spec get_alarm_state( AlarmName ::string() - ) -> string() | no_return(). + ) -> string(). get_alarm_state(AlarmName) -> get_alarm_state(AlarmName, default_config()). @@ -592,7 +592,7 @@ get_alarm_state(AlarmName) -> -spec get_alarm_state( AlarmName ::string(), Config ::aws_config() - ) -> string() | no_return(). + ) -> string(). get_alarm_state(AlarmName, #aws_config{} = Config) -> Params = [{"AlarmNames.member.1", AlarmName}], diff --git a/src/erlcloud_mturk.erl b/src/erlcloud_mturk.erl index 99bfb64ec..a0a94097a 100644 --- a/src/erlcloud_mturk.erl +++ b/src/erlcloud_mturk.erl @@ -92,22 +92,22 @@ configure(AccessKeyId, SecretAccessKey, Host) -> default_config() -> erlcloud_aws:default_config(). --spec approve_assignment(string(), string() | none) -> ok | no_return(). +-spec approve_assignment(string(), string() | none) -> ok. approve_assignment(AssignmentId, RequesterFeedback) -> approve_assignment(AssignmentId, RequesterFeedback, default_config()). --spec approve_assignment(string(), string() | none, aws_config()) -> ok | no_return(). +-spec approve_assignment(string(), string() | none, aws_config()) -> ok. approve_assignment(AssignmentId, RequesterFeedback, Config) when is_list(AssignmentId), is_list(RequesterFeedback) orelse RequesterFeedback =:= none -> mturk_simple_request(Config, "ApproveAssignment", [{"AssignmentId", AssignmentId}, {"RequesterFeedback", RequesterFeedback}]). --spec assign_qualification(string(), string()) -> ok | no_return(). +-spec assign_qualification(string(), string()) -> ok. assign_qualification(QualificationTypeId, WorkerId) -> assign_qualification(QualificationTypeId, WorkerId, default_config()). --spec assign_qualification(string(), string(), integer() | aws_config()) -> ok | no_return(). +-spec assign_qualification(string(), string(), integer() | aws_config()) -> ok. assign_qualification(QualificationTypeId, WorkerId, Config) when is_record(Config, aws_config) -> assign_qualification(QualificationTypeId, WorkerId, 1, Config); @@ -115,7 +115,7 @@ assign_qualification(QualificationTypeId, WorkerId, IntegerValue) -> assign_qualification(QualificationTypeId, WorkerId, IntegerValue, false). --spec assign_qualification(string(), string(), integer(), boolean() | aws_config()) -> ok | no_return(). +-spec assign_qualification(string(), string(), integer(), boolean() | aws_config()) -> ok. assign_qualification(QualificationTypeId, WorkerId, IntegerValue, Config) when is_record(Config, aws_config) -> assign_qualification(QualificationTypeId, WorkerId, IntegerValue, false, Config); @@ -123,7 +123,7 @@ assign_qualification(QualificationTypeId, WorkerId, IntegerValue, SendNotificati assign_qualification(QualificationTypeId, WorkerId, IntegerValue, SendNotification, default_config()). --spec assign_qualification(string(), string(), integer(), boolean(), aws_config()) -> ok | no_return(). +-spec assign_qualification(string(), string(), integer(), boolean(), aws_config()) -> ok. assign_qualification(QualificationTypeId, WorkerId, IntegerValue, SendNotification, Config) when is_list(QualificationTypeId), is_list(WorkerId), @@ -134,34 +134,34 @@ assign_qualification(QualificationTypeId, WorkerId, IntegerValue, SendNotificati {"IntegerValue", IntegerValue}, {"SendNotification", SendNotification}]). --spec block_worker(string(), string()) -> ok | no_return(). +-spec block_worker(string(), string()) -> ok. block_worker(WorkerId, Reason) -> block_worker(WorkerId, Reason, default_config()). --spec block_worker(string(), string(), aws_config()) -> ok | no_return(). +-spec block_worker(string(), string(), aws_config()) -> ok. block_worker(WorkerId, Reason, Config) when is_list(WorkerId), is_list(Reason) -> mturk_simple_request(Config, "BlockWorker", [{"WorkerId", WorkerId}, {"Reason", Reason}]). --spec change_hit_type_of_hit(string(), string()) -> ok | no_return(). +-spec change_hit_type_of_hit(string(), string()) -> ok. change_hit_type_of_hit(HITId, HITTypeId) -> change_hit_type_of_hit(HITId, HITTypeId, default_config()). --spec change_hit_type_of_hit(string(), string(), aws_config()) -> ok | no_return(). +-spec change_hit_type_of_hit(string(), string(), aws_config()) -> ok. change_hit_type_of_hit(HITId, HITTypeId, Config) when is_list(HITId), is_list(HITTypeId) -> mturk_simple_request(Config, "ChangeHITTypeOfHIT", [{"HITId", HITId}, {"HITTypeId", HITTypeId}]). -spec create_hit(string(), mturk_question(), 30..3153600, - 1..1000000000, string() | none) -> proplist() | no_return(). + 1..1000000000, string() | none) -> proplist(). create_hit(HITTypeId, Question, LifetimeInSeconds, MaxAssignments, RequesterAnnotation) -> create_hit(HITTypeId, Question, LifetimeInSeconds, MaxAssignments, RequesterAnnotation, default_config()). -spec create_hit(string(), mturk_question(), 30..3153600, - 1..1000000000, string() | none, aws_config()) -> proplist() | no_return(). + 1..1000000000, string() | none, aws_config()) -> proplist(). create_hit(HITTypeId, Question, LifetimeInSeconds, MaxAssignments, RequesterAnnotation, Config) when is_list(HITTypeId), @@ -187,11 +187,11 @@ create_hit(HITTypeId, Question, LifetimeInSeconds, MaxAssignments, Doc ). --spec create_hit(#mturk_hit{}) -> proplist() | no_return(). +-spec create_hit(#mturk_hit{}) -> proplist(). create_hit(HIT) -> create_hit(HIT, default_config()). --spec create_hit(#mturk_hit{}, aws_config()) -> proplist() | no_return(). +-spec create_hit(#mturk_hit{}, aws_config()) -> proplist(). create_hit(HIT, Config) -> QuestionXML = xml_to_string(encode_xml(HIT#mturk_hit.question)), Params = [ @@ -245,11 +245,11 @@ encode_locale_value(undefined) -> []; encode_locale_value(#mturk_locale{country_code=Country}) -> [{"Country", Country}]. --spec create_qualification_type(#mturk_qualification_type{}) -> proplist() | no_return(). +-spec create_qualification_type(#mturk_qualification_type{}) -> proplist(). create_qualification_type(QType) -> create_qualification_type(QType, default_config()). --spec create_qualification_type(#mturk_qualification_type{}, aws_config()) -> proplist() | no_return(). +-spec create_qualification_type(#mturk_qualification_type{}, aws_config()) -> proplist(). create_qualification_type(QType, Config) when is_record(QType, mturk_qualification_type) -> Doc = mturk_xml_request(Config, "CreateQualificationType", @@ -274,38 +274,38 @@ qualification_type_params(QType) -> {"AutoGrantedValue", case AutoGranted of true -> AutoGrantedValue; false -> undefined end} ]. --spec disable_hit(string()) -> ok | no_return(). +-spec disable_hit(string()) -> ok. disable_hit(HITId) -> disable_hit(HITId, default_config()). --spec disable_hit(string(), aws_config()) -> ok | no_return(). +-spec disable_hit(string(), aws_config()) -> ok. disable_hit(HITId, Config) when is_list(HITId) -> mturk_simple_request(Config, "DisableHIT", [{"HITId", HITId}]). --spec dispose_hit(string()) -> ok | no_return(). +-spec dispose_hit(string()) -> ok. dispose_hit(HITId) -> dispose_hit(HITId, default_config()). --spec dispose_hit(string(), aws_config()) -> ok | no_return(). +-spec dispose_hit(string(), aws_config()) -> ok. dispose_hit(HITId, Config) when is_list(HITId) -> mturk_simple_request(Config, "DisposeHIT", [{"HITId", HITId}]). --spec dispose_qualification_type(string()) -> ok | no_return(). +-spec dispose_qualification_type(string()) -> ok. dispose_qualification_type(QualificationTypeId) -> dispose_qualification_type(QualificationTypeId, default_config()). --spec dispose_qualification_type(string(), aws_config()) -> ok | no_return(). +-spec dispose_qualification_type(string(), aws_config()) -> ok. dispose_qualification_type(QualificationTypeId, Config) when is_list(QualificationTypeId) -> mturk_simple_request(Config, "DisposeQualificationType", [{"QualificationTypeId", QualificationTypeId}]). --spec extend_hit(string(), 1..1000000000 | none, 3600..31536000 | none) -> ok | no_return(). +-spec extend_hit(string(), 1..1000000000 | none, 3600..31536000 | none) -> ok. extend_hit(HITId, MaxAssignmentsIncrement, ExpirationIncrementInSeconds) -> extend_hit(HITId, MaxAssignmentsIncrement, ExpirationIncrementInSeconds, default_config()). --spec extend_hit(string(), 1..1000000000 | none, 3600..31536000 | none, aws_config()) -> ok | no_return(). +-spec extend_hit(string(), 1..1000000000 | none, 3600..31536000 | none, aws_config()) -> ok. extend_hit(HITId, MaxAssignmentsIncrement, ExpirationIncrementInSeconds, Config) when is_list(HITId), (MaxAssignmentsIncrement >= 1 andalso MaxAssignmentsIncrement =< 1000000000) orelse MaxAssignmentsIncrement =:= none, @@ -316,19 +316,19 @@ extend_hit(HITId, MaxAssignmentsIncrement, ExpirationIncrementInSeconds, Config) {"MaxAssignmentsIncrement", MaxAssignmentsIncrement}, {"ExpirationIncrementInSeconds", ExpirationIncrementInSeconds}]). --spec force_expire_hit(string()) -> ok | no_return(). +-spec force_expire_hit(string()) -> ok. force_expire_hit(HITId) -> force_expire_hit(HITId, default_config()). --spec force_expire_hit(string(), aws_config()) -> ok | no_return(). +-spec force_expire_hit(string(), aws_config()) -> ok. force_expire_hit(HITId, Config) when is_list(HITId) -> mturk_simple_request(Config, "ForceExpireHIT", [{"HITId", HITId}]). --spec get_account_balance() -> proplist() | no_return(). +-spec get_account_balance() -> proplist(). get_account_balance() -> get_account_balance(default_config()). --spec get_account_balance(aws_config()) -> proplist() | no_return(). +-spec get_account_balance(aws_config()) -> proplist(). get_account_balance(Config) -> Doc = mturk_xml_request(Config, "GetAccountBalance", []), erlcloud_xml:decode( @@ -339,18 +339,18 @@ get_account_balance(Config) -> Doc ). --spec get_assignments_for_hit(string()) -> proplist() | no_return(). +-spec get_assignments_for_hit(string()) -> proplist(). get_assignments_for_hit(HITId) -> get_assignments_for_hit(HITId, []). --spec get_assignments_for_hit(string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec get_assignments_for_hit(string(), proplist() | aws_config()) -> proplist(). get_assignments_for_hit(HITId, Config) when is_record(Config, aws_config) -> get_assignments_for_hit(HITId, [], Config); get_assignments_for_hit(HITId, Options) -> get_assignments_for_hit(HITId, Options, default_config()). --spec get_assignments_for_hit(string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec get_assignments_for_hit(string(), proplist(), aws_config()) -> proplist(). get_assignments_for_hit(HITId, Options, Config) when is_list(HITId), is_list(Options) -> Params = [ @@ -411,11 +411,11 @@ extract_assignment(Assignment) -> Assignment ). --spec get_bonus_payments_for_hit(string(), proplist()) -> proplist() | no_return(). +-spec get_bonus_payments_for_hit(string(), proplist()) -> proplist(). get_bonus_payments_for_hit(HITId, Options) -> get_bonus_payments_for_hit(HITId, Options, default_config()). --spec get_bonus_payments_for_hit(string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec get_bonus_payments_for_hit(string(), proplist(), aws_config()) -> proplist(). get_bonus_payments_for_hit(HITId, Options, Config) when is_list(HITId), is_list(Options) -> Params = [ @@ -426,11 +426,11 @@ get_bonus_payments_for_hit(HITId, Options, Config) Doc = mturk_xml_request(Config, "GetBonusPayments", Params), extract_bonus_payments(Doc). --spec get_bonus_payments_for_assignment(string(), proplist()) -> proplist() | no_return(). +-spec get_bonus_payments_for_assignment(string(), proplist()) -> proplist(). get_bonus_payments_for_assignment(AssignmentId, Options) -> get_bonus_payments_for_assignment(AssignmentId, Options, default_config()). --spec get_bonus_payments_for_assignment(string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec get_bonus_payments_for_assignment(string(), proplist(), aws_config()) -> proplist(). get_bonus_payments_for_assignment(AssignmentId, Options, Config) when is_list(AssignmentId), is_list(Options) -> Params = [ @@ -466,11 +466,11 @@ extract_bonus_payment(Payment) -> Payment ). --spec get_file_upload_url(string(), string()) -> string() | no_return(). +-spec get_file_upload_url(string(), string()) -> string(). get_file_upload_url(AssignmentId, QuestionIdentifier) -> get_file_upload_url(AssignmentId, QuestionIdentifier, default_config()). --spec get_file_upload_url(string(), string(), aws_config()) -> string() | no_return(). +-spec get_file_upload_url(string(), string(), aws_config()) -> string(). get_file_upload_url(AssignmentId, QuestionIdentifier, Config) when is_record(Config, aws_config) -> Params = [ @@ -480,27 +480,27 @@ get_file_upload_url(AssignmentId, QuestionIdentifier, Config) Doc = mturk_xml_request(Config, "GetFileUploadURL", Params), erlcloud_xml:get_text("FileUploadURL", Doc). --spec get_hit(string()) -> #mturk_hit{} | no_return(). +-spec get_hit(string()) -> #mturk_hit{}. get_hit(HITId) -> get_hit(HITId, default_config()). --spec get_hit(string(), aws_config()) -> #mturk_hit{} | no_return(). +-spec get_hit(string(), aws_config()) -> #mturk_hit{}. get_hit(HITId, Config) when is_list(HITId) -> Doc = mturk_xml_request(Config, "GetHIT", [{"HITId", HITId}]), hd(extract_hits([Doc])). --spec get_hits_for_qualification_type(string()) -> proplist() | no_return(). +-spec get_hits_for_qualification_type(string()) -> proplist(). get_hits_for_qualification_type(QualificationTypeId) -> get_hits_for_qualification_type(QualificationTypeId, []). --spec get_hits_for_qualification_type(string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec get_hits_for_qualification_type(string(), proplist() | aws_config()) -> proplist(). get_hits_for_qualification_type(QualificationTypeId, Config) when is_record(Config, aws_config) -> get_hits_for_qualification_type(QualificationTypeId, [], Config); get_hits_for_qualification_type(QualificationTypeId, Options) -> get_hits_for_qualification_type(QualificationTypeId, Options, default_config()). --spec get_hits_for_qualification_type(string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec get_hits_for_qualification_type(string(), proplist(), aws_config()) -> proplist(). get_hits_for_qualification_type(QualificationTypeId, Options, Config) when is_list(Options) -> Params = [ @@ -519,18 +519,18 @@ get_hits_for_qualification_type(QualificationTypeId, Options, Config) Doc ). --spec get_reviewable_hits() -> proplist() | no_return(). +-spec get_reviewable_hits() -> proplist(). get_reviewable_hits() -> get_reviewable_hits([]). --spec get_reviewable_hits(proplist() | aws_config()) -> proplist() | no_return(). +-spec get_reviewable_hits(proplist() | aws_config()) -> proplist(). get_reviewable_hits(Config) when is_record(Config, aws_config) -> get_reviewable_hits([], Config); get_reviewable_hits(Options) -> get_reviewable_hits(Options, default_config()). --spec get_reviewable_hits(proplist(), aws_config()) -> proplist() | no_return(). +-spec get_reviewable_hits(proplist(), aws_config()) -> proplist(). get_reviewable_hits(Options, Config) when is_list(Options) -> Params = [ @@ -573,11 +573,11 @@ get_reviewable_hits(Options, Config) Doc ). --spec get_qualification_score(string(), string()) -> proplist() | no_return(). +-spec get_qualification_score(string(), string()) -> proplist(). get_qualification_score(QualificationTypeId, SubjectId) -> get_qualification_score(QualificationTypeId, SubjectId, default_config()). --spec get_qualification_score(string(), string(), aws_config()) -> proplist() | no_return(). +-spec get_qualification_score(string(), string(), aws_config()) -> proplist(). get_qualification_score(QualificationTypeId, SubjectId, Config) when is_list(QualificationTypeId), is_list(SubjectId) -> Doc = mturk_xml_request(Config, "GetQualificationScore", @@ -596,11 +596,11 @@ get_qualification_score(QualificationTypeId, SubjectId, Config) Doc ). --spec get_qualification_type(string()) -> #mturk_qualification_type{} | no_return(). +-spec get_qualification_type(string()) -> #mturk_qualification_type{}. get_qualification_type(QualificationTypeId) -> get_qualification_type(QualificationTypeId, default_config()). --spec get_qualification_type(string(), aws_config()) -> #mturk_qualification_type{} | no_return(). +-spec get_qualification_type(string(), aws_config()) -> #mturk_qualification_type{}. get_qualification_type(QualificationTypeId, Config) when is_record(Config, aws_config) -> Doc = mturk_xml_request(Config, "GetQualificationType", @@ -635,18 +635,18 @@ extract_qualification_type(Node) -> decode_keywords(String) -> [string:strip(Keyword) || Keyword <- string:tokens(String, ",")]. --spec get_qualifications_for_qualification_type(string()) -> [proplist()] | no_return(). +-spec get_qualifications_for_qualification_type(string()) -> [proplist()]. get_qualifications_for_qualification_type(QualificationTypeId) -> get_qualifications_for_qualification_type(QualificationTypeId, default_config()). --spec get_qualifications_for_qualification_type(string(), proplist() | aws_config()) -> [proplist()] | no_return(). +-spec get_qualifications_for_qualification_type(string(), proplist() | aws_config()) -> [proplist()]. get_qualifications_for_qualification_type(QualificationTypeId, Config) when is_record(Config, aws_config) -> get_qualifications_for_qualification_type(QualificationTypeId, [], Config); get_qualifications_for_qualification_type(QualificationTypeId, Options) -> get_qualifications_for_qualification_type(QualificationTypeId, Options, default_config()). --spec get_qualifications_for_qualification_type(string(), proplist(), aws_config()) -> [proplist()] | no_return(). +-spec get_qualifications_for_qualification_type(string(), proplist(), aws_config()) -> [proplist()]. get_qualifications_for_qualification_type(QualificationTypeId, Options, Config) when is_list(QualificationTypeId), is_list(Options) -> Params = [ @@ -672,18 +672,18 @@ get_qualifications_for_qualification_type(QualificationTypeId, Options, Config) Item ) || Item <- xmerl_xpath:string("Qualification", Doc)]. --spec get_qualification_requests() -> proplist() | no_return(). +-spec get_qualification_requests() -> proplist(). get_qualification_requests() -> get_qualification_requests([]). --spec get_qualification_requests(proplist() | aws_config()) -> proplist() | no_return(). +-spec get_qualification_requests(proplist() | aws_config()) -> proplist(). get_qualification_requests(Config) when is_record(Config, aws_config) -> get_qualification_requests([], Config); get_qualification_requests(Options) -> get_qualification_requests(Options, default_config()). --spec get_qualification_requests(proplist(), aws_config()) -> proplist() | no_return(). +-spec get_qualification_requests(proplist(), aws_config()) -> proplist(). get_qualification_requests(Options, Config) when is_list(Options) -> Params = [ @@ -732,18 +732,18 @@ extract_qualification_request(Request) -> Request ). --spec get_requester_statistic(string(), one_day | seven_days | thirty_days | life_to_date) -> [{datetime(), float()}] | no_return(). +-spec get_requester_statistic(string(), one_day | seven_days | thirty_days | life_to_date) -> [{datetime(), float()}]. get_requester_statistic(Statistic, TimePeriod) -> get_requester_statistic(Statistic, TimePeriod, default_config()). --spec get_requester_statistic(string(), one_day | seven_days | thirty_days | life_to_date, pos_integer() | aws_config()) -> [{datetime(), float()}] | no_return(). +-spec get_requester_statistic(string(), one_day | seven_days | thirty_days | life_to_date, pos_integer() | aws_config()) -> [{datetime(), float()}]. get_requester_statistic(Statistic, TimePeriod, Config) when is_record(Config, aws_config) -> get_requester_statistic(Statistic, TimePeriod, 1, Config); get_requester_statistic(Statistic, TimePeriod, Count) -> get_requester_statistic(Statistic, TimePeriod, Count, default_config()). --spec get_requester_statistic(string(), one_day | seven_days | thirty_days | life_to_date, pos_integer(), aws_config()) -> [{datetime(), float()}] | no_return(). +-spec get_requester_statistic(string(), one_day | seven_days | thirty_days | life_to_date, pos_integer(), aws_config()) -> [{datetime(), float()}]. get_requester_statistic(Statistic, TimePeriod, Count, Config) when is_list(Statistic), TimePeriod =:= one_day orelse TimePeriod =:= seven_days orelse @@ -769,11 +769,11 @@ get_requester_statistic(Statistic, TimePeriod, Count, Config) end} || DP <- xmerl_xpath:string("DataPoint", Doc)]. --spec grant_bonus(string(), string(), #mturk_money{}, string()) -> ok | no_return(). +-spec grant_bonus(string(), string(), #mturk_money{}, string()) -> ok. grant_bonus(WorkerId, AssignmentId, BonusAmount, Reason) -> grant_bonus(WorkerId, AssignmentId, BonusAmount, Reason, default_config()). --spec grant_bonus(string(), string(), #mturk_money{}, string(), aws_config()) -> ok | no_return(). +-spec grant_bonus(string(), string(), #mturk_money{}, string(), aws_config()) -> ok. grant_bonus(WorkerId, AssignmentId, BonusAmount, Reason, Config) -> mturk_simple_request(Config, "GrantBonus", [ @@ -784,18 +784,18 @@ grant_bonus(WorkerId, AssignmentId, BonusAmount, Reason, Config) -> ] ). --spec grant_qualification(string()) -> ok | no_return(). +-spec grant_qualification(string()) -> ok. grant_qualification(QualificationRequestId) -> grant_qualification(QualificationRequestId, none). --spec grant_qualification(string(), integer() | none | aws_config()) -> ok | no_return(). +-spec grant_qualification(string(), integer() | none | aws_config()) -> ok. grant_qualification(QualificationRequestId, Config) when is_record(Config, aws_config) -> grant_qualification(QualificationRequestId, none, Config); grant_qualification(QualificationRequestId, Value) -> grant_qualification(QualificationRequestId, Value, default_config()). --spec grant_qualification(string(), integer() | none, aws_config()) -> ok | no_return(). +-spec grant_qualification(string(), integer() | none, aws_config()) -> ok. grant_qualification(QualificationRequestId, Value, Config) when is_list(QualificationRequestId), is_integer(Value) orelse Value =:= none -> @@ -1090,11 +1090,11 @@ extract_money(Money) -> encode_money(#mturk_money{amount=Amount, currency_code=CurrencyCode}) -> [{"Amount", Amount}, {"CurrencyCode", CurrencyCode}]. --spec notify_workers(string(), string(), [string()]) -> ok | no_return(). +-spec notify_workers(string(), string(), [string()]) -> ok. notify_workers(Subject, MessageText, WorkerIds) -> notify_workers(Subject, MessageText, WorkerIds, default_config()). --spec notify_workers(string(), string(), [string()], aws_config()) -> ok | no_return(). +-spec notify_workers(string(), string(), [string()], aws_config()) -> ok. notify_workers(Subject, MessageText, WorkerIds, Config) when is_list(Subject), is_list(MessageText), is_list(WorkerIds), length(WorkerIds) =< 100 -> @@ -1106,11 +1106,11 @@ notify_workers(Subject, MessageText, WorkerIds, Config) ] ). --spec register_hit_type(#mturk_hit{}) -> proplist() | no_return(). +-spec register_hit_type(#mturk_hit{}) -> proplist(). register_hit_type(HIT) -> register_hit_type(HIT, default_config()). --spec register_hit_type(#mturk_hit{}, aws_config()) -> proplist() | no_return(). +-spec register_hit_type(#mturk_hit{}, aws_config()) -> proplist(). register_hit_type(HIT, Config) -> Params = [ {"Title", HIT#mturk_hit.title}, @@ -1130,18 +1130,18 @@ register_hit_type(HIT, Config) -> Doc ). --spec reject_assignment(string()) -> ok | no_return(). +-spec reject_assignment(string()) -> ok. reject_assignment(AssignmentId) -> reject_assignment(AssignmentId, none). --spec reject_assignment(string(), string() | none | aws_config()) -> ok | no_return(). +-spec reject_assignment(string(), string() | none | aws_config()) -> ok. reject_assignment(AssignmentId, Config) when is_record(Config, aws_config) -> reject_assignment(AssignmentId, none, Config); reject_assignment(AssignmentId, Reason) -> reject_assignment(AssignmentId, Reason, default_config()). --spec reject_assignment(string(), string() | none, aws_config()) -> ok | no_return(). +-spec reject_assignment(string(), string() | none, aws_config()) -> ok. reject_assignment(AssignmentId, Reason, Config) when is_list(AssignmentId), is_list(Reason) orelse Reason =:= none -> @@ -1152,18 +1152,18 @@ reject_assignment(AssignmentId, Reason, Config) ] ). --spec reject_qualification_request(string()) -> ok | no_return(). +-spec reject_qualification_request(string()) -> ok. reject_qualification_request(QualificationRequestId) -> reject_qualification_request(QualificationRequestId, none). --spec reject_qualification_request(string(), string() | none | aws_config()) -> ok | no_return(). +-spec reject_qualification_request(string(), string() | none | aws_config()) -> ok. reject_qualification_request(QualificationRequestId, Config) when is_record(Config, aws_config) -> reject_qualification_request(QualificationRequestId, none, Config); reject_qualification_request(QualificationRequestId, Reason) -> reject_qualification_request(QualificationRequestId, Reason, default_config()). --spec reject_qualification_request(string(), string() | none, aws_config()) -> ok | no_return(). +-spec reject_qualification_request(string(), string() | none, aws_config()) -> ok. reject_qualification_request(QualificationRequestId, Reason, Config) when is_list(QualificationRequestId), is_list(Reason) orelse Reason =:= none -> @@ -1174,18 +1174,18 @@ reject_qualification_request(QualificationRequestId, Reason, Config) ] ). --spec revoke_qualification(string(), string()) -> ok | no_return(). +-spec revoke_qualification(string(), string()) -> ok. revoke_qualification(QualificationTypeId, WorkerId) -> revoke_qualification(QualificationTypeId, WorkerId, none). --spec revoke_qualification(string(), string(), string() | none | aws_config()) -> ok | no_return(). +-spec revoke_qualification(string(), string(), string() | none | aws_config()) -> ok. revoke_qualification(QualificationTypeId, WorkerId, Config) when is_record(Config, aws_config) -> revoke_qualification(QualificationTypeId, WorkerId, none, Config); revoke_qualification(QualificationTypeId, WorkerId, Reason) -> revoke_qualification(QualificationTypeId, WorkerId, Reason, default_config()). --spec revoke_qualification(string(), string(), string() | none, aws_config()) -> ok | no_return(). +-spec revoke_qualification(string(), string(), string() | none, aws_config()) -> ok. revoke_qualification(QualificationTypeId, WorkerId, Reason, Config) -> mturk_simple_request(Config, "RevokeQualification", [ @@ -1195,18 +1195,18 @@ revoke_qualification(QualificationTypeId, WorkerId, Reason, Config) -> ] ). --spec search_hits() -> proplist() | no_return(). +-spec search_hits() -> proplist(). search_hits() -> search_hits([]). --spec search_hits(proplist() | aws_config()) -> proplist() | no_return(). +-spec search_hits(proplist() | aws_config()) -> proplist(). search_hits(Config) when is_record(Config, aws_config) -> search_hits([], Config); search_hits(Options) -> search_hits(Options, default_config()). --spec search_hits(proplist(), aws_config()) -> proplist() | no_return(). +-spec search_hits(proplist(), aws_config()) -> proplist(). search_hits(Options, Config) when is_list(Options) -> Params = [ @@ -1241,18 +1241,18 @@ search_hits(Options, Config) Doc ). --spec search_qualification_types() -> proplist() | no_return(). +-spec search_qualification_types() -> proplist(). search_qualification_types() -> search_qualification_types([]). --spec search_qualification_types(proplist() | aws_config()) -> proplist() | no_return(). +-spec search_qualification_types(proplist() | aws_config()) -> proplist(). search_qualification_types(Config) when is_record(Config, aws_config) -> search_qualification_types([], Config); search_qualification_types(Options) -> search_qualification_types(Options, default_config()). --spec search_qualification_types(proplist(), aws_config()) -> proplist() | no_return(). +-spec search_qualification_types(proplist(), aws_config()) -> proplist(). search_qualification_types(Options, Config) -> Params = [ {"Query", proplists:get_value(search_query, Options)}, @@ -1285,11 +1285,11 @@ search_qualification_types(Options, Config) -> Doc ). --spec send_test_event_notification(proplist(), mturk_event_type()) -> ok | no_return(). +-spec send_test_event_notification(proplist(), mturk_event_type()) -> ok. send_test_event_notification(Notificaiton, TestEventType) -> send_test_event_notification(Notificaiton, TestEventType, default_config()). --spec send_test_event_notification(proplist(), mturk_event_type(), aws_config()) -> ok | no_return(). +-spec send_test_event_notification(proplist(), mturk_event_type(), aws_config()) -> ok. send_test_event_notification(Notification, TestEventType, Config) -> mturk_simple_request(Config, "SendTestEventNotification", [ @@ -1301,18 +1301,18 @@ send_test_event_notification(Notification, TestEventType, Config) -> ] ). --spec set_hit_as_reviewing(string()) -> ok | no_return(). +-spec set_hit_as_reviewing(string()) -> ok. set_hit_as_reviewing(HITId) -> set_hit_as_reviewing(HITId, false). --spec set_hit_as_reviewing(string(), boolean() | aws_config()) -> ok | no_return(). +-spec set_hit_as_reviewing(string(), boolean() | aws_config()) -> ok. set_hit_as_reviewing(HITId, Config) when is_record(Config, aws_config) -> set_hit_as_reviewing(HITId, false, Config); set_hit_as_reviewing(HITId, Revert) -> set_hit_as_reviewing(HITId, Revert, default_config()). --spec set_hit_as_reviewing(string(), boolean(), aws_config()) -> ok | no_return(). +-spec set_hit_as_reviewing(string(), boolean(), aws_config()) -> ok. set_hit_as_reviewing(HITId, Revert, Config) -> mturk_simple_request(Config, "SetHITAsReviewing", [ @@ -1321,18 +1321,18 @@ set_hit_as_reviewing(HITId, Revert, Config) -> ] ). --spec set_hit_type_notification(string(), proplist()) -> ok | no_return(). +-spec set_hit_type_notification(string(), proplist()) -> ok. set_hit_type_notification(HITTypeId, Notification) -> set_hit_type_notification(HITTypeId, Notification, undefined). --spec set_hit_type_notification(string(), proplist(), boolean() | undefined | aws_config()) -> ok | no_return(). +-spec set_hit_type_notification(string(), proplist(), boolean() | undefined | aws_config()) -> ok. set_hit_type_notification(HITTypeId, Notification, Config) when is_record(Config, aws_config) -> set_hit_type_notification(HITTypeId, Notification, undefined, Config); set_hit_type_notification(HITTypeId, Notification, Active) -> set_hit_type_notification(HITTypeId, Notification, Active, default_config()). --spec set_hit_type_notification(string(), proplist(), boolean() | undefined, aws_config()) -> ok | no_return(). +-spec set_hit_type_notification(string(), proplist(), boolean() | undefined, aws_config()) -> ok. set_hit_type_notification(HITTypeId, Notification, Active, Config) when is_list(HITTypeId), is_list(Notification), is_boolean(Active) orelse Active =:= undefined -> @@ -1358,29 +1358,29 @@ encode_transport(email) -> "Email"; encode_transport(soap) -> "SOAP"; encode_transport(rest) -> "REST". --spec unblock_worker(string()) -> ok | no_return(). +-spec unblock_worker(string()) -> ok. unblock_worker(WorkerId) -> unblock_worker(WorkerId, none). --spec unblock_worker(string(), string() | none | aws_config()) -> ok | no_return(). +-spec unblock_worker(string(), string() | none | aws_config()) -> ok. unblock_worker(WorkerId, Config) when is_record(Config, aws_config) -> unblock_worker(WorkerId, none, Config); unblock_worker(WorkerId, Reason) -> unblock_worker(WorkerId, Reason, default_config()). --spec unblock_worker(string(), string() | none, aws_config()) -> ok | no_return(). +-spec unblock_worker(string(), string() | none, aws_config()) -> ok. unblock_worker(WorkerId, Reason, Config) when is_list(WorkerId), is_list(Reason) orelse Reason =:= none -> mturk_simple_request(Config, "UnblockWorker", [{"WorkerId", WorkerId}, {"Reason", Reason}]). --spec update_qualification_score(string(), string(), integer()) -> ok | no_return(). +-spec update_qualification_score(string(), string(), integer()) -> ok. update_qualification_score(QualificationTypeId, SubjectId, IntegerValue) -> update_qualification_score(QualificationTypeId, SubjectId, IntegerValue, default_config()). --spec update_qualification_score(string(), string(), integer(), aws_config()) -> ok | no_return(). +-spec update_qualification_score(string(), string(), integer(), aws_config()) -> ok. update_qualification_score(QualificationTypeId, SubjectId, IntegerValue, Config) when is_list(SubjectId), is_list(QualificationTypeId), is_integer(IntegerValue) -> @@ -1392,11 +1392,11 @@ update_qualification_score(QualificationTypeId, SubjectId, IntegerValue, Config) ] ). --spec update_qualification_type(#mturk_qualification_type{}) -> #mturk_qualification_type{} | no_return(). +-spec update_qualification_type(#mturk_qualification_type{}) -> #mturk_qualification_type{}. update_qualification_type(QType) -> update_qualification_type(QType, default_config()). --spec update_qualification_type(#mturk_qualification_type{}, aws_config()) -> #mturk_qualification_type{} | no_return(). +-spec update_qualification_type(#mturk_qualification_type{}, aws_config()) -> #mturk_qualification_type{}. update_qualification_type(QType, Config) -> Doc = mturk_xml_request(Config, "UpdateQualificationType", [ diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 96336f6ae..d0d05228d 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -164,12 +164,12 @@ configure(AccessKeyID, SecretAccessKey, Host, Port, Scheme) -> -define(XMLNS_S3, "http://s3.amazonaws.com/doc/2006-03-01/"). -define(XMLNS_SCHEMA_INSTANCE, "http://www.w3.org/2001/XMLSchema-instance"). --spec copy_object(string(), string(), string(), string()) -> proplist() | no_return(). +-spec copy_object(string(), string(), string(), string()) -> proplist(). copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName) -> copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, []). --spec copy_object(string(), string(), string(), string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec copy_object(string(), string(), string(), string(), proplist() | aws_config()) -> proplist(). copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Config) when is_record(Config, aws_config) -> @@ -179,7 +179,7 @@ copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Options) -> copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Options, default_config()). --spec copy_object(string(), string(), string(), string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec copy_object(string(), string(), string(), string(), proplist(), aws_config()) -> proplist(). copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Options, Config) -> SrcVersion = case proplists:get_value(version_id, Options) of undefined -> ""; @@ -198,12 +198,12 @@ copy_object(DestBucketName, DestKeyName, SrcBucketName, SrcKeyName, Options, Con [{copy_source_version_id, proplists:get_value("x-amz-copy-source-version-id", Headers, "false")}, {version_id, proplists:get_value("x-amz-version-id", Headers, "null")}]. --spec create_bucket(string()) -> ok | no_return(). +-spec create_bucket(string()) -> ok. create_bucket(BucketName) -> create_bucket(BucketName, private). --spec create_bucket(string(), s3_bucket_acl() | aws_config()) -> ok | no_return(). +-spec create_bucket(string(), s3_bucket_acl() | aws_config()) -> ok. create_bucket(BucketName, Config) when is_record(Config, aws_config) -> @@ -212,7 +212,7 @@ create_bucket(BucketName, Config) create_bucket(BucketName, ACL) -> create_bucket(BucketName, ACL, none). --spec create_bucket(string(), s3_bucket_acl(), s3_location_constraint() | aws_config()) -> ok | no_return(). +-spec create_bucket(string(), s3_bucket_acl(), s3_location_constraint() | aws_config()) -> ok. create_bucket(BucketName, ACL, Config) when is_record(Config, aws_config) -> @@ -221,7 +221,7 @@ create_bucket(BucketName, ACL, Config) create_bucket(BucketName, ACL, LocationConstraint) -> create_bucket(BucketName, ACL, LocationConstraint, default_config()). --spec create_bucket(string(), s3_bucket_acl(), s3_location_constraint(), aws_config()) -> ok | no_return(). +-spec create_bucket(string(), s3_bucket_acl(), s3_location_constraint(), aws_config()) -> ok. create_bucket(BucketName, ACL, LocationConstraint, Config) when is_list(BucketName), is_atom(ACL), is_atom(LocationConstraint) -> @@ -269,12 +269,12 @@ encode_acl(authenticated_read) -> "authenticated-read"; encode_acl(bucket_owner_read) -> "bucket-owner-read"; encode_acl(bucket_owner_full_control) -> "bucket-owner-full-control". --spec delete_bucket(string()) -> ok | no_return(). +-spec delete_bucket(string()) -> ok. delete_bucket(BucketName) -> delete_bucket(BucketName, default_config()). --spec delete_bucket(string(), aws_config()) -> ok | no_return(). +-spec delete_bucket(string(), aws_config()) -> ok. delete_bucket(BucketName, Config) when is_list(BucketName) -> @@ -298,11 +298,11 @@ check_bucket_access(BucketName, Config) end. --spec delete_objects_batch(string(), list()) -> proplist() | no_return(). +-spec delete_objects_batch(string(), list()) -> proplist(). delete_objects_batch(BucketName, KeyList) -> delete_objects_batch(BucketName, KeyList, default_config()). --spec delete_objects_batch(string(), list(), aws_config()) -> proplist() | no_return(). +-spec delete_objects_batch(string(), list(), aws_config()) -> proplist(). delete_objects_batch(BucketName, KeyList, Config) when is_list(BucketName), is_list(KeyList) -> Data = lists:map(fun(Item) -> @@ -345,12 +345,12 @@ to_flat_format([{key,Key},{code,Code},{message,Message}]) -> %% "sailfish/deleteme/deep/ZZZ_0.txt"] %% ok %% --spec explore_dirstructure(string(), list(), list()) -> list() | no_return(). +-spec explore_dirstructure(string(), list(), list()) -> list(). explore_dirstructure(Bucketname, Branches, Accum) -> explore_dirstructure(Bucketname, Branches, Accum, default_config()). -spec explore_dirstructure(string(), list(), list(), aws_config()) -> - list() | no_return(). + list(). explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config) -> explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config, []). @@ -385,12 +385,12 @@ explore_dirstructure(Bucketname, [Branch|Tail], Accum, Config, Marker) explore_dirstructure(Bucketname, Tail, [SubFiles, TruncFiles, Files|Accum], Config, []). --spec delete_object(string(), string()) -> proplist() | no_return(). +-spec delete_object(string(), string()) -> proplist(). delete_object(BucketName, Key) -> delete_object(BucketName, Key, default_config()). --spec delete_object(string(), string(), aws_config()) -> proplist() | no_return(). +-spec delete_object(string(), string(), aws_config()) -> proplist(). delete_object(BucketName, Key, Config) when is_list(BucketName), is_list(Key) -> @@ -400,12 +400,12 @@ delete_object(BucketName, Key, Config) [{delete_marker, list_to_existing_atom(Marker)}, {version_id, Id}]. --spec delete_object_version(string(), string(), string()) -> proplist() | no_return(). +-spec delete_object_version(string(), string(), string()) -> proplist(). delete_object_version(BucketName, Key, Version) -> delete_object_version(BucketName, Key, Version, default_config()). --spec delete_object_version(string(), string(), string(), aws_config()) -> proplist() | no_return(). +-spec delete_object_version(string(), string(), string(), aws_config()) -> proplist(). delete_object_version(BucketName, Key, Version, Config) when is_list(BucketName), @@ -418,12 +418,12 @@ delete_object_version(BucketName, Key, Version, Config) [{delete_marker, list_to_existing_atom(Marker)}, {version_id, Id}]. --spec list_buckets() -> proplist() | no_return(). +-spec list_buckets() -> proplist(). list_buckets() -> list_buckets(default_config()). --spec list_buckets(aws_config()) -> proplist() | no_return(). +-spec list_buckets(aws_config()) -> proplist(). list_buckets(Config) -> Doc = s3_xml_request(Config, get, "", "/", "", [], <<>>, []), @@ -461,11 +461,11 @@ get_bucket_policy(BucketName, Config) Error end. --spec put_bucket_policy(string(), binary()) -> ok | no_return(). +-spec put_bucket_policy(string(), binary()) -> ok. put_bucket_policy(BucketName, Policy) -> put_bucket_policy(BucketName, Policy, default_config()). --spec put_bucket_policy(string(), binary(), aws_config()) -> ok | no_return(). +-spec put_bucket_policy(string(), binary(), aws_config()) -> ok. put_bucket_policy(BucketName, Policy, Config) when is_list(BucketName), is_binary(Policy), is_record(Config, aws_config) -> s3_simple_request(Config, put, BucketName, "/", "policy", [], Policy, []). @@ -484,11 +484,11 @@ get_bucket_lifecycle(BucketName, Config) Error end. --spec put_bucket_lifecycle(string(), list() | binary()) -> ok | {error, Reason::term()} | no_return(). +-spec put_bucket_lifecycle(string(), list() | binary()) -> ok | {error, Reason::term()}. put_bucket_lifecycle(BucketName, Policy) -> put_bucket_lifecycle(BucketName, Policy, default_config()). --spec put_bucket_lifecycle(string(), list() | binary(), aws_config()) -> ok | {error, Reason::term()} | no_return(). +-spec put_bucket_lifecycle(string(), list() | binary(), aws_config()) -> ok | {error, Reason::term()}. put_bucket_lifecycle(BucketName, Policy, Config) when is_list(BucketName), is_list(Policy), is_record(Config, aws_config) -> XmlPolicy = encode_lifecycle(Policy), @@ -499,21 +499,21 @@ put_bucket_lifecycle(BucketName, XmlPolicy, Config) s3_simple_request(Config, put, BucketName, "/", "lifecycle", [], XmlPolicy, [{"Content-MD5", Md5}]). --spec delete_bucket_lifecycle(string()) -> ok | {error, Reason::term()} | no_return(). +-spec delete_bucket_lifecycle(string()) -> ok | {error, Reason::term()}. delete_bucket_lifecycle(BucketName) -> delete_bucket_lifecycle(BucketName, default_config()). -spec delete_bucket_lifecycle(string(), #aws_config{}) - -> ok | {error, Reason::term()} | no_return(). + -> ok | {error, Reason::term()}. delete_bucket_lifecycle(BucketName, AwsConfig) -> s3_simple_request(AwsConfig, delete, BucketName, "/", "lifecycle", [], <<>>, []). --spec list_objects(string()) -> proplist() | no_return(). +-spec list_objects(string()) -> proplist(). list_objects(BucketName) -> list_objects(BucketName, []). --spec list_objects(string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec list_objects(string(), proplist() | aws_config()) -> proplist(). list_objects(BucketName, Config) when is_record(Config, aws_config) -> @@ -522,7 +522,7 @@ list_objects(BucketName, Config) list_objects(BucketName, Options) -> list_objects(BucketName, Options, default_config()). --spec list_objects(string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec list_objects(string(), proplist(), aws_config()) -> proplist(). list_objects(BucketName, Options, Config) when is_list(BucketName), @@ -566,12 +566,12 @@ extract_user([Node]) -> ], erlcloud_xml:decode(Attributes, Node). --spec get_bucket_attribute(string(), s3_bucket_attribute_name()) -> term() | no_return(). +-spec get_bucket_attribute(string(), s3_bucket_attribute_name()) -> term(). get_bucket_attribute(BucketName, AttributeName) -> get_bucket_attribute(BucketName, AttributeName, default_config()). --spec get_bucket_attribute(string(), s3_bucket_attribute_name(), aws_config()) -> term() | no_return(). +-spec get_bucket_attribute(string(), s3_bucket_attribute_name(), aws_config()) -> term(). get_bucket_attribute(BucketName, AttributeName, Config) when is_list(BucketName), is_atom(AttributeName) -> @@ -780,12 +780,12 @@ decode_permission("WRITE_ACP") -> write_acp; decode_permission("READ") -> read; decode_permission("READ_ACP") -> read_acp. --spec head_object(string(), string()) -> proplist() | no_return(). +-spec head_object(string(), string()) -> proplist(). head_object(BucketName, Key) -> head_object(BucketName, Key, []). --spec head_object(string(), string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec head_object(string(), string(), proplist() | aws_config()) -> proplist(). head_object(BucketName, Key, Config) when is_record(Config, aws_config) -> @@ -793,17 +793,17 @@ head_object(BucketName, Key, Config) head_object(BucketName, Key, Options) -> head_object(BucketName, Key, Options, default_config()). --spec head_object(string(), string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec head_object(string(), string(), proplist(), aws_config()) -> proplist(). head_object(BucketName, Key, Options, Config) -> get_or_head(head, BucketName, Key, Options, Config). --spec get_object(string(), string()) -> proplist() | no_return(). +-spec get_object(string(), string()) -> proplist(). get_object(BucketName, Key) -> get_object(BucketName, Key, []). --spec get_object(string(), string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec get_object(string(), string(), proplist() | aws_config()) -> proplist(). get_object(BucketName, Key, Config) when is_record(Config, aws_config) -> @@ -812,7 +812,7 @@ get_object(BucketName, Key, Config) get_object(BucketName, Key, Options) -> get_object(BucketName, Key, Options, default_config()). --spec get_object(string(), string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec get_object(string(), string(), proplist(), aws_config()) -> proplist(). get_object(BucketName, Key, Options, Config) -> get_or_head(get, BucketName, Key, Options, Config). @@ -842,12 +842,12 @@ get_or_head(Method, BucketName, Key, Options, Config) -> {content, Body}| extract_metadata(Headers)]. --spec get_object_acl(string(), string()) -> proplist() | no_return(). +-spec get_object_acl(string(), string()) -> proplist(). get_object_acl(BucketName, Key) -> get_object_acl(BucketName, Key, default_config()). --spec get_object_acl(string(), string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec get_object_acl(string(), string(), proplist() | aws_config()) -> proplist(). get_object_acl(BucketName, Key, Config) when is_record(Config, aws_config) -> @@ -856,7 +856,7 @@ get_object_acl(BucketName, Key, Config) get_object_acl(BucketName, Key, Options) -> get_object_acl(BucketName, Key, Options, default_config()). --spec get_object_acl(string(), string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec get_object_acl(string(), string(), proplist(), aws_config()) -> proplist(). get_object_acl(BucketName, Key, Options, Config) when is_list(BucketName), is_list(Key), is_list(Options) -> @@ -869,12 +869,12 @@ get_object_acl(BucketName, Key, Options, Config) {access_control_list, "AccessControlList/Grant", fun extract_acl/1}], erlcloud_xml:decode(Attributes, Doc). --spec get_object_metadata(string(), string()) -> proplist() | no_return(). +-spec get_object_metadata(string(), string()) -> proplist(). get_object_metadata(BucketName, Key) -> get_object_metadata(BucketName, Key, []). --spec get_object_metadata(string(), string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec get_object_metadata(string(), string(), proplist() | aws_config()) -> proplist(). get_object_metadata(BucketName, Key, Config) when is_record(Config, aws_config) -> @@ -883,7 +883,7 @@ get_object_metadata(BucketName, Key, Config) get_object_metadata(BucketName, Key, Options) -> get_object_metadata(BucketName, Key, Options, default_config()). --spec get_object_metadata(string(), string(), proplist(), proplist() | aws_config()) -> proplist() | no_return(). +-spec get_object_metadata(string(), string(), proplist(), proplist() | aws_config()) -> proplist(). get_object_metadata(BucketName, Key, Options, Config) -> RequestHeaders = [{"If-Modified-Since", proplists:get_value(if_modified_since, Options)}, @@ -914,12 +914,12 @@ decode_replication_status("COMPLETED") -> completed; decode_replication_status("FAILED") -> failed; decode_replication_status("REPLICA") -> replica. --spec get_object_torrent(string(), string()) -> proplist() | no_return(). +-spec get_object_torrent(string(), string()) -> proplist(). get_object_torrent(BucketName, Key) -> get_object_torrent(BucketName, Key, default_config()). --spec get_object_torrent(string(), string(), aws_config()) -> proplist() | no_return(). +-spec get_object_torrent(string(), string(), aws_config()) -> proplist(). get_object_torrent(BucketName, Key, Config) -> {Headers, Body} = s3_request(Config, get, BucketName, [$/|Key], "torrent", [], <<>>, []), @@ -927,12 +927,12 @@ get_object_torrent(BucketName, Key, Config) -> {version_id, proplists:get_value("x-amz-version-id", Headers, "null")}, {torrent, Body}]. --spec list_object_versions(string()) -> proplist() | no_return(). +-spec list_object_versions(string()) -> proplist(). list_object_versions(BucketName) -> list_object_versions(BucketName, []). --spec list_object_versions(string(), proplist() | aws_config()) -> proplist() | no_return(). +-spec list_object_versions(string(), proplist() | aws_config()) -> proplist(). list_object_versions(BucketName, Config) when is_record(Config, aws_config) -> @@ -941,7 +941,7 @@ list_object_versions(BucketName, Config) list_object_versions(BucketName, Options) -> list_object_versions(BucketName, Options, default_config()). --spec list_object_versions(string(), proplist(), aws_config()) -> proplist() | no_return(). +-spec list_object_versions(string(), proplist(), aws_config()) -> proplist(). list_object_versions(BucketName, Options, Config) when is_list(BucketName), is_list(Options) -> @@ -992,12 +992,12 @@ extract_bucket(Node) -> {creation_date, "CreationDate", time}], Node). --spec put_object(string(), string(), iodata()) -> proplist() | no_return(). +-spec put_object(string(), string(), iodata()) -> proplist(). put_object(BucketName, Key, Value) -> put_object(BucketName, Key, Value, []). --spec put_object(string(), string(), iodata(), proplist() | aws_config()) -> proplist() | no_return(). +-spec put_object(string(), string(), iodata(), proplist() | aws_config()) -> proplist(). put_object(BucketName, Key, Value, Config) when is_record(Config, aws_config) -> @@ -1006,7 +1006,7 @@ put_object(BucketName, Key, Value, Config) put_object(BucketName, Key, Value, Options) -> put_object(BucketName, Key, Value, Options, default_config()). --spec put_object(string(), string(), iodata(), proplist(), [{string(), string()}] | aws_config()) -> proplist() | no_return(). +-spec put_object(string(), string(), iodata(), proplist(), [{string(), string()}] | aws_config()) -> proplist(). put_object(BucketName, Key, Value, Options, Config) when is_record(Config, aws_config) -> @@ -1015,7 +1015,7 @@ put_object(BucketName, Key, Value, Options, Config) put_object(BucketName, Key, Value, Options, HTTPHeaders) -> put_object(BucketName, Key, Value, Options, HTTPHeaders, default_config()). --spec put_object(string(), string(), iodata(), proplist(), [{string(), string()}], aws_config()) -> proplist() | no_return(). +-spec put_object(string(), string(), iodata(), proplist(), [{string(), string()}], aws_config()) -> proplist(). put_object(BucketName, Key, Value, Options, HTTPHeaders, Config) when is_list(BucketName), is_list(Key), is_list(Value) orelse is_binary(Value), @@ -1030,12 +1030,12 @@ put_object(BucketName, Key, Value, Options, HTTPHeaders, Config) POSTData, RequestHeaders), [{version_id, proplists:get_value("x-amz-version-id", Headers, "null")} | Headers]. --spec set_object_acl(string(), string(), proplist()) -> ok | no_return(). +-spec set_object_acl(string(), string(), proplist()) -> ok. set_object_acl(BucketName, Key, ACL) -> set_object_acl(BucketName, Key, ACL, default_config()). --spec set_object_acl(string(), string(), proplist(), aws_config()) -> ok | no_return(). +-spec set_object_acl(string(), string(), proplist(), aws_config()) -> ok. set_object_acl(BucketName, Key, ACL, Config) when is_list(BucketName), is_list(Key), is_list(ACL) -> @@ -1238,12 +1238,12 @@ list_multipart_uploads(BucketName, Options, HTTPHeaders, Config) end. --spec set_bucket_attribute(string(), atom(), term()) -> ok | no_return(). +-spec set_bucket_attribute(string(), atom(), term()) -> ok. set_bucket_attribute(BucketName, AttributeName, Value) -> set_bucket_attribute(BucketName, AttributeName, Value, default_config()). --spec set_bucket_attribute(string(), atom(), term(), aws_config()) -> ok | no_return(). +-spec set_bucket_attribute(string(), atom(), term(), aws_config()) -> ok. set_bucket_attribute(BucketName, AttributeName, Value, Config) when is_list(BucketName) -> @@ -1513,14 +1513,14 @@ get_bucket_inventory(BucketName, InventoryId, #aws_config{} = Config) Error end. --spec put_bucket_inventory(string(), list()) -> ok | {error, Reason::term()} | no_return(). +-spec put_bucket_inventory(string(), list()) -> ok | {error, Reason::term()}. put_bucket_inventory(BucketName, Inventory) when is_list(BucketName), is_list(Inventory) -> put_bucket_inventory(BucketName, Inventory, default_config()). -spec put_bucket_inventory(string(), list(), aws_config()) - -> ok | {error, Reason::term()} | no_return(). + -> ok | {error, Reason::term()}. put_bucket_inventory(BucketName, Inventory, #aws_config{} = Config) when is_list(BucketName), is_list(Inventory) -> @@ -1529,7 +1529,7 @@ put_bucket_inventory(BucketName, Inventory, #aws_config{} = Config) put_bucket_inventory(BucketName, InventoryId, list_to_binary(XmlInventory), Config). -spec put_bucket_inventory(string(), string(), binary(), aws_config()) - -> ok | {error, Reason::term()} | no_return(). + -> ok | {error, Reason::term()}. put_bucket_inventory(BucketName, InventoryId, XmlInventory, #aws_config{} = Config) when is_list(BucketName), is_list(InventoryId), is_binary(XmlInventory) -> Md5 = base64:encode(crypto:hash(md5, XmlInventory)), @@ -1537,11 +1537,11 @@ put_bucket_inventory(BucketName, InventoryId, XmlInventory, #aws_config{} = Conf Headers = [{"Content-MD5", Md5}, {"content-type", "application/xml"}], s3_simple_request(Config, put, BucketName, "/", "inventory", Params, XmlInventory, Headers). --spec delete_bucket_inventory(string(), string()) -> ok | {error, Reason::term()} | no_return(). +-spec delete_bucket_inventory(string(), string()) -> ok | {error, Reason::term()}. delete_bucket_inventory(BucketName, InventoryId) when is_list(BucketName), is_list(InventoryId) -> delete_bucket_inventory(BucketName, InventoryId, default_config()). --spec delete_bucket_inventory(string(), string(), aws_config()) -> ok | {error, Reason::term()} | no_return(). +-spec delete_bucket_inventory(string(), string(), aws_config()) -> ok | {error, Reason::term()}. delete_bucket_inventory(BucketName, InventoryId, #aws_config{} = Config) when is_list(BucketName), is_list(InventoryId) -> diff --git a/src/erlcloud_sdb.erl b/src/erlcloud_sdb.erl index 5d88b2e29..d33fec9e1 100644 --- a/src/erlcloud_sdb.erl +++ b/src/erlcloud_sdb.erl @@ -59,29 +59,29 @@ configure(AccessKeyID, SecretAccessKey, Host) -> default_config() -> erlcloud_aws:default_config(). --spec create_domain(string()) -> proplist() | no_return(). +-spec create_domain(string()) -> proplist(). create_domain(Name) -> create_domain(Name, default_config()). --spec create_domain(string(), aws_config()) -> proplist() | no_return(). +-spec create_domain(string(), aws_config()) -> proplist(). create_domain(Name, Config) when is_list(Name) -> sdb_simple_request(Config, "CreateDomain", [{"DomainName", Name}]). --spec delete_domain(string()) -> proplist() | no_return(). +-spec delete_domain(string()) -> proplist(). delete_domain(Name) -> delete_domain(Name, default_config()). --spec delete_domain(string(), aws_config()) -> proplist() | no_return(). +-spec delete_domain(string(), aws_config()) -> proplist(). delete_domain(Name, Config) when is_list(Name) -> sdb_simple_request(Config, "DeleteDomain", [{"DomainName", Name}]). --spec domain_metadata(string()) -> proplist() | no_return(). +-spec domain_metadata(string()) -> proplist(). domain_metadata(Name) -> domain_metadata(Name, default_config()). --spec domain_metadata(string(), aws_config()) -> proplist() | no_return(). +-spec domain_metadata(string(), aws_config()) -> proplist(). domain_metadata(Name, Config) when is_list(Name) -> {Doc, Result} = sdb_request(Config, "DomainMetadata", [{"DomainName", Name}]), @@ -97,36 +97,36 @@ domain_metadata(Name, Config) ], MR), [{domain_metadata, Metadata}|Result]. --spec batch_put_attributes(string(), [{string(), sdb_attributes()}]) -> proplist() | no_return(). +-spec batch_put_attributes(string(), [{string(), sdb_attributes()}]) -> proplist(). batch_put_attributes(DomainName, Items) -> batch_put_attributes(DomainName, Items, default_config()). --spec batch_put_attributes(string(), [{string(), sdb_attributes()}], aws_config()) -> proplist() | no_return(). +-spec batch_put_attributes(string(), [{string(), sdb_attributes()}], aws_config()) -> proplist(). batch_put_attributes(DomainName, Items, Config) when is_list(DomainName), is_list(Items) -> ItemParams = [[{"ItemName", Name}|attributes_list(Attrs)] || {Name, Attrs} <- Items], sdb_simple_request(Config, "BatchPutAttributes", [{"DomainName", DomainName}|erlcloud_aws:param_list(ItemParams, "Item")]). --spec delete_attributes(string(), string()) -> proplist() | no_return(). +-spec delete_attributes(string(), string()) -> proplist(). delete_attributes(DomainName, ItemName) -> delete_attributes(DomainName, ItemName, []). --spec delete_attributes(string(), string(), sdb_delete_attributes() | aws_config()) -> proplist() | no_return(). +-spec delete_attributes(string(), string(), sdb_delete_attributes() | aws_config()) -> proplist(). delete_attributes(DomainName, ItemName, Config) when is_record(Config, aws_config) -> delete_attributes(DomainName, ItemName, [], Config); delete_attributes(DomainName, ItemName, Attributes) -> delete_attributes(DomainName, ItemName, Attributes, []). --spec delete_attributes(string(), string(), sdb_delete_attributes(), sdb_conditionals() | aws_config()) -> proplist() | no_return(). +-spec delete_attributes(string(), string(), sdb_delete_attributes(), sdb_conditionals() | aws_config()) -> proplist(). delete_attributes(DomainName, ItemName, Attributes, Config) when is_record(Config, aws_config) -> delete_attributes(DomainName, ItemName, Attributes, [], Config); delete_attributes(DomainName, ItemName, Attributes, Conditionals) -> delete_attributes(DomainName, ItemName, Attributes, Conditionals, default_config()). --spec delete_attributes(string(), string(), sdb_delete_attributes(), sdb_conditionals(), aws_config()) -> proplist() | no_return(). +-spec delete_attributes(string(), string(), sdb_delete_attributes(), sdb_conditionals(), aws_config()) -> proplist(). delete_attributes(DomainName, ItemName, Attributes, Conditionals, Config) when is_list(DomainName), is_list(ItemName), is_list(Attributes), is_list(Conditionals) -> @@ -134,11 +134,11 @@ delete_attributes(DomainName, ItemName, Attributes, Conditionals, Config) delete_attributes_list(Attributes)] ++ conditionals_list(Conditionals), sdb_simple_request(Config, "DeleteAttributes", Params). --spec get_attributes(string(), string()) -> proplist() | no_return(). +-spec get_attributes(string(), string()) -> proplist(). get_attributes(DomainName, ItemName) -> get_attributes(DomainName, ItemName, []). --spec get_attributes(string(), string(), [string()] | boolean() | aws_config()) -> proplist() | no_return(). +-spec get_attributes(string(), string(), [string()] | boolean() | aws_config()) -> proplist(). get_attributes(DomainName, ItemName, Config) when is_record(Config, aws_config) -> get_attributes(DomainName, ItemName, [], Config); @@ -148,7 +148,7 @@ get_attributes(DomainName, ItemName, ConsistentRead) get_attributes(DomainName, ItemName, AttributeNames) -> get_attributes(DomainName, ItemName, AttributeNames, false). --spec get_attributes(string(), string(), [string()], boolean() | aws_config()) -> proplist() | no_return(). +-spec get_attributes(string(), string(), [string()], boolean() | aws_config()) -> proplist(). get_attributes(DomainName, ItemName, AttributeNames, Config) when is_record(Config, aws_config) -> get_attributes(DomainName, ItemName, AttributeNames, false, Config); @@ -156,7 +156,7 @@ get_attributes(DomainName, ItemName, AttributeNames, ConsistentRead) -> get_attributes(DomainName, ItemName, AttributeNames, ConsistentRead, default_config()). --spec get_attributes(string(), string(), [string()], boolean(), aws_config()) -> proplist() | no_return(). +-spec get_attributes(string(), string(), [string()], boolean(), aws_config()) -> proplist(). get_attributes(DomainName, ItemName, AttributeNames, ConsistentRead, Config) -> {Doc, Result} = sdb_request(Config, "GetAttributes", [{"DomainName", DomainName}, {"ItemName", ItemName}, @@ -172,11 +172,11 @@ extract_attributes(Attributes) -> extract_attribute(Node) -> {erlcloud_xml:get_text("Name", Node), erlcloud_xml:get_text("Value", Node)}. --spec list_domains() -> proplist() | no_return(). +-spec list_domains() -> proplist(). list_domains() -> list_domains(default_config()). --spec list_domains(string() | 1..100 | none | aws_config()) -> proplist() | no_return(). +-spec list_domains(string() | 1..100 | none | aws_config()) -> proplist(). list_domains(Config) when is_record(Config, aws_config) -> list_domains("", Config); list_domains(MaxDomains) when is_integer(MaxDomains); MaxDomains =:= none -> @@ -184,7 +184,7 @@ list_domains(MaxDomains) when is_integer(MaxDomains); MaxDomains =:= none -> list_domains(FirstToken) -> list_domains(FirstToken, none). --spec list_domains(string(), 1..100 | none | aws_config()) -> proplist() | no_return(). +-spec list_domains(string(), 1..100 | none | aws_config()) -> proplist(). list_domains(FirstToken, Config) when is_record(Config, aws_config) -> list_domains(FirstToken, none, Config); list_domains(FirstToken, MaxDomains) -> @@ -200,13 +200,13 @@ maybe_add_nexttoken([], Params) -> maybe_add_nexttoken(Token, Params) -> [{"NextToken", Token} | Params]. --spec list_domains(string(), 1..100 | none, aws_config()) -> proplist() | no_return(). +-spec list_domains(string(), 1..100 | none, aws_config()) -> proplist(). list_domains(FirstToken, MaxDomains, Config) when is_list(FirstToken), is_integer(MaxDomains) orelse MaxDomains =:= none -> - Params = - maybe_add_nexttoken(FirstToken, + Params = + maybe_add_nexttoken(FirstToken, maybe_add_maxdomains(MaxDomains, [])), {Doc, Result} = sdb_request(Config, "ListDomains", Params), @@ -214,18 +214,18 @@ list_domains(FirstToken, MaxDomains, Config) [{domains, erlcloud_xml:get_list("/ListDomainsResponse/ListDomainsResult/DomainName", Doc)}, {next_token, erlcloud_xml:get_text("/ListDomainsResponse/ListDomainsResult/NextToken", Doc)}|Result]. --spec put_attributes(string(), string(), sdb_attributes()) -> proplist() | no_return(). +-spec put_attributes(string(), string(), sdb_attributes()) -> proplist(). put_attributes(DomainName, ItemName, Attributes) -> put_attributes(DomainName, ItemName, Attributes, []). --spec put_attributes(string(), string(), sdb_attributes(), sdb_conditionals() | aws_config()) -> proplist() | no_return(). +-spec put_attributes(string(), string(), sdb_attributes(), sdb_conditionals() | aws_config()) -> proplist(). put_attributes(DomainName, ItemName, Attributes, Config) when is_record(Config, aws_config) -> put_attributes(DomainName, ItemName, Attributes, [], Config); put_attributes(DomainName, ItemName, Attributes, Conditionals) -> put_attributes(DomainName, ItemName, Attributes, Conditionals, default_config()). --spec put_attributes(string(), string(), sdb_attributes(), sdb_conditionals(), aws_config()) -> proplist() | no_return(). +-spec put_attributes(string(), string(), sdb_attributes(), sdb_conditionals(), aws_config()) -> proplist(). put_attributes(DomainName, ItemName, Attributes, Conditionals, Config) when is_list(DomainName), is_list(ItemName), is_list(Attributes), is_list(Conditionals) -> @@ -236,10 +236,10 @@ put_attributes(DomainName, ItemName, Attributes, Conditionals, Config) %% These functions will return the first page of results along with %% a token to retrieve the next page, if any. --spec select(string()) -> proplist() | no_return(). +-spec select(string()) -> proplist(). select(SelectExpression) -> select(SelectExpression, none). --spec select(string(), string() | none | boolean() | aws_config()) -> proplist() | no_return(). +-spec select(string(), string() | none | boolean() | aws_config()) -> proplist(). select(SelectExpression, Config) when is_record(Config, aws_config) -> select(SelectExpression, none, Config); @@ -249,14 +249,14 @@ select(SelectExpression, ConsistentRead) select(SelectExpression, NextToken) -> select(SelectExpression, NextToken, false). --spec select(string(), string() | none, boolean() | aws_config()) -> proplist() | no_return(). +-spec select(string(), string() | none, boolean() | aws_config()) -> proplist(). select(SelectExpression, NextToken, Config) when is_record(Config, aws_config) -> select(SelectExpression, NextToken, false, Config); select(SelectExpression, NextToken, ConsistentRead) -> select(SelectExpression, NextToken, ConsistentRead, default_config()). --spec select(string(), string() | none, boolean(), aws_config()) -> proplist() | no_return(). +-spec select(string(), string() | none, boolean(), aws_config()) -> proplist(). select(SelectExpression, NextToken, ConsistentRead, Config) when is_list(SelectExpression), is_list(NextToken) orelse NextToken =:= none, @@ -274,24 +274,24 @@ select(SelectExpression, NextToken, ConsistentRead, Config) %% These functions will make multiple requests until all %% pages of results have been consumed. --spec select_all(string()) -> proplist() | no_return(). +-spec select_all(string()) -> proplist(). select_all(SelectExpression) -> select_all(SelectExpression, false). --spec select_all(string(), boolean()) -> proplist() | no_return(). +-spec select_all(string(), boolean()) -> proplist(). select_all(SelectExpression, ConsistentRead) when is_boolean(ConsistentRead) -> select_all(SelectExpression, ConsistentRead, default_config()); select_all(SelectExpression, Config) -> select_all(SelectExpression, false, Config). --spec select_all(string(), boolean(), aws_config()) -> proplist() | no_return(). +-spec select_all(string(), boolean(), aws_config()) -> proplist(). select_all(SelectExpression, ConsistentRead, Config) when is_list(SelectExpression), is_boolean(ConsistentRead) -> select_all(SelectExpression, none, ConsistentRead, Config, [], []). --spec select_all(string(), string() | none | done, boolean(), aws_config(), proplist(), proplist()) -> proplist() | no_return(). +-spec select_all(string(), string() | none | done, boolean(), aws_config(), proplist(), proplist()) -> proplist(). select_all(_, done, _, _, Items, Metadata) -> [{items, Items}|Metadata]; select_all(SelectExpression, NextToken, ConsistentRead, Config, Items, Metadata) -> diff --git a/src/erlcloud_sns.erl b/src/erlcloud_sns.erl index 092cce902..7e348bc5b 100644 --- a/src/erlcloud_sns.erl +++ b/src/erlcloud_sns.erl @@ -114,12 +114,12 @@ -export_type([sns_acl/0, sns_endpoint_attribute/0, sns_message_attributes/0, sns_message/0, sns_application/0, sns_endpoint/0]). --spec add_permission(string(), string(), sns_acl()) -> ok | no_return(). +-spec add_permission(string(), string(), sns_acl()) -> ok. add_permission(TopicArn, Label, Permissions) -> add_permission(TopicArn, Label, Permissions, default_config()). --spec add_permission(string(), string(), sns_acl(), aws_config()) -> ok | no_return(). +-spec add_permission(string(), string(), sns_acl(), aws_config()) -> ok. add_permission(TopicArn, Label, Permissions, Config) when is_list(TopicArn), is_list(Label), length(Label) =< 80, @@ -132,19 +132,19 @@ add_permission(TopicArn, Label, Permissions, Config) --spec create_platform_endpoint(string(), string()) -> string() | no_return(). +-spec create_platform_endpoint(string(), string()) -> string(). create_platform_endpoint(PlatformApplicationArn, Token) -> create_platform_endpoint(PlatformApplicationArn, Token, ""). --spec create_platform_endpoint(string(), string(), string()) -> string() | no_return(). +-spec create_platform_endpoint(string(), string(), string()) -> string(). create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData) -> create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, []). --spec create_platform_endpoint(string(), string(), string(), [{sns_endpoint_attribute(), string()}]) -> string() | no_return(). +-spec create_platform_endpoint(string(), string(), string(), [{sns_endpoint_attribute(), string()}]) -> string(). create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attributes) -> create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attributes, default_config()). --spec create_platform_endpoint(string(), string(), string(), [{sns_endpoint_attribute(), string()}], aws_config()) -> string() | no_return(). +-spec create_platform_endpoint(string(), string(), string(), [{sns_endpoint_attribute(), string()}], aws_config()) -> string(). create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attributes, Config) -> Doc = sns_xml_request( @@ -157,43 +157,43 @@ create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attribut erlcloud_xml:get_text( "CreatePlatformEndpointResult/EndpointArn", Doc). --spec create_platform_endpoint(string(), string(), string(), [{sns_endpoint_attribute(), string()}], string(), string()) -> string() | no_return(). +-spec create_platform_endpoint(string(), string(), string(), [{sns_endpoint_attribute(), string()}], string(), string()) -> string(). create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attributes, AccessKeyID, SecretAccessKey) -> create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attributes, new_config(AccessKeyID, SecretAccessKey)). --spec create_topic(string()) -> string() | no_return(). +-spec create_topic(string()) -> string(). create_topic(TopicName) -> create_topic(TopicName, default_config()). --spec create_topic(string(), aws_config()) -> string() | no_return(). +-spec create_topic(string(), aws_config()) -> string(). create_topic(TopicName, Config) when is_record(Config, aws_config) -> Doc = sns_xml_request(Config, "CreateTopic", [{"Name", TopicName}]), erlcloud_xml:get_text("/CreateTopicResponse/CreateTopicResult/TopicArn", Doc). --spec confirm_subscription(sns_event()) -> string() | no_return(). +-spec confirm_subscription(sns_event()) -> string(). confirm_subscription(SnsEvent) -> confirm_subscription(SnsEvent, default_config()). --spec confirm_subscription(sns_event(), aws_config()) -> string() | no_return(). +-spec confirm_subscription(sns_event(), aws_config()) -> string(). confirm_subscription(SnsEvent, Config) -> Token = binary_to_list(proplists:get_value(<<"Token">>, SnsEvent, <<>>)), TopicArn = binary_to_list(proplists:get_value(<<"TopicArn">>, SnsEvent, <<>>)), confirm_subscription2(Token, TopicArn, Config). --spec confirm_subscription(sns_event(), string(), string()) -> string() | no_return(). +-spec confirm_subscription(sns_event(), string(), string()) -> string(). confirm_subscription(SnsEvent, AccessKeyID, SecretAccessKey) -> confirm_subscription(SnsEvent, new_config(AccessKeyID, SecretAccessKey)). --spec confirm_subscription2(string(), string()) -> string() | no_return(). +-spec confirm_subscription2(string(), string()) -> string(). confirm_subscription2(Token, TopicArn) -> confirm_subscription2(Token, TopicArn, default_config()). --spec confirm_subscription2(string(), string(), aws_config()) -> string() | no_return(). +-spec confirm_subscription2(string(), string(), aws_config()) -> string(). confirm_subscription2(Token, TopicArn, Config) -> Doc = sns_xml_request( @@ -204,42 +204,42 @@ confirm_subscription2(Token, TopicArn, Config) -> erlcloud_xml:get_text( "ConfirmSubscriptionResult/SubscriptionArn", Doc). --spec confirm_subscription2(string(), string(), string(), string()) -> string() | no_return(). +-spec confirm_subscription2(string(), string(), string(), string()) -> string(). confirm_subscription2(Token, TopicArn, AccessKeyID, SecretAccessKey) -> confirm_subscription2(Token, TopicArn, new_config(AccessKeyID, SecretAccessKey)). --spec delete_endpoint(string()) -> ok | no_return(). +-spec delete_endpoint(string()) -> ok. delete_endpoint(EndpointArn) -> delete_endpoint(EndpointArn, default_config()). --spec delete_endpoint(string(), aws_config()) -> ok | no_return(). +-spec delete_endpoint(string(), aws_config()) -> ok. delete_endpoint(EndpointArn, Config) -> sns_simple_request(Config, "DeleteEndpoint", [{"EndpointArn", EndpointArn}]). --spec delete_endpoint(string(), string(), string()) -> ok | no_return(). +-spec delete_endpoint(string(), string(), string()) -> ok. delete_endpoint(EndpointArn, AccessKeyID, SecretAccessKey) -> delete_endpoint(EndpointArn, new_config(AccessKeyID, SecretAccessKey)). --spec delete_topic(string()) -> ok | no_return(). +-spec delete_topic(string()) -> ok. delete_topic(TopicArn) -> delete_topic(TopicArn, default_config()). --spec delete_topic(string(), aws_config()) -> ok | no_return(). +-spec delete_topic(string(), aws_config()) -> ok. delete_topic(TopicArn, Config) when is_record(Config, aws_config) -> sns_simple_request(Config, "DeleteTopic", [{"TopicArn", TopicArn}]). --spec get_endpoint_attributes(string()) -> sns_endpoint() | no_return(). +-spec get_endpoint_attributes(string()) -> sns_endpoint(). get_endpoint_attributes(EndpointArn) -> get_endpoint_attributes(EndpointArn, default_config()). --spec get_endpoint_attributes(string(), aws_config()) -> sns_endpoint() | no_return(). +-spec get_endpoint_attributes(string(), aws_config()) -> sns_endpoint(). get_endpoint_attributes(EndpointArn, Config) -> Params = [{"EndpointArn", EndpointArn}], Doc = sns_xml_request(Config, "GetEndpointAttributes", Params), @@ -251,16 +251,16 @@ get_endpoint_attributes(EndpointArn, Config) -> Doc), [{arn, EndpointArn} | Decoded]. --spec get_endpoint_attributes(string(), string(), string()) -> sns_endpoint() | no_return(). +-spec get_endpoint_attributes(string(), string(), string()) -> sns_endpoint(). get_endpoint_attributes(EndpointArn, AccessKeyID, SecretAccessKey) -> get_endpoint_attributes(EndpointArn, new_config(AccessKeyID, SecretAccessKey)). --spec set_endpoint_attributes(string(), [{sns_endpoint_attribute(), string()}]) -> string() | no_return(). +-spec set_endpoint_attributes(string(), [{sns_endpoint_attribute(), string()}]) -> string(). set_endpoint_attributes(EndpointArn, Attributes) -> set_endpoint_attributes(EndpointArn, Attributes, default_config()). --spec set_endpoint_attributes(string(), [{sns_endpoint_attribute(), string()}], aws_config()) -> string() | no_return(). +-spec set_endpoint_attributes(string(), [{sns_endpoint_attribute(), string()}], aws_config()) -> string(). set_endpoint_attributes(EndpointArn, Attributes, Config) -> Doc = sns_xml_request(Config, "SetEndpointAttributes", [{"EndpointArn", EndpointArn} | encode_attributes(Attributes)]), @@ -268,13 +268,13 @@ set_endpoint_attributes(EndpointArn, Attributes, Config) -> --spec list_endpoints_by_platform_application(string()) -> [{endpoints, [sns_endpoint()]} | {next_token, string()}] | no_return(). +-spec list_endpoints_by_platform_application(string()) -> [{endpoints, [sns_endpoint()]} | {next_token, string()}]. list_endpoints_by_platform_application(PlatformApplicationArn) -> list_endpoints_by_platform_application(PlatformApplicationArn, undefined). --spec list_endpoints_by_platform_application(string(), undefined|string()) -> [{endpoints, [sns_endpoint()]} | {next_token, string()}] | no_return(). +-spec list_endpoints_by_platform_application(string(), undefined|string()) -> [{endpoints, [sns_endpoint()]} | {next_token, string()}]. list_endpoints_by_platform_application(PlatformApplicationArn, NextToken) -> list_endpoints_by_platform_application(PlatformApplicationArn, NextToken, default_config()). --spec list_endpoints_by_platform_application(string(), undefined|string(), aws_config()) -> [{endpoints, [sns_endpoint()]} | {next_token, string()}] | no_return(). +-spec list_endpoints_by_platform_application(string(), undefined|string(), aws_config()) -> [{endpoints, [sns_endpoint()]} | {next_token, string()}]. list_endpoints_by_platform_application(PlatformApplicationArn, NextToken, Config) -> Params = case NextToken of @@ -290,21 +290,21 @@ list_endpoints_by_platform_application(PlatformApplicationArn, NextToken, Config {next_token, "ListEndpointsByPlatformApplicationResult/NextToken", text}], Doc), Decoded. --spec list_endpoints_by_platform_application(string(), undefined|string(), string(), string()) -> [{endpoints, [sns_endpoint()]} | {next_token, string()}] | no_return(). +-spec list_endpoints_by_platform_application(string(), undefined|string(), string(), string()) -> [{endpoints, [sns_endpoint()]} | {next_token, string()}]. list_endpoints_by_platform_application(PlatformApplicationArn, NextToken, AccessKeyID, SecretAccessKey) -> list_endpoints_by_platform_application(PlatformApplicationArn, NextToken, new_config(AccessKeyID, SecretAccessKey)). --spec list_topics() -> [{topics, [[{arn, string()}]]} | {next_token, string()}] | no_return(). +-spec list_topics() -> [{topics, [[{arn, string()}]]} | {next_token, string()}]. list_topics() -> list_topics(default_config()). --spec list_topics(undefined | string() | aws_config()) -> [{topics, [[{arn, string()}]]} | {next_token, string()}] | no_return(). +-spec list_topics(undefined | string() | aws_config()) -> [{topics, [[{arn, string()}]]} | {next_token, string()}]. list_topics(Config) when is_record(Config, aws_config) -> list_topics(undefined, Config); list_topics(NextToken) -> list_topics(NextToken, default_config()). --spec list_topics(undefined | string(), aws_config()) -> [{topics, [[{arn, string()}]]} | {next_token, string()}] | no_return(). +-spec list_topics(undefined | string(), aws_config()) -> [{topics, [[{arn, string()}]]} | {next_token, string()}]. list_topics(NextToken, Config) -> Params = case NextToken of @@ -321,11 +321,11 @@ list_topics(NextToken, Config) -> Doc), Decoded. --spec list_topics_all() -> [[{arn, string()}]] | no_return(). +-spec list_topics_all() -> [[{arn, string()}]]. list_topics_all() -> list_topics_all(default_config()). --spec list_topics_all(aws_config()) -> [[{arn, string()}]] | no_return(). +-spec list_topics_all(aws_config()) -> [[{arn, string()}]]. list_topics_all(Config) -> list_all(fun list_topics/2, topics, Config, undefined, []). @@ -370,18 +370,18 @@ list_subscriptions_all(Config) -> --spec list_subscriptions_by_topic(string()) -> proplist() | no_return(). +-spec list_subscriptions_by_topic(string()) -> proplist(). list_subscriptions_by_topic(TopicArn) when is_list(TopicArn) -> list_subscriptions_by_topic(TopicArn, default_config()). --spec list_subscriptions_by_topic(string(), string() | aws_config()) -> proplist() | no_return(). +-spec list_subscriptions_by_topic(string(), string() | aws_config()) -> proplist(). list_subscriptions_by_topic(TopicArn, Config) when is_record(Config, aws_config) -> list_subscriptions_by_topic(TopicArn, undefined, Config); list_subscriptions_by_topic(TopicArn, NextToken) when is_list(NextToken) -> list_subscriptions_by_topic(TopicArn, NextToken, default_config()). --spec list_subscriptions_by_topic(string(), undefined | string(), aws_config()) -> proplist() | no_return(). +-spec list_subscriptions_by_topic(string(), undefined | string(), aws_config()) -> proplist(). list_subscriptions_by_topic(TopicArn, NextToken, Config) when is_record(Config, aws_config) -> Params = case NextToken of @@ -398,11 +398,11 @@ list_subscriptions_by_topic(TopicArn, NextToken, Config) when is_record(Config, Doc), Decoded. --spec list_subscriptions_by_topic_all(string()) -> proplist() | no_return(). +-spec list_subscriptions_by_topic_all(string()) -> proplist(). list_subscriptions_by_topic_all(TopicArn) -> list_subscriptions_by_topic_all(TopicArn, default_config()). --spec list_subscriptions_by_topic_all(string(), aws_config()) -> proplist() | no_return(). +-spec list_subscriptions_by_topic_all(string(), aws_config()) -> proplist(). list_subscriptions_by_topic_all(TopicArn, Config) -> list_all(fun (Token, Cfg) -> list_subscriptions_by_topic(TopicArn, Token, Cfg) end, @@ -410,15 +410,15 @@ list_subscriptions_by_topic_all(TopicArn, Config) -> --spec list_platform_applications() -> [sns_application()] | no_return(). +-spec list_platform_applications() -> [sns_application()]. list_platform_applications() -> list_platform_applications(undefined). --spec list_platform_applications(undefined|string()) -> [sns_application()] | no_return(). +-spec list_platform_applications(undefined|string()) -> [sns_application()]. list_platform_applications(NextToken) -> list_platform_applications(NextToken, default_config()). --spec list_platform_applications(undefined|string(), aws_config()) -> [sns_application()] | no_return(). +-spec list_platform_applications(undefined|string(), aws_config()) -> [sns_application()]. list_platform_applications(NextToken, Config) -> Params = case NextToken of @@ -434,54 +434,54 @@ list_platform_applications(NextToken, Config) -> Doc), proplists:get_value(applications, Decoded, []). --spec list_platform_applications(undefined|string(), string(), string()) -> [sns_application()] | no_return(). +-spec list_platform_applications(undefined|string(), string(), string()) -> [sns_application()]. list_platform_applications(NextToken, AccessKeyID, SecretAccessKey) -> list_platform_applications(NextToken, new_config(AccessKeyID, SecretAccessKey)). --spec publish_to_topic(string(), sns_message()) -> string() | no_return(). +-spec publish_to_topic(string(), sns_message()) -> string(). publish_to_topic(TopicArn, Message) -> publish_to_topic(TopicArn, Message, undefined). --spec publish_to_topic(string(), sns_message(), undefined|string()) -> string() | no_return(). +-spec publish_to_topic(string(), sns_message(), undefined|string()) -> string(). publish_to_topic(TopicArn, Message, Subject) -> publish_to_topic(TopicArn, Message, Subject, default_config()). --spec publish_to_topic(string(), sns_message(), undefined|string(), aws_config()) -> string() | no_return(). +-spec publish_to_topic(string(), sns_message(), undefined|string(), aws_config()) -> string(). publish_to_topic(TopicArn, Message, Subject, Config) -> publish(topic, TopicArn, Message, Subject, Config). --spec publish_to_topic(string(), sns_message(), undefined|string(), string(), string()) -> string() | no_return(). +-spec publish_to_topic(string(), sns_message(), undefined|string(), string(), string()) -> string(). publish_to_topic(TopicArn, Message, Subject, AccessKeyID, SecretAccessKey) -> publish_to_topic(TopicArn, Message, Subject, new_config(AccessKeyID, SecretAccessKey)). --spec publish_to_target(string(), sns_message()) -> string() | no_return(). +-spec publish_to_target(string(), sns_message()) -> string(). publish_to_target(TargetArn, Message) -> publish_to_target(TargetArn, Message, undefined). --spec publish_to_target(string(), sns_message(), undefined|string()) -> string() | no_return(). +-spec publish_to_target(string(), sns_message(), undefined|string()) -> string(). publish_to_target(TargetArn, Message, Subject) -> publish_to_target(TargetArn, Message, Subject, default_config()). --spec publish_to_target(string(), sns_message(), undefined|string(), aws_config()) -> string() | no_return(). +-spec publish_to_target(string(), sns_message(), undefined|string(), aws_config()) -> string(). publish_to_target(TargetArn, Message, Subject, Config) -> publish(target, TargetArn, Message, Subject, Config). --spec publish_to_target(string(), sns_message(), undefined|string(), string(), string()) -> string() | no_return(). +-spec publish_to_target(string(), sns_message(), undefined|string(), string(), string()) -> string(). publish_to_target(TargetArn, Message, Subject, AccessKeyID, SecretAccessKey) -> publish_to_target(TargetArn, Message, Subject, new_config(AccessKeyID, SecretAccessKey)). %% TargetArn can be a phone number string, e.g. "+55 (11) 9999-7777" --spec publish_to_phone(string(), sns_message()) -> string() | no_return(). +-spec publish_to_phone(string(), sns_message()) -> string(). publish_to_phone(TargetArn, Message) -> publish_to_phone(TargetArn, Message, default_config()). --spec publish_to_phone(string(), sns_message(), aws_config()) -> string() | no_return(). +-spec publish_to_phone(string(), sns_message(), aws_config()) -> string(). publish_to_phone(TargetArn, Message, Config) -> publish(phone, TargetArn, Message, undefined, Config). --spec publish_to_phone(string(), sns_message(), string(), string()) -> string() | no_return(). +-spec publish_to_phone(string(), sns_message(), string(), string()) -> string(). publish_to_phone(TargetArn, Message, AccessKeyID, SecretAccessKey) -> publish(phone, TargetArn, Message, undefined, new_config(AccessKeyID, SecretAccessKey)). @@ -489,11 +489,11 @@ publish_to_phone(TargetArn, Message, AccessKeyID, SecretAccessKey) -> %% Publish API: %% [http://docs.aws.amazon.com/sns/latest/api/API_Publish.html] --spec publish(topic|target|phone, string(), sns_message(), undefined|string(), aws_config()) -> string() | no_return(). +-spec publish(topic|target|phone, string(), sns_message(), undefined|string(), aws_config()) -> string(). publish(Type, RecipientArn, Message, Subject, Config) -> publish(Type, RecipientArn, Message, Subject, [], Config). --spec publish(topic|target|phone, string(), sns_message(), undefined|string(), sns_message_attributes(), aws_config()) -> string() | no_return(). +-spec publish(topic|target|phone, string(), sns_message(), undefined|string(), sns_message_attributes(), aws_config()) -> string(). publish(Type, RecipientArn, Message, Subject, Attributes, Config) -> RecipientParam = case Type of @@ -555,11 +555,11 @@ get_notification_attribute(Attribute, Notification) -> --spec set_topic_attributes(sns_topic_attribute_name(), string()|binary(), string()) -> ok | no_return(). +-spec set_topic_attributes(sns_topic_attribute_name(), string()|binary(), string()) -> ok. set_topic_attributes(AttributeName, AttributeValue, TopicArn) -> set_topic_attributes(AttributeName, AttributeValue, TopicArn, default_config()). --spec set_topic_attributes(sns_topic_attribute_name(), string()|binary(), string(), aws_config()) -> ok | no_return(). +-spec set_topic_attributes(sns_topic_attribute_name(), string()|binary(), string(), aws_config()) -> ok. set_topic_attributes(AttributeName, AttributeValue, TopicArn, Config) when is_record(Config, aws_config) -> sns_simple_request(Config, "SetTopicAttributes", [ @@ -568,11 +568,11 @@ set_topic_attributes(AttributeName, AttributeValue, TopicArn, Config) {"TopicArn", TopicArn}]). --spec get_topic_attributes (string()) -> [{attributes, [{atom(), string()}]}] | no_return(). +-spec get_topic_attributes (string()) -> [{attributes, [{atom(), string()}]}]. get_topic_attributes(TopicArn) -> get_topic_attributes(TopicArn, default_config()). --spec get_topic_attributes(string(), aws_config()) -> [{attributes, [{atom(), string()}]}] | no_return(). +-spec get_topic_attributes(string(), aws_config()) -> [{attributes, [{atom(), string()}]}]. get_topic_attributes(TopicArn, Config) when is_record(Config, aws_config) -> Params = [{"TopicArn", TopicArn}], @@ -587,11 +587,11 @@ get_topic_attributes(TopicArn, Config) --spec set_subscription_attributes(sns_subscription_attribute_name(), string()|binary(), string()) -> ok | no_return(). +-spec set_subscription_attributes(sns_subscription_attribute_name(), string()|binary(), string()) -> ok. set_subscription_attributes(AttributeName, AttributeValue, SubscriptionArn) -> set_subscription_attributes(AttributeName, AttributeValue, SubscriptionArn, default_config()). --spec set_subscription_attributes(sns_subscription_attribute_name(), string()|binary(), string(), aws_config()) -> ok | no_return(). +-spec set_subscription_attributes(sns_subscription_attribute_name(), string()|binary(), string(), aws_config()) -> ok. set_subscription_attributes(AttributeName, AttributeValue, SubscriptionArn, Config) when is_record(Config, aws_config) -> sns_simple_request(Config, "SetSubscriptionAttributes", [ @@ -600,11 +600,11 @@ set_subscription_attributes(AttributeName, AttributeValue, SubscriptionArn, Conf {"SubscriptionArn", SubscriptionArn}]). --spec get_subscription_attributes (string()) -> [{attributes, [{sns_subscription_attribute_name() | atom(), string()}]}] | no_return(). +-spec get_subscription_attributes (string()) -> [{attributes, [{sns_subscription_attribute_name() | atom(), string()}]}]. get_subscription_attributes(SubscriptionArn) -> get_subscription_attributes(SubscriptionArn, default_config()). --spec get_subscription_attributes(string(), aws_config()) -> [{attributes, [{sns_subscription_attribute_name() | atom(), string()}]}] | no_return(). +-spec get_subscription_attributes(string(), aws_config()) -> [{attributes, [{sns_subscription_attribute_name() | atom(), string()}]}]. get_subscription_attributes(SubscriptionArn, Config) when is_record(Config, aws_config) -> Params = [{"SubscriptionArn", SubscriptionArn}], @@ -617,11 +617,11 @@ get_subscription_attributes(SubscriptionArn, Config) Doc), Decoded. --spec subscribe(string(), sns_subscribe_protocol_type(), string()) -> Arn::string() | no_return(). +-spec subscribe(string(), sns_subscribe_protocol_type(), string()) -> Arn::string(). subscribe(Endpoint, Protocol, TopicArn) -> subscribe(Endpoint, Protocol, TopicArn, default_config()). --spec subscribe(string(), sns_subscribe_protocol_type(), string(), aws_config()) -> Arn::string() | no_return(). +-spec subscribe(string(), sns_subscribe_protocol_type(), string(), aws_config()) -> Arn::string(). subscribe(Endpoint, Protocol, TopicArn, Config) when is_record(Config, aws_config) -> Doc = sns_xml_request(Config, "Subscribe", [ @@ -630,11 +630,11 @@ subscribe(Endpoint, Protocol, TopicArn, Config) {"TopicArn", TopicArn}]), erlcloud_xml:get_text("/SubscribeResponse/SubscribeResult/SubscriptionArn", Doc). --spec unsubscribe(string()) -> ok | no_return(). +-spec unsubscribe(string()) -> ok. unsubscribe(SubArn) -> unsubscribe(SubArn, default_config()). --spec unsubscribe(string(), aws_config()) -> ok | no_return(). +-spec unsubscribe(string(), aws_config()) -> ok. unsubscribe(SubArn, Config) when is_record(Config, aws_config) -> sns_simple_request(Config, "Unsubscribe", [{"SubscriptionArn", SubArn}]). diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index 5928934d9..d9a023733 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -75,11 +75,11 @@ configure(AccessKeyID, SecretAccessKey, Host) -> ok. --spec add_permission(string(), string(), sqs_acl()) -> ok | no_return(). +-spec add_permission(string(), string(), sqs_acl()) -> ok. add_permission(QueueName, Label, Permissions) -> add_permission(QueueName, Label, Permissions, default_config()). --spec add_permission(string(), string(), sqs_acl(), aws_config()) -> ok | no_return(). +-spec add_permission(string(), string(), sqs_acl(), aws_config()) -> ok. add_permission(QueueName, Label, Permissions, Config) when is_list(QueueName), is_list(Label), length(Label) =< 80, @@ -101,48 +101,48 @@ encode_permission({AccountId, Permission}) -> get_queue_attributes -> "GetQueueAttributes" end}. --spec change_message_visibility(string(), string(), 0..43200) -> ok | no_return(). +-spec change_message_visibility(string(), string(), 0..43200) -> ok. change_message_visibility(QueueName, ReceiptHandle, VisibilityTimeout) -> change_message_visibility(QueueName, ReceiptHandle, VisibilityTimeout, default_config()). --spec change_message_visibility(string(), string(), 0..43200, aws_config()) -> ok | no_return(). +-spec change_message_visibility(string(), string(), 0..43200, aws_config()) -> ok. change_message_visibility(QueueName, ReceiptHandle, VisibilityTimeout, Config) -> sqs_simple_request(Config, QueueName, "ChangeMessageVisibility", [{"ReceiptHandle", ReceiptHandle}, {"VisibilityTimeout", VisibilityTimeout}]). --spec create_queue(string()) -> proplist() | no_return(). +-spec create_queue(string()) -> proplist(). create_queue(QueueName) -> create_queue(QueueName, default_config()). --spec create_queue(string(), 0..43200 | none | aws_config()) -> proplist() | no_return(). +-spec create_queue(string(), 0..43200 | none | aws_config()) -> proplist(). create_queue(QueueName, Config) when is_record(Config, aws_config) -> create_queue(QueueName, none, Config); create_queue(QueueName, DefaultVisibilityTimeout) -> create_queue(QueueName, DefaultVisibilityTimeout, default_config()). --spec create_queue(string(), 0..43200 | none, aws_config()) -> proplist() | no_return(). +-spec create_queue(string(), 0..43200 | none, aws_config()) -> proplist(). create_queue(QueueName, DefaultVisibilityTimeout, Config) -> create_queue_impl(QueueName, DefaultVisibilityTimeout, Config, []). --spec create_fifo_queue(string()) -> proplist() | no_return(). +-spec create_fifo_queue(string()) -> proplist(). create_fifo_queue(QueueName) -> create_fifo_queue(QueueName, default_config()). --spec create_fifo_queue(string(), 0..43200 | none | aws_config()) -> proplist() | no_return(). +-spec create_fifo_queue(string(), 0..43200 | none | aws_config()) -> proplist(). create_fifo_queue(QueueName, Config) when is_record(Config, aws_config) -> create_fifo_queue(QueueName, none, Config); create_fifo_queue(QueueName, DefaultVisibilityTimeout) -> create_fifo_queue(QueueName, DefaultVisibilityTimeout, default_config()). --spec create_fifo_queue(string(), 0..43200 | none, aws_config()) -> proplist() | no_return(). +-spec create_fifo_queue(string(), 0..43200 | none, aws_config()) -> proplist(). create_fifo_queue(QueueName, DefaultVisibilityTimeout, Config) -> Attributes = erlcloud_aws:param_list([[{"Name", "FifoQueue"}, {"Value", true}]], "Attribute"), create_queue_impl(QueueName, DefaultVisibilityTimeout, Config, Attributes). --spec create_queue_impl(string(), 0..43200 | none, aws_config(), proplists:proplist()) -> proplist() | no_return(). +-spec create_queue_impl(string(), 0..43200 | none, aws_config(), proplists:proplist()) -> proplist(). create_queue_impl(QueueName, DefaultVisibilityTimeout, Config, Attributes) when is_list(QueueName), (is_integer(DefaultVisibilityTimeout) andalso @@ -159,39 +159,39 @@ create_queue_impl(QueueName, DefaultVisibilityTimeout, Config, Attributes) Doc ). --spec delete_message(string(), string()) -> ok | no_return(). +-spec delete_message(string(), string()) -> ok. delete_message(QueueName, ReceiptHandle) -> delete_message(QueueName, ReceiptHandle, default_config()). --spec delete_message(string(), string(), aws_config()) -> ok | no_return(). +-spec delete_message(string(), string(), aws_config()) -> ok. delete_message(QueueName, ReceiptHandle, Config) when is_list(QueueName), is_list(ReceiptHandle), is_record(Config, aws_config) -> sqs_simple_request(Config, QueueName, "DeleteMessage", [{"ReceiptHandle", ReceiptHandle}]). --spec delete_queue(string()) -> ok | no_return(). +-spec delete_queue(string()) -> ok. delete_queue(QueueName) -> delete_queue(QueueName, default_config()). --spec delete_queue(string(), aws_config()) -> ok | no_return(). +-spec delete_queue(string(), aws_config()) -> ok. delete_queue(QueueName, Config) when is_list(QueueName), is_record(Config, aws_config) -> sqs_simple_request(Config, QueueName, "DeleteQueue", []). --spec purge_queue(string()) -> ok | no_return(). +-spec purge_queue(string()) -> ok. purge_queue(QueueName) -> purge_queue(QueueName, default_config()). --spec purge_queue(string(), aws_config()) -> ok | no_return(). +-spec purge_queue(string(), aws_config()) -> ok. purge_queue(QueueName, Config) when is_list(QueueName), is_record(Config, aws_config) -> sqs_simple_request(Config, QueueName, "PurgeQueue", []). --spec get_queue_url(string()) -> proplist() | no_return(). +-spec get_queue_url(string()) -> proplist(). get_queue_url(QueueName) -> get_queue_url(QueueName, default_config()). --spec get_queue_url(string(), aws_config()) -> proplist() | no_return(). +-spec get_queue_url(string(), aws_config()) -> proplist(). get_queue_url(QueueName, Config) -> Doc = sqs_xml_request(Config, "/", "GetQueueUrl", [{"QueueName", QueueName}]), erlcloud_xml:decode( @@ -201,18 +201,18 @@ get_queue_url(QueueName, Config) -> Doc ). --spec get_queue_attributes(string()) -> proplist() | no_return(). +-spec get_queue_attributes(string()) -> proplist(). get_queue_attributes(QueueName) -> get_queue_attributes(QueueName, all). --spec get_queue_attributes(string(), all | [sqs_queue_attribute_name()] | aws_config()) -> proplist() | no_return(). +-spec get_queue_attributes(string(), all | [sqs_queue_attribute_name()] | aws_config()) -> proplist(). get_queue_attributes(QueueName, Config) when is_record(Config, aws_config) -> get_queue_attributes(QueueName, all, Config); get_queue_attributes(QueueName, AttributeNames) -> get_queue_attributes(QueueName, AttributeNames, default_config()). --spec get_queue_attributes(string(), all | [sqs_queue_attribute_name()], aws_config()) -> proplist() | no_return(). +-spec get_queue_attributes(string(), all | [sqs_queue_attribute_name()], aws_config()) -> proplist(). get_queue_attributes(QueueName, all, Config) when is_record(Config, aws_config) -> get_queue_attributes(QueueName, [all], Config); get_queue_attributes(QueueName, AttributeNames, Config) @@ -269,43 +269,43 @@ decode_attribute_value(_, "false") -> false; decode_attribute_value(_, Value) -> list_to_integer(Value). --spec list_queues() -> [string()] | no_return(). +-spec list_queues() -> [string()]. list_queues() -> list_queues(""). --spec list_queues(string() | aws_config()) -> [string()] | no_return(). +-spec list_queues(string() | aws_config()) -> [string()]. list_queues(Config) when is_record(Config, aws_config) -> list_queues("", Config); list_queues(QueueNamePrefix) -> list_queues(QueueNamePrefix, default_config()). --spec list_queues(string(), aws_config()) -> [string()] | no_return(). +-spec list_queues(string(), aws_config()) -> [string()]. list_queues(QueueNamePrefix, Config) when is_list(QueueNamePrefix), is_record(Config, aws_config) -> Doc = sqs_xml_request(Config, "/", "ListQueues", [{"QueueNamePrefix", QueueNamePrefix}]), erlcloud_xml:get_list("ListQueuesResult/QueueUrl", Doc). --spec receive_message(string()) -> proplist() | no_return(). +-spec receive_message(string()) -> proplist(). receive_message(QueueName) -> receive_message(QueueName, default_config()). --spec receive_message(string(), [sqs_msg_attribute_name()] | all | aws_config()) -> proplist() | no_return(). +-spec receive_message(string(), [sqs_msg_attribute_name()] | all | aws_config()) -> proplist(). receive_message(QueueName, Config) when is_record(Config, aws_config) -> receive_message(QueueName, [], Config); receive_message(QueueName, AttributeNames) -> receive_message(QueueName, AttributeNames, default_config()). --spec receive_message(string(), [sqs_msg_attribute_name()] | all, 1..10 | aws_config()) -> proplist() | no_return(). +-spec receive_message(string(), [sqs_msg_attribute_name()] | all, 1..10 | aws_config()) -> proplist(). receive_message(QueueName, AttributeNames, Config) when is_record(Config, aws_config) -> receive_message(QueueName, AttributeNames, 1, Config); receive_message(QueueName, AttributeNames, MaxNumberOfMessages) -> receive_message(QueueName, AttributeNames, MaxNumberOfMessages, default_config()). --spec receive_message(string(), [sqs_msg_attribute_name()] | all, 1..10, 0..43200 | none | aws_config()) -> proplist() | no_return(). +-spec receive_message(string(), [sqs_msg_attribute_name()] | all, 1..10, 0..43200 | none | aws_config()) -> proplist(). receive_message(QueueName, AttributeNames, MaxNumberOfMessages, Config) when is_record(Config, aws_config) -> receive_message(QueueName, AttributeNames, MaxNumberOfMessages, none, Config); @@ -314,7 +314,7 @@ receive_message(QueueName, AttributeNames, MaxNumberOfMessages, VisibilityTimeou VisibilityTimeout, default_config()). -spec receive_message(string(), [sqs_msg_attribute_name()] | all, 1..10, - 0..43200 | none, 0..20 | none | aws_config()) -> proplist() | no_return(). + 0..43200 | none, 0..20 | none | aws_config()) -> proplist(). receive_message(QueueName, AttributeNames, MaxNumberOfMessages, VisibilityTimeout, Config) when is_record(Config, aws_config) -> @@ -326,7 +326,7 @@ receive_message(QueueName, AttributeNames, MaxNumberOfMessages, VisibilityTimeout, WaitTimeSeconds, default_config()). -spec receive_message(string(), [sqs_msg_attribute_name()] | all, 1..10, - 0..43200 | none, 0..20 | none, aws_config()) -> proplist() | no_return(). + 0..43200 | none, 0..20 | none, aws_config()) -> proplist(). receive_message(QueueName, all, MaxNumberOfMessages, VisibilityTimeout, WaitTimeoutSeconds, Config) when is_record(Config, aws_config) -> receive_message(QueueName, [all], MaxNumberOfMessages, @@ -468,37 +468,37 @@ decode_attributes(Attrs) -> [{erlcloud_xml:get_text("Name", Attr), erlcloud_xml:get_text("Value", Attr)} || Attr <- Attrs]. --spec remove_permission(string(), string()) -> ok | no_return(). +-spec remove_permission(string(), string()) -> ok. remove_permission(QueueName, Label) -> remove_permission(QueueName, Label, default_config()). --spec remove_permission(string(), string(), aws_config()) -> ok | no_return(). +-spec remove_permission(string(), string(), aws_config()) -> ok. remove_permission(QueueName, Label, Config) when is_list(QueueName), is_list(Label), is_record(Config, aws_config) -> sqs_simple_request(Config, QueueName, "RemovePermission", [{"Label", Label}]). --spec send_message(string(), string()) -> proplist() | no_return(). +-spec send_message(string(), string()) -> proplist(). send_message(QueueName, MessageBody) -> send_message(QueueName, MessageBody, default_config()). --spec send_message(string(), string(), proplists:proplist() | 0..900 | none | aws_config()) -> proplist() | no_return(). +-spec send_message(string(), string(), proplists:proplist() | 0..900 | none | aws_config()) -> proplist(). send_message(QueueName, MessageBody, #aws_config{} = Config) -> send_message(QueueName, MessageBody, none, Config); -send_message(QueueName, MessageBody, DelaySeconds) +send_message(QueueName, MessageBody, DelaySeconds) when ((DelaySeconds >= 0 andalso DelaySeconds =< 900) orelse DelaySeconds =:= none) -> send_message(QueueName, MessageBody, DelaySeconds, default_config()); send_message(QueueName, MessageBody, Opts) when is_list(Opts) -> send_message(QueueName, MessageBody, Opts, default_config()). --spec send_message(string(), string(), proplists:proplist() | 0..900 | none, aws_config()) -> proplist() | no_return(). +-spec send_message(string(), string(), proplists:proplist() | 0..900 | none, aws_config()) -> proplist(). send_message(QueueName, MessageBody, DelaySeconds, Config) when ((DelaySeconds >= 0 andalso DelaySeconds =< 900) orelse DelaySeconds =:= none) -> send_message(QueueName, MessageBody, [{delay_seconds, DelaySeconds}], [], Config); send_message(QueueName, MessageBody, Opts, Config) when is_list(Opts) -> send_message(QueueName, MessageBody, Opts, [], Config). --spec send_message(string(), string(), proplists:proplist() | 0..900 | none, [message_attribute()], aws_config()) -> proplist() | no_return(). +-spec send_message(string(), string(), proplists:proplist() | 0..900 | none, [message_attribute()], aws_config()) -> proplist(). send_message(QueueName, MessageBody, DelaySeconds, MessageAttributes, #aws_config{}=Config) when ((DelaySeconds >= 0 andalso DelaySeconds =< 900) orelse DelaySeconds =:= none) -> send_message(QueueName, MessageBody, [{delay_seconds, DelaySeconds}], MessageAttributes, Config); @@ -516,11 +516,11 @@ send_message(QueueName, MessageBody, Opts, MessageAttributes, #aws_config{}=Conf Doc ). --spec set_queue_attributes(string(), proplists:proplist()) -> ok | no_return(). +-spec set_queue_attributes(string(), proplists:proplist()) -> ok. set_queue_attributes(QueueName, Attributes) -> set_queue_attributes(QueueName, Attributes, default_config()). --spec set_queue_attributes(string(), proplists:proplist(), aws_config()) -> ok | no_return(). +-spec set_queue_attributes(string(), proplists:proplist(), aws_config()) -> ok. set_queue_attributes(QueueName, Attributes, Config) when is_list(QueueName), is_list(Attributes), is_record(Config, aws_config) -> Params = erlcloud_aws:param_list([ @@ -528,18 +528,18 @@ set_queue_attributes(QueueName, Attributes, Config) {"Value", Value}] || {Name, Value} <- Attributes], "Attribute"), sqs_simple_request(Config, QueueName, "SetQueueAttributes", Params). --spec send_message_batch(string(), [batch_entry()]) -> proplist() | no_return(). +-spec send_message_batch(string(), [batch_entry()]) -> proplist(). send_message_batch(QueueName, BatchMessages) -> send_message_batch(QueueName, BatchMessages, default_config()). --spec send_message_batch(string(), [batch_entry()], 0..900 | none | aws_config()) -> proplist() | no_return(). +-spec send_message_batch(string(), [batch_entry()], 0..900 | none | aws_config()) -> proplist(). send_message_batch(QueueName, BatchMessages, Config) when is_record(Config, aws_config) -> send_message_batch(QueueName, BatchMessages, none, Config); send_message_batch(QueueName, BatchMessages, DelaySeconds) -> send_message_batch(QueueName, BatchMessages, DelaySeconds, default_config()). --spec send_message_batch(string(), [batch_entry()], 0..900 | none, aws_config()) -> proplist() | no_return(). +-spec send_message_batch(string(), [batch_entry()], 0..900 | none, aws_config()) -> proplist(). send_message_batch(QueueName, BatchMessages, DelaySeconds, Config) when is_list(QueueName), is_record(Config, aws_config), (DelaySeconds >= 0 andalso DelaySeconds =< 900) orelse @@ -558,11 +558,11 @@ send_message_batch(QueueName, BatchMessages, DelaySeconds, Config) erlcloud_xml:decode(BatchResponse, Doc). --spec delete_message_batch(string(), [batch_entry()]) -> proplists:proplist() | no_return(). +-spec delete_message_batch(string(), [batch_entry()]) -> proplists:proplist(). delete_message_batch(QueueName, BatchReceiptHandles) -> delete_message_batch(QueueName, BatchReceiptHandles, default_config()). --spec delete_message_batch(string(), [batch_entry()], aws_config()) -> proplists:proplist() | no_return(). +-spec delete_message_batch(string(), [batch_entry()], aws_config()) -> proplists:proplist(). delete_message_batch(QueueName, [{Id, Handle}|_]=BatchReceiptHandles, Config) when is_list(QueueName), is_list(Id), is_list(Handle), is_record(Config, aws_config) -> @@ -580,12 +580,12 @@ delete_message_batch(QueueName, [{Id, Handle}|_]=BatchReceiptHandles, Config) {failed, "DeleteMessageBatchResult/BatchResultErrorEntry", fun decode_batch_result_error/1}], erlcloud_xml:decode(BatchResponse, Doc). --spec change_message_visibility_batch(string(), [batch_entry()], 0..43200) -> proplists:proplist() | no_return(). +-spec change_message_visibility_batch(string(), [batch_entry()], 0..43200) -> proplists:proplist(). change_message_visibility_batch(QueueName, BatchReceiptHandles, VisibilityTimeout) -> change_message_visibility_batch(QueueName, BatchReceiptHandles, VisibilityTimeout, default_config()). --spec change_message_visibility_batch(string(), [batch_entry()], 0..43200, aws_config()) -> proplists:proplist() | no_return(). +-spec change_message_visibility_batch(string(), [batch_entry()], 0..43200, aws_config()) -> proplists:proplist(). change_message_visibility_batch(QueueName, [{Id, Handle}|_]=BatchReceiptHandles, VisibilityTimeout, Config) when is_list(QueueName), is_list(Id), is_list(Handle), is_record(Config, aws_config) -> diff --git a/src/erlcloud_sts.erl b/src/erlcloud_sts.erl index bd6e8c508..a115e6a02 100644 --- a/src/erlcloud_sts.erl +++ b/src/erlcloud_sts.erl @@ -23,7 +23,7 @@ assume_role(AwsConfig, RoleArn, RoleSessionName, DurationSeconds) -> % See http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html --spec assume_role(#aws_config{}, string(), string(), 900..43200, undefined | string()) -> {#aws_config{}, proplist()} | no_return(). +-spec assume_role(#aws_config{}, string(), string(), 900..43200, undefined | string()) -> {#aws_config{}, proplist()}. assume_role(AwsConfig, RoleArn, RoleSessionName, DurationSeconds, ExternalId) when length(RoleArn) >= 20, length(RoleSessionName) >= 2, length(RoleSessionName) =< 64, @@ -70,7 +70,7 @@ assume_role(AwsConfig, RoleArn, RoleSessionName, DurationSeconds, ExternalId) -type caller_identity_prop() :: {account, string()} | {arn, string()} | {userId, string()}. --spec get_caller_identity(#aws_config{}) -> {ok, [caller_identity_prop()]} | no_return(). +-spec get_caller_identity(#aws_config{}) -> {ok, [caller_identity_prop()]}. get_caller_identity(AwsConfig) -> Xml = sts_query(AwsConfig, "GetCallerIdentity", []), Proplists = erlcloud_xml:decode( @@ -86,7 +86,7 @@ get_federation_token(AwsConfig, DurationSeconds, Name) -> get_federation_token(AwsConfig, DurationSeconds, Name, undefined). % See http://docs.aws.amazon.com/STS/latest/APIReference/API_GetFederationToken.html --spec get_federation_token(#aws_config{}, 900..129600, string(), undefined | string()) -> {#aws_config{}, proplist()} | no_return(). +-spec get_federation_token(#aws_config{}, 900..129600, string(), undefined | string()) -> {#aws_config{}, proplist()}. get_federation_token(AwsConfig, DurationSeconds, Name, Policy) when length(Name) >= 2, length(Name) =< 32, DurationSeconds >= 900, DurationSeconds =< 129600 -> From 3a8b3b72947d558fa214b12a75810f6c4c785aee Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Sat, 8 Feb 2020 16:15:43 -0600 Subject: [PATCH 100/310] Upgrade eini to 1.2.7 Pin eini to [1.2.7](https://github.com/erlcloud/eini/releases/tag/1.2.7) to get https://github.com/erlcloud/eini/pull/17. --- rebar.config | 2 +- rebar.config.script | 2 +- rebar.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index 19535e886..bed306cef 100644 --- a/rebar.config +++ b/rebar.config @@ -19,7 +19,7 @@ {deps, [ {jsx, "2.9.0"}, {lhttpc, "1.6.2"}, - {eini, "1.2.6"}, + {eini, "1.2.7"}, {base16, "1.0.0"} ]}. diff --git a/rebar.config.script b/rebar.config.script index 8a278af28..d729da63c 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -8,7 +8,7 @@ case erlang:function_exported(rebar3, main, 1) of [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.13"}}}, {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "2.9.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, - {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.6"}}}, + {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.7"}}}, {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, {base16, ".*", {git, "https://github.com/goj/base16.git", {tag, "1.0.0"}}}]} | lists:keydelete(deps, 1, CONFIG)] diff --git a/rebar.lock b/rebar.lock index 4302de18f..ec2248c8e 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,12 +1,12 @@ {"1.1.0", [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, - {<<"eini">>,{pkg,<<"eini">>,<<"1.2.6">>},0}, + {<<"eini">>,{pkg,<<"eini">>,<<"1.2.7">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.9.0">>},0}, {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.6.2">>},0}]}. [ {pkg_hash,[ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, - {<<"eini">>, <<"DFFA48476FD89FB6E41CEEA0ADFA1BC6E7862CCD6584417442F8BB37E5D34715">>}, + {<<"eini">>, <<"EFC9D836E88591A47550BD34CE964E21CA1369F8716B24F73CFEA513FA99F666">>}, {<<"jsx">>, <<"D2F6E5F069C00266CAD52FB15D87C428579EA4D7D73A33669E12679E203329DD">>}, {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} ]. From 10ccd717a117151b9162301136c255402555f91b Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Mon, 17 Feb 2020 15:56:18 +0000 Subject: [PATCH 101/310] Fix all dialyzer warnings on erlcloud_util --- src/erlcloud_util.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index d8836a1d0..b33d3b38d 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -110,14 +110,14 @@ query_all(QueryFun, Config, Action, Params, MaxItems, Marker, Acc) -> end. -spec encode_list(string(), [term()]) -> - {ok, proplists:proplist()}. + proplists:proplist(). encode_list(ElementName, Elements) -> Numbered = lists:zip(lists:seq(1, length(Elements)), Elements), [{ElementName ++ ".member." ++ integer_to_list(N), Element} || {N, Element} <- Numbered]. -spec encode_object(string(), proplists:proplist()) -> - {ok, proplists:proplist()}. + proplists:proplist(). encode_object(ElementName, ElementParameters) -> lists:map( fun({Key, Value}) -> @@ -127,7 +127,7 @@ encode_object(ElementName, ElementParameters) -> ). -spec encode_object_list(string(), [proplists:proplist()]) -> - {ok, proplists:proplist()}. + proplists:proplist(). encode_object_list(Prefix, ElementParameterList) -> lists:flatten(lists:foldl( fun(ElementMap, Acc) -> From 8078b6e6deb1cb5b17f10037106fe900ddd46f81 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Mon, 17 Feb 2020 16:23:48 +0000 Subject: [PATCH 102/310] Fix all dialyzer warnings on erlcloud_application_autoscaler --- src/erlcloud_application_autoscaler.erl | 106 ++++++++++++++++-------- 1 file changed, 73 insertions(+), 33 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 567ad23a3..b05dba511 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -58,20 +58,20 @@ %% Type Definitions %% ------------------------------------------------------------------ --type response_attribute() :: binary(). +-type response_attribute() :: string() | integer(). -type response_key() :: atom(). --type response() :: [{response_key(), response_attribute()} | proplist:proplist()]. +-type response() :: [{response_key(), response_attribute()}]. -type aws_aas_request_body() :: [proplist:proplist()]. --spec extract_alarm(J :: binary()) -> response(). +-spec extract_alarm(J :: proplist()) -> response(). extract_alarm(J) -> erlcloud_json:decode([ {alarm_name, <<"AlarmName">>, optional_string}, {alarm_arn, <<"AlarmARN">>, optional_string} ], J). --spec extract_step_adjustments(J :: binary) -> response(). +-spec extract_step_adjustments(J :: proplist()) -> response(). extract_step_adjustments(J) -> erlcloud_json:decode([ {metric_interval_lower_bound, <<"MetricIntervalLowerBound">>, optional_integer}, @@ -79,7 +79,7 @@ extract_step_adjustments(J) -> {scaling_adjustment, <<"ScalingAdjustment">>, optional_integer} ], J). --spec extract_step_scaling_policy(J :: binary) -> response(). +-spec extract_step_scaling_policy(J :: proplist()) -> response(). extract_step_scaling_policy(J) -> Scaling = erlcloud_json:decode([ {adjustment_type, <<"AdjustmentType">>, optional_string}, @@ -94,21 +94,21 @@ extract_step_scaling_policy(J) -> [{step_adjustments, [ extract_step_adjustments(Step) || Step <- Steps]} | Scaling] end. --spec extract_dimensions(J :: binary) -> response(). +-spec extract_dimensions(J :: proplist()) -> response(). extract_dimensions(J) -> erlcloud_json:decode([ {name, <<"Name">>, optional_string}, {value, <<"Value">>, optional_string} ], J). --spec extract_predefined_metric_specifications(J :: binary) -> response(). +-spec extract_predefined_metric_specifications(J :: proplist()) -> response(). extract_predefined_metric_specifications(J) -> erlcloud_json:decode([ {predefined_metric_type, <<"PredefinedMetricType">>, optional_string}, {resource_label, <<"ResourceLabel">>, optional_string} ], J). --spec extract_customized_metric_specification(J :: binary) -> response(). +-spec extract_customized_metric_specification(J :: proplist()) -> response(). extract_customized_metric_specification(J) -> CustomizedMetricSpecification = erlcloud_json:decode([ {metric_name, <<"MetricName">>, optional_string}, @@ -124,7 +124,7 @@ extract_customized_metric_specification(J) -> end, CustomizedMetricSpecification ++ MaybeHasDimension. --spec extract_target_tracking_policy(J :: binary) -> response(). +-spec extract_target_tracking_policy(J :: proplist()) -> response(). extract_target_tracking_policy(J) -> Target = erlcloud_json:decode([ {disable_scale_in, <<"DisableScaleIn">>, optional_boolean}, @@ -146,7 +146,7 @@ extract_target_tracking_policy(J) -> end, Target ++ MaybeHasCustomizedMetrics ++ MaybeHasPredefinedMetrics. --spec extract_scaling_policies(J :: binary()) -> response(). +-spec extract_scaling_policies(J :: proplist()) -> response(). extract_scaling_policies(J) -> Policy = erlcloud_json:decode([ {creation_time, <<"CreationTime">>, optional_integer}, @@ -178,7 +178,7 @@ extract_scaling_policies(J) -> Policy ++ MaybeHasAlarms ++ MaybeHasStepScaling ++ MaybeHasTargetTracking. --spec extract_scalable_targets(J :: binary()) -> response(). +-spec extract_scalable_targets(J :: proplist()) -> response(). extract_scalable_targets(J) -> erlcloud_json:decode([ {creation_time, <<"CreationTime">>, optional_integer}, @@ -190,7 +190,7 @@ extract_scalable_targets(J) -> {service_namespace, <<"ServiceNamespace">>, optional_string} ], J). --spec extract_scaling_activities(J :: binary) -> response(). +-spec extract_scaling_activities(J :: proplist()) -> response(). extract_scaling_activities(J) -> erlcloud_json:decode([ {activity_id, <<"ActivityId">>, optional_string}, @@ -204,7 +204,7 @@ extract_scaling_activities(J) -> {status_message, <<"StatusMessage">>, optional_string} ], J). --spec extract_scheduled_action(J :: binary) -> response(). +-spec extract_scheduled_action(J :: proplist()) -> response(). extract_scheduled_action(J) -> Res = erlcloud_json:decode([ {creation_time, <<"CreationTime">>, optional_integer}, @@ -318,7 +318,9 @@ delete_scaling_policy(Configuration, BodyConfiguration) -> ScalableDimension :: binary(), ScheduledActionName :: binary(), ServiceNamespace :: binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. delete_scheduled_action(Configuration, ResourceId, ScalableDimension, ScheduledActionName, ServiceNamespace) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -329,7 +331,9 @@ delete_scheduled_action(Configuration, ResourceId, ScalableDimension, ScheduledA -spec delete_scheduled_action(Configuration :: aws_config(), BodyConfiguration :: aws_aas_request_body() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. delete_scheduled_action(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DeleteScheduledAction"). @@ -345,7 +349,9 @@ delete_scheduled_action(Configuration, BodyConfiguration) -> ResourceId :: binary(), ScalableDimension :: binary(), ServiceNamespace :: binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. deregister_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace) -> BodyProps = [{<<"ResourceId">>, ResourceId}, {<<"ScalableDimension">>, ScalableDimension}, @@ -355,7 +361,9 @@ deregister_scalable_target(Configuration, ResourceId, ScalableDimension, Service -spec deregister_scalable_target( Configuration :: erlcloud_aws:aws_config(), BodyConfiguration :: aws_aas_request_body() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. deregister_scalable_target(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DeregisterScalableTarget"). @@ -369,7 +377,9 @@ deregister_scalable_target(Configuration, BodyConfiguration) -> -spec describe_scalable_targets( erlcloud_aws:aws_config(), aws_aas_request_body() | binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. describe_scalable_targets(Configuration, ServiceNamespace) when is_binary(ServiceNamespace)-> describe_scalable_targets(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scalable_targets(Configuration, BodyConfiguration) -> @@ -397,7 +407,9 @@ describe_scalable_targets(Configuration, BodyConfiguration) -> -spec describe_scaling_activities( erlcloud_aws:aws_config(), aws_aas_request_body() | binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. describe_scaling_activities(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scaling_activities(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scaling_activities(Configuration, BodyConfiguration) -> @@ -426,7 +438,9 @@ describe_scaling_activities(Configuration, BodyConfiguration) -> -spec describe_scaling_policies( erlcloud_aws:aws_config(), aws_aas_request_body() | binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. describe_scaling_policies(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scaling_policies(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scaling_policies(Configuration, BodyConfiguration) -> @@ -455,7 +469,9 @@ describe_scaling_policies(Configuration, BodyConfiguration) -> -spec describe_scheduled_actions( erlcloud_aws:aws_config(), aws_aas_request_body() | binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. describe_scheduled_actions(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scheduled_actions(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scheduled_actions(Configuration, BodyConfiguration) -> @@ -487,7 +503,9 @@ describe_scheduled_actions(Configuration, BodyConfiguration) -> ResourceId :: binary(), ScalableDimension :: binary(), ServiceNamespace :: binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, ServiceNamespace) -> BodyProps = [{<<"PolicyName">>, PolicyName}, {<<"ResourceId">>, ResourceId}, @@ -503,7 +521,9 @@ put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, Ser ServiceNamespace :: binary(), PolicyType :: binary(), Policy :: [proplist:proplist()] - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, ServiceNamespace, PolicyType, Policy) -> BodyProps = [{<<"PolicyName">>, PolicyName}, {<<"ResourceId">>, ResourceId}, @@ -520,7 +540,9 @@ put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, Ser -spec put_scaling_policy( Configuration :: erlcloud_aws:aws_config(), BodyConfiguration :: aws_aas_request_body() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. put_scaling_policy(Configuration, BodyConfiguration) -> case request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.PutScalingPolicy") of {ok, Result} -> @@ -544,7 +566,9 @@ put_scaling_policy(Configuration, BodyConfiguration) -> ScalableDimension :: binary(), ServiceNamespace :: binary(), ScheduledActionName :: binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -559,7 +583,9 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp ServiceNamespace :: binary(), ScheduledActionName :: binary(), Schedule :: binary() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -576,7 +602,9 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp ScheduledActionName :: binary(), Schedule :: binary(), StartTime :: pos_integer() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule, StartTime) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -595,7 +623,9 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp Schedule :: binary(), StartTime :: pos_integer(), EndTime :: pos_integer() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule, StartTime, EndTime) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -608,7 +638,9 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp -spec put_scheduled_action(Configuration :: aws_config(), BodyConfiguration :: aws_aas_request_body() - ) -> response(). + ) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. put_scheduled_action(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.PutScheduledAction"). @@ -622,7 +654,9 @@ put_scheduled_action(Configuration, BodyConfiguration) -> -spec register_scalable_target(Configuration :: aws_config(), ResourceId :: binary(), ScalableDimension :: binary(), - ServiceNamespace :: binary()) -> response(). + ServiceNamespace :: binary()) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace) -> BodyProps = [{<<"ResourceId">>, ResourceId}, {<<"ScalableDimension">>, ScalableDimension}, @@ -633,7 +667,9 @@ register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNa ResourceId :: binary(), ScalableDimension :: binary(), ServiceNamespace :: binary(), - ResourceARN :: binary()) -> response(). + ResourceARN :: binary()) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ResourceARN) -> BodyProps = [{<<"ResourceId">>, ResourceId}, {<<"ScalableDimension">>, ScalableDimension}, @@ -647,7 +683,9 @@ register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNa ServiceNamespace :: binary(), ResourceARN :: binary(), MinCapacity :: integer() | undefined, - MaxCapacity :: integer() | undefined) -> response(). + MaxCapacity :: integer() | undefined) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ResourceARN, MinCapacity, MaxCapacity) -> BodyProps = [{<<"ResourceId">>, ResourceId}, {<<"ScalableDimension">>, ScalableDimension}, @@ -666,7 +704,9 @@ register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNa -spec register_scalable_target( Configuration :: erlcloud_aws:aws_config(), BodyConfiguration :: aws_aas_request_body() -) -> response(). +) -> {ok, jsx:json_term()} | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. register_scalable_target(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.RegisterScalableTarget"). From c4fbc3a0c77e3190b67d7305b6a856a3f9724fb2 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 10:07:43 +0000 Subject: [PATCH 103/310] Fix some dialyzer warnings (those we can) on erlcloud_aws Calls to `application:unset_env` and `application:get_env` with a non-atom() Par make use of undocumented (albeit working) application: calls --- src/erlcloud_aws.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 3aaed9936..d07e5777e 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -549,7 +549,7 @@ config_env() -> _ -> {error, environment_config_unavailable} end. --spec config_metadata(task_credentials | instance_metadata) -> {ok, #metadata_credentials{}} | {error, metadata_not_available | container_credentials_unavailable | httpc_result_error()}. +-spec config_metadata(task_credentials | instance_metadata) -> {ok, aws_config()} | {error, metadata_not_available | container_credentials_unavailable | httpc_result_error()}. config_metadata(Source) -> Config = #aws_config{}, case get_metadata_credentials( Source, Config ) of From 82b2b25624fa138ab5b18522e8a4ada01b6465b9 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 10:11:08 +0000 Subject: [PATCH 104/310] Fix all dialyzer warnings on erlcloud_cloudformation This function is only ever used internally and always with a #cloudformation_rollback_configuration{} record --- src/erlcloud_cloudformation.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/erlcloud_cloudformation.erl b/src/erlcloud_cloudformation.erl index 24592c733..3b80caad3 100644 --- a/src/erlcloud_cloudformation.erl +++ b/src/erlcloud_cloudformation.erl @@ -841,9 +841,7 @@ cloudformation_rollback_configuration_fields(#cloudformation_rollback_configurat RollbackConfig#cloudformation_rollback_configuration.rollback_triggers ) ) - ]); -cloudformation_rollback_configuration_fields(_) -> - []. + ]). cloudformation_rollback_triggers_fields(RollbackTriggers) -> lists:map(fun cloudformation_rollback_trigger_fields/1, RollbackTriggers). From 288c58d9c79604a8c9f0235d3fa800688c00f8c5 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 10:16:38 +0000 Subject: [PATCH 105/310] Fix all dialyzer warnings on erlcloud_cloudwatch_logs --- src/erlcloud_cloudwatch_logs.erl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index f37478482..42086860b 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -42,7 +42,10 @@ -type metric_filters() :: jsx:json_term(). -type tag():: {binary(), binary()}. --type tags_return() :: jsx:json_term(). +-type tags_return() :: {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()} + | {ok, jsx:json_term()}. %% Library initialization @@ -483,7 +486,9 @@ list_tags_log_group(LogGroup, Config) -> -spec tag_log_group( log_group_name(), list(tag()) -) -> ok. +) -> {ok, []} + | {ok, jsx:json_term()} + | {error, erlcloud_aws:httpc_result_error()}. tag_log_group(LogGroup, Tags) when is_list(Tags) -> tag_log_group(LogGroup, Tags, default_config()). @@ -493,7 +498,9 @@ tag_log_group(LogGroup, Tags) when is_list(Tags) -> log_group_name(), list(tag()), aws_config() -) -> ok. +) -> {ok, []} + | {ok, jsx:json_term()} + | {error, erlcloud_aws:httpc_result_error()}. tag_log_group(LogGroup, Tags, Config) when is_list(Tags) -> Params = [{<<"logGroupName">>, LogGroup}, From 078fae23723ccfb3816f8a84611bf2a9b5e39526 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 11:09:57 +0000 Subject: [PATCH 106/310] Fix all dialyzer warnings on erlcloud_ddb2 out/5 never returns {simple, _} which is why it was removed from the case in transact_write_items/3 erlcloud_aws:get_timeout/1 already handles the `undefined` case which is why the code never matched undefined in maybe_update_config_timeout/2's case --- src/erlcloud_ddb2.erl | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index cf72bcc2f..392e7027b 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -3446,7 +3446,8 @@ tag_resource(ResourceArn, Tags, Config) -> projection_expression_opt(). -type transact_get_items_opts() :: [transact_get_items_transact_item_opts()]. --type transact_get_items_get_item() :: {table_name(), key(), transact_get_items_transact_item_opts()}. +-type transact_get_items_get_item() :: {table_name(), key()} + | {table_name(), key(), transact_get_items_opts()}. -type transact_get_items_get() :: {get, transact_get_items_get_item()}. @@ -3456,7 +3457,7 @@ tag_resource(ResourceArn, Tags, Config) -> -type transact_get_items_return() :: ddb_return(#ddb2_transact_get_items{}, out_item()). -spec dynamize_transact_get_items_transact_items(transact_write_items_transact_items()) - -> json_pair(). + -> [jsx:json_term()]. dynamize_transact_get_items_transact_items(TransactItems) -> dynamize_maybe_list(fun dynamize_transact_get_items_transact_item/1, TransactItems). @@ -3605,7 +3606,7 @@ dynamize_transact_write_items_transact_item({update, {TableName, Key, UpdateExpr [{<<"Update">>, [{<<"TableName">>, TableName}, {<<"Key">>, dynamize_key(Key)}, {<<"UpdateExpression">>, dynamize_expression(UpdateExpression)} | AwsOpts]}]. -spec dynamize_transact_write_items_transact_items(transact_write_items_transact_items()) - -> json_pair(). + -> [jsx:json_term()]. dynamize_transact_write_items_transact_items(TransactItems) -> dynamize_maybe_list(fun dynamize_transact_write_items_transact_item/1, TransactItems). @@ -3677,7 +3678,6 @@ transact_write_items(TransactItems, Opts, Config) -> case out(Return, fun(Json, UOpts) -> undynamize_record(transact_write_items_record(), Json, UOpts) end, DdbOpts, #ddb2_transact_write_items.attributes, {ok, []}) of - {simple, Record} -> {ok, Record}; {ok, _} = Out -> Out; {error, _} = Out -> Out end. @@ -4338,11 +4338,8 @@ dynamize_replica_auto_scaling_updates(ReplicaUpdates) -> Update) || Update <- ReplicaUpdates]. -spec maybe_update_config_timeout(aws_config(), MinDesiredTimeout :: pos_integer()) -> aws_config(). -maybe_update_config_timeout(Config, MinDesiredTimeout) -> - UpdatedTimeout = case erlcloud_aws:get_timeout(Config) of - undefined -> MinDesiredTimeout; - ProvidedTimeout -> ProvidedTimeout - end, +maybe_update_config_timeout(Config, _MinDesiredTimeout) -> + UpdatedTimeout = erlcloud_aws:get_timeout(Config), Config#aws_config{timeout = UpdatedTimeout}. -type update_table_replica_auto_scaling_opt() :: {global_secondary_index_updates, [global_secondary_index_auto_scaling_update_opts()]}| From 1d3c0f5b83ae1227ce39e0ec4d076dcce9e0b05e Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 11:12:43 +0000 Subject: [PATCH 107/310] Fix all dialyzer warnings on erlcloud_ddb_util --- src/erlcloud_ddb_util.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index 415e87eec..24a22da64 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -513,7 +513,7 @@ batch_write_retry(RequestItems, Config) -> %% @end %%------------------------------------------------------------------------------ --spec wait_for_table_active(table_name(), pos_integer() | infinity, non_neg_integer(), aws_config()) -> +-spec wait_for_table_active(table_name(), pos_integer() | infinity, non_neg_integer() | infinity, aws_config()) -> ok | {error, deleting | retry_threshold_exceeded | any()}. wait_for_table_active(Table, Interval, RetryTimes, Config) when is_binary(Table), Interval > 0, RetryTimes >= 0 -> case erlcloud_ddb2:describe_table(Table, [{out, record}], Config) of From 54cffd4328d374f9cb4f0799b72c2780bf7ababb Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 11:25:27 +0000 Subject: [PATCH 108/310] Fix all dialyzer warnings on erlcloud_elb describe_load_balancers_all/2 calls describe_all/4 describe_all/4 calls fun(Marker, AwsConfig) where Marker is none fun(Marker, AwsConfig) calls describe_load_balancers/4 where Marker is none and this function only ever returns {ok, _} | {{paged, _}, _} through erlcloud_util:next_token/4 and extract_elb/1 --- src/erlcloud_elb.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/erlcloud_elb.erl b/src/erlcloud_elb.erl index 3f65ff540..bdd532e5d 100644 --- a/src/erlcloud_elb.erl +++ b/src/erlcloud_elb.erl @@ -540,8 +540,6 @@ describe_all(Fun, AwsConfig, Marker, Acc) -> case Fun(Marker, AwsConfig) of {ok, Res} -> {ok, lists:append(Acc, Res)}; - {ok, Res, NewMarker} -> - describe_all(Fun, AwsConfig, NewMarker, lists:append(Acc, Res)); {{paged, NewMarker}, Res} -> describe_all(Fun, AwsConfig, NewMarker, lists:append(Acc, Res)); {error, Reason} -> From eb0e45aad3b3eb3144b17a505f041c2b59794d11 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 11:30:11 +0000 Subject: [PATCH 109/310] Fix all dialyzer warnings on erlcloud_guardduty encode_body/1 is only ever called locally via guardduty_request_no_update/5 guardduty_request_no_update/5 is only ever called locally via guardduty_request/5 guardduty_request/5 is only ever called locally with Body =:= undefined (maybe it was intended, initially, to cover more use cases, but undefined ended up being the only one present) --- src/erlcloud_guardduty.erl | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/erlcloud_guardduty.erl b/src/erlcloud_guardduty.erl index 3f5d8e602..f3eb6a75c 100644 --- a/src/erlcloud_guardduty.erl +++ b/src/erlcloud_guardduty.erl @@ -82,8 +82,8 @@ list_detectors(Config) -> list_detectors(Marker, MaxItems) -> list_detectors(Marker, MaxItems, default_config()). --spec list_detectors(Marker :: binary(), - MaxItems :: integer(), +-spec list_detectors(Marker :: undefined | binary(), + MaxItems :: undefined | integer(), Config :: aws_config()) -> gd_return(). list_detectors(Marker, MaxItems, Config) -> Path = "/detector", @@ -123,11 +123,7 @@ guardduty_request_no_update(Config, Method, Path, Body, QParam) -> end. encode_body(undefined) -> - <<>>; -encode_body([]) -> - <<"{}">>; -encode_body(Body) -> - jsx:encode(Body). + <<>>. headers(Method, Uri, Config, Body, QParam) -> Headers = [{"host", Config#aws_config.guardduty_host}, From ab6033218f4dd6cf00746c38dfd2f7e61976f5d8 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 11:50:03 +0000 Subject: [PATCH 110/310] Fix all dialyzer warnings on erlcloud_iam --- src/erlcloud_iam.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index cd0871743..6a6f8296e 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -786,7 +786,7 @@ list_virtual_mfa_devices(AssignmentStatus, Marker, #aws_config{} = Config) -> list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems) -> list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems, default_config()). --spec list_virtual_mfa_devices(string(), string(), string(), aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +-spec list_virtual_mfa_devices(undefined | string(), undefined | string(), undefined | string(), aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems, #aws_config{} = Config) -> Params = make_list_virtual_mfa_devices_params(AssignmentStatus, Marker, MaxItems), ItemPath = "/ListVirtualMFADevicesResponse/ListVirtualMFADevicesResult/VirtualMFADevices/member", @@ -803,7 +803,7 @@ list_virtual_mfa_devices_all(#aws_config{} = Config) -> list_virtual_mfa_devices_all(AssignmentStatus) -> list_virtual_mfa_devices_all(AssignmentStatus, default_config()). --spec list_virtual_mfa_devices_all(string(), aws_config()) -> {ok, proplist()} | {error, any()}. +-spec list_virtual_mfa_devices_all(undefined | string(), aws_config()) -> {ok, proplist()} | {error, any()}. list_virtual_mfa_devices_all(AssignmentStatus, #aws_config{} = Config) -> Params = make_list_virtual_mfa_devices_params(AssignmentStatus, undefined, undefined), ItemPath = "/ListVirtualMFADevicesResponse/ListVirtualMFADevicesResult/VirtualMFADevices/member", From 63163ee643378e9ac8a70ec3e95fa68ba2ce399e Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 11:51:46 +0000 Subject: [PATCH 111/310] Fix all dialyzer warnings on erlcloud_states --- src/erlcloud_states.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_states.erl b/src/erlcloud_states.erl index 908cbe279..3c00873f3 100644 --- a/src/erlcloud_states.erl +++ b/src/erlcloud_states.erl @@ -539,7 +539,7 @@ start_execution(StateMachineArn, Options) -> start_execution(StateMachineArn, Options, default_config()). -spec start_execution(StateMachineArn :: binary(), - Options :: list(), + Options :: list() | map(), Config :: aws_config()) -> {ok, map()} | {error, any()}. start_execution(StateMachineArn, Options, Config) @@ -573,7 +573,7 @@ stop_execution(ExecutionArn, Options) -> stop_execution(ExecutionArn, Options, default_config()). -spec stop_execution(ExecutionArn :: binary(), - Options :: list(), + Options :: list() | map(), Config :: aws_config()) -> {ok, map()} | {error, any()}. stop_execution(ExecutionArn, Options, Config) From 1feeff5530e73e7ee8f8f509263c9eea69883c08 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 14:18:54 +0000 Subject: [PATCH 112/310] Fix some more dialyzer warnings on erlcloud_aws We isolate all the application: related issues (get_env, set_env, unset_env) in 3 small and contained functions and have dialyzer ignore them in its analysis --- src/erlcloud_aws.erl | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index d07e5777e..e93e9a482 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -602,13 +602,13 @@ update_config(#aws_config{} = Config) -> -spec clear_config(aws_config()) -> ok. clear_config(#aws_config{assume_role = #aws_assume_role{role_arn = Arn, external_id = ExtId}}) -> - application:unset_env(erlcloud, {role_credentials, Arn, ExtId}). + unset_env_for_role_credentials(Arn, ExtId). -spec clear_expired_configs() -> ok. clear_expired_configs() -> Env = application:get_all_env(erlcloud), Now = calendar:datetime_to_gregorian_seconds(calendar:universal_time()), - [application:unset_env(erlcloud, {role_credentials, Arn, ExtId}) || + [unset_env_for_role_credentials(Arn, ExtId) || {{role_credentials, Arn, ExtId}, #role_credentials{expiration_gregorian_seconds = Ts}} <- Env, Ts < Now], @@ -892,10 +892,7 @@ prop_to_list_defined( Name, Props ) -> -spec get_role_credentials(aws_config()) -> {ok, #role_credentials{}}. get_role_credentials(#aws_config{assume_role = AssumeRole} = Config) -> - case application:get_env(erlcloud, - {role_credentials, - AssumeRole#aws_assume_role.role_arn, - AssumeRole#aws_assume_role.external_id}) of + case get_env_for_role_credentials(AssumeRole#aws_assume_role.role_arn, AssumeRole#aws_assume_role.external_id) of {ok, #role_credentials{expiration_gregorian_seconds = Expiration} = Credentials} -> Now = calendar:datetime_to_gregorian_seconds(calendar:universal_time()), %% Get new credentials if these will expire in less than 5 minutes @@ -925,11 +922,7 @@ get_credentials_from_role(#aws_config{assume_role = AssumeRole} = Config) -> secret_access_key = proplists:get_value(secret_access_key, Creds), session_token = proplists:get_value(session_token, Creds), expiration_gregorian_seconds = ExpireAt}, - application:set_env(erlcloud, - {role_credentials, - AssumeRole#aws_assume_role.role_arn, - AssumeRole#aws_assume_role.external_id}, - Record), + set_env_for_role_credentials(AssumeRole#aws_assume_role.role_arn, AssumeRole#aws_assume_role.external_id, Record), {ok, Record}. port_to_str(Port) when is_integer(Port) -> @@ -1368,3 +1361,29 @@ error_msg( Message ) -> error_msg( Format, Values ) -> Error = iolist_to_binary( io_lib:format( Format, Values ) ), throw( {error, Error} ). + +-dialyzer({nowarn_function, unset_env_for_role_credentials/2}). +-spec unset_env_for_role_credentials(Arn, ExtId) -> ok + when Arn :: string() | undefined, + ExtId :: string() | undefined. +unset_env_for_role_credentials(Arn, ExtId) -> + % application:unset_env is undocumented in regards to type(Par) =/= atom() + application:unset_env(erlcloud, {role_credentials, Arn, ExtId}). + +-dialyzer({nowarn_function, get_env_for_role_credentials/2}). +-spec get_env_for_role_credentials(Arn, ExtId) -> undefined | {ok, Val} + when Arn :: string() | undefined, + ExtId :: string() | undefined, + Val :: term(). +get_env_for_role_credentials(Arn, ExtId) -> + % application:get_env is undocumented in regards to type(Par) =/= atom() + application:get_env(erlcloud, {role_credentials, Arn, ExtId}). + +-dialyzer({nowarn_function, set_env_for_role_credentials/3}). +-spec set_env_for_role_credentials(Arn, ExtId, Val) -> ok + when Arn :: string() | undefined, + ExtId :: string() | undefined, + Val :: term(). +set_env_for_role_credentials(Arn, ExtId, Val) -> + % application:set_env is undocumented in regards to type(Par) =/= atom() + application:set_env(erlcloud, {role_credentials, Arn, ExtId}, Val). From 9c93a7645631323662b99ff1557c34fff737db6c Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 14:19:30 +0000 Subject: [PATCH 113/310] Ignore dialyzer warnings where no failure is happening --- src/erlcloud_aws.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index e93e9a482..bca4160c5 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -600,10 +600,12 @@ update_config(#aws_config{} = Config) -> security_token = Credentials#metadata_credentials.security_token}} end. +-dialyzer({no_return, clear_config/1}). -spec clear_config(aws_config()) -> ok. clear_config(#aws_config{assume_role = #aws_assume_role{role_arn = Arn, external_id = ExtId}}) -> unset_env_for_role_credentials(Arn, ExtId). +-dialyzer({no_match, clear_expired_configs/0}). -spec clear_expired_configs() -> ok. clear_expired_configs() -> Env = application:get_all_env(erlcloud), @@ -890,6 +892,7 @@ prop_to_list_defined( Name, Props ) -> end. +-dialyzer({no_return, get_role_credentials/1}). -spec get_role_credentials(aws_config()) -> {ok, #role_credentials{}}. get_role_credentials(#aws_config{assume_role = AssumeRole} = Config) -> case get_env_for_role_credentials(AssumeRole#aws_assume_role.role_arn, AssumeRole#aws_assume_role.external_id) of @@ -904,6 +907,7 @@ get_role_credentials(#aws_config{assume_role = AssumeRole} = Config) -> get_credentials_from_role(Config) end. +-compile({nowarn_unused_function, get_credentials_from_role/1}). -spec get_credentials_from_role(aws_config()) -> {ok, #role_credentials{}}. get_credentials_from_role(#aws_config{assume_role = AssumeRole} = Config) -> %% We have to reset the assume role to make sure we do not From 4431f83846fa9ef267b4936efe79871b087c7b5f Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 14:51:11 +0000 Subject: [PATCH 114/310] Revert previous change and improve on record typespec --- include/erlcloud_cloudformation.hrl | 2 +- src/erlcloud_cloudformation.erl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/erlcloud_cloudformation.hrl b/include/erlcloud_cloudformation.hrl index 1e82060a3..da1ef8373 100644 --- a/include/erlcloud_cloudformation.hrl +++ b/include/erlcloud_cloudformation.hrl @@ -13,7 +13,7 @@ parameters = [] :: [cloudformation_parameter()], resource_types = [] :: [string()], role_arn :: string(), - rollback_configuration :: cloudformation_rollback_configuration(), + rollback_configuration :: undefined | cloudformation_rollback_configuration(), stack_name :: string(), stack_policy_body :: string(), stack_policy_url :: string(), diff --git a/src/erlcloud_cloudformation.erl b/src/erlcloud_cloudformation.erl index 3b80caad3..24592c733 100644 --- a/src/erlcloud_cloudformation.erl +++ b/src/erlcloud_cloudformation.erl @@ -841,7 +841,9 @@ cloudformation_rollback_configuration_fields(#cloudformation_rollback_configurat RollbackConfig#cloudformation_rollback_configuration.rollback_triggers ) ) - ]). + ]); +cloudformation_rollback_configuration_fields(_) -> + []. cloudformation_rollback_triggers_fields(RollbackTriggers) -> lists:map(fun cloudformation_rollback_trigger_fields/1, RollbackTriggers). From 067807c49c72d28eb65d050dd8da630732d9ee3b Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 15:19:05 +0000 Subject: [PATCH 115/310] Fix bad `-spec(_).` --- src/erlcloud_states.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_states.erl b/src/erlcloud_states.erl index 3c00873f3..9a85c3a69 100644 --- a/src/erlcloud_states.erl +++ b/src/erlcloud_states.erl @@ -539,7 +539,7 @@ start_execution(StateMachineArn, Options) -> start_execution(StateMachineArn, Options, default_config()). -spec start_execution(StateMachineArn :: binary(), - Options :: list() | map(), + Options :: map(), Config :: aws_config()) -> {ok, map()} | {error, any()}. start_execution(StateMachineArn, Options, Config) @@ -573,7 +573,7 @@ stop_execution(ExecutionArn, Options) -> stop_execution(ExecutionArn, Options, default_config()). -spec stop_execution(ExecutionArn :: binary(), - Options :: list() | map(), + Options :: map(), Config :: aws_config()) -> {ok, map()} | {error, any()}. stop_execution(ExecutionArn, Options, Config) From 7b523380db6496b1ace316279d966cf7f00aab38 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 18 Feb 2020 15:49:46 +0000 Subject: [PATCH 116/310] Reduce amount of copy-pasted text --- src/erlcloud_application_autoscaler.erl | 84 +++++++------------------ 1 file changed, 24 insertions(+), 60 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index b05dba511..3679a6568 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -61,6 +61,10 @@ -type response_attribute() :: string() | integer(). -type response_key() :: atom(). -type response() :: [{response_key(), response_attribute()}]. +-type ok_error_response() :: {ok, jsx:json_term()} + | {error, metadata_not_available + | container_credentials_unavailable + | erlcloud_aws:httpc_result_error()}. -type aws_aas_request_body() :: [proplist:proplist()]. @@ -318,9 +322,7 @@ delete_scaling_policy(Configuration, BodyConfiguration) -> ScalableDimension :: binary(), ScheduledActionName :: binary(), ServiceNamespace :: binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). delete_scheduled_action(Configuration, ResourceId, ScalableDimension, ScheduledActionName, ServiceNamespace) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -331,9 +333,7 @@ delete_scheduled_action(Configuration, ResourceId, ScalableDimension, ScheduledA -spec delete_scheduled_action(Configuration :: aws_config(), BodyConfiguration :: aws_aas_request_body() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). delete_scheduled_action(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DeleteScheduledAction"). @@ -349,9 +349,7 @@ delete_scheduled_action(Configuration, BodyConfiguration) -> ResourceId :: binary(), ScalableDimension :: binary(), ServiceNamespace :: binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). deregister_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace) -> BodyProps = [{<<"ResourceId">>, ResourceId}, {<<"ScalableDimension">>, ScalableDimension}, @@ -361,9 +359,7 @@ deregister_scalable_target(Configuration, ResourceId, ScalableDimension, Service -spec deregister_scalable_target( Configuration :: erlcloud_aws:aws_config(), BodyConfiguration :: aws_aas_request_body() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). deregister_scalable_target(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.DeregisterScalableTarget"). @@ -377,9 +373,7 @@ deregister_scalable_target(Configuration, BodyConfiguration) -> -spec describe_scalable_targets( erlcloud_aws:aws_config(), aws_aas_request_body() | binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). describe_scalable_targets(Configuration, ServiceNamespace) when is_binary(ServiceNamespace)-> describe_scalable_targets(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scalable_targets(Configuration, BodyConfiguration) -> @@ -407,9 +401,7 @@ describe_scalable_targets(Configuration, BodyConfiguration) -> -spec describe_scaling_activities( erlcloud_aws:aws_config(), aws_aas_request_body() | binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). describe_scaling_activities(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scaling_activities(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scaling_activities(Configuration, BodyConfiguration) -> @@ -438,9 +430,7 @@ describe_scaling_activities(Configuration, BodyConfiguration) -> -spec describe_scaling_policies( erlcloud_aws:aws_config(), aws_aas_request_body() | binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). describe_scaling_policies(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scaling_policies(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scaling_policies(Configuration, BodyConfiguration) -> @@ -469,9 +459,7 @@ describe_scaling_policies(Configuration, BodyConfiguration) -> -spec describe_scheduled_actions( erlcloud_aws:aws_config(), aws_aas_request_body() | binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). describe_scheduled_actions(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> describe_scheduled_actions(Configuration, [{<<"ServiceNamespace">>, ServiceNamespace}]); describe_scheduled_actions(Configuration, BodyConfiguration) -> @@ -503,9 +491,7 @@ describe_scheduled_actions(Configuration, BodyConfiguration) -> ResourceId :: binary(), ScalableDimension :: binary(), ServiceNamespace :: binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, ServiceNamespace) -> BodyProps = [{<<"PolicyName">>, PolicyName}, {<<"ResourceId">>, ResourceId}, @@ -521,9 +507,7 @@ put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, Ser ServiceNamespace :: binary(), PolicyType :: binary(), Policy :: [proplist:proplist()] - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, ServiceNamespace, PolicyType, Policy) -> BodyProps = [{<<"PolicyName">>, PolicyName}, {<<"ResourceId">>, ResourceId}, @@ -540,9 +524,7 @@ put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, Ser -spec put_scaling_policy( Configuration :: erlcloud_aws:aws_config(), BodyConfiguration :: aws_aas_request_body() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). put_scaling_policy(Configuration, BodyConfiguration) -> case request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.PutScalingPolicy") of {ok, Result} -> @@ -566,9 +548,7 @@ put_scaling_policy(Configuration, BodyConfiguration) -> ScalableDimension :: binary(), ServiceNamespace :: binary(), ScheduledActionName :: binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -583,9 +563,7 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp ServiceNamespace :: binary(), ScheduledActionName :: binary(), Schedule :: binary() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -602,9 +580,7 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp ScheduledActionName :: binary(), Schedule :: binary(), StartTime :: pos_integer() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule, StartTime) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -623,9 +599,7 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp Schedule :: binary(), StartTime :: pos_integer(), EndTime :: pos_integer() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ScheduledActionName, Schedule, StartTime, EndTime) -> BodyProps = [{<<"ScheduledActionName">>, ScheduledActionName}, {<<"ResourceId">>, ResourceId}, @@ -638,9 +612,7 @@ put_scheduled_action(Configuration, ResourceId, ScalableDimension, ServiceNamesp -spec put_scheduled_action(Configuration :: aws_config(), BodyConfiguration :: aws_aas_request_body() - ) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ) -> ok_error_response(). put_scheduled_action(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.PutScheduledAction"). @@ -654,9 +626,7 @@ put_scheduled_action(Configuration, BodyConfiguration) -> -spec register_scalable_target(Configuration :: aws_config(), ResourceId :: binary(), ScalableDimension :: binary(), - ServiceNamespace :: binary()) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ServiceNamespace :: binary()) -> ok_error_response(). register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace) -> BodyProps = [{<<"ResourceId">>, ResourceId}, {<<"ScalableDimension">>, ScalableDimension}, @@ -667,9 +637,7 @@ register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNa ResourceId :: binary(), ScalableDimension :: binary(), ServiceNamespace :: binary(), - ResourceARN :: binary()) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + ResourceARN :: binary()) -> ok_error_response(). register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ResourceARN) -> BodyProps = [{<<"ResourceId">>, ResourceId}, {<<"ScalableDimension">>, ScalableDimension}, @@ -683,9 +651,7 @@ register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNa ServiceNamespace :: binary(), ResourceARN :: binary(), MinCapacity :: integer() | undefined, - MaxCapacity :: integer() | undefined) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + MaxCapacity :: integer() | undefined) -> ok_error_response(). register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNamespace, ResourceARN, MinCapacity, MaxCapacity) -> BodyProps = [{<<"ResourceId">>, ResourceId}, {<<"ScalableDimension">>, ScalableDimension}, @@ -704,9 +670,7 @@ register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNa -spec register_scalable_target( Configuration :: erlcloud_aws:aws_config(), BodyConfiguration :: aws_aas_request_body() -) -> {ok, jsx:json_term()} | {error, metadata_not_available - | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. +) -> ok_error_response(). register_scalable_target(Configuration, BodyConfiguration) -> request_with_action(Configuration, BodyConfiguration, "AnyScaleFrontendService.RegisterScalableTarget"). From 663cf3fc557ad6ed01b248f269f5f14b8934541b Mon Sep 17 00:00:00 2001 From: alexo-spb Date: Tue, 3 Mar 2020 22:26:03 +0000 Subject: [PATCH 117/310] Extend VPC description with cidr_block_association_set --- src/erlcloud_ec2.erl | 24 ++++++++++++++-- test/erlcloud_ec2_tests.erl | 57 +++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index 87220c00b..3d79ad293 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -2459,10 +2459,30 @@ extract_vpc(Node) -> {dhcp_options_id, get_text("dhcpOptionsId", Node)}, {instance_tenancy, get_text("instanceTenancy", Node)}, {is_default, get_bool("isDefault", Node)}, - {tag_set, + {cidr_block_association_set, extract_cidr_block_association_set(Node)}, + {tag_set, [extract_tag_item(Item) || Item <- xmerl_xpath:string("tagSet/item", Node)]} - ]. + ]. + +extract_cidr_block_association_set(Node) -> + Items = xmerl_xpath:string("cidrBlockAssociationSet/item", Node), + [extract_cidr_block_association_item(Item) || Item <- Items]. + +extract_cidr_block_association_item(Node) -> + [ + {cidr_block, get_text("cidrBlock", Node)}, + {association_id, get_text("associationId", Node)}, + {cidr_block_state, + [{state, get_text("cidrBlockState/state", Node)}] ++ + case get_text("cidrBlockState/statusMessage", Node, undefined) of + undefined -> + []; + StatusMessage -> + [{status_message, StatusMessage}] + end + } + ]. -spec detach_internet_gateway(string(), string()) -> ok. detach_internet_gateway(GatewayID, VpcID) -> diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 48de885cb..9809cf6ef 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -38,6 +38,7 @@ describe_test_() -> fun start/0, fun stop/1, [ + fun describe_vpcs_tests/1, fun describe_tags_input_tests/1, fun describe_tags_output_tests/1, fun request_spot_fleet_input_tests/1, @@ -181,6 +182,62 @@ output_tests(Fun, Tests) -> %%% Actual test specifiers %%%=================================================================== +describe_vpcs_tests(_) -> + Tests = [ + ?_ec2_test({ + "Describe all the vpcs", + " + 9a0571b9-6e91-47e7-b75f-785125322853 + + + vpc-00000000000000001 + 000000000001 + available + 10.0.0.0/16 + dopt-00000001 + + + 10.0.0.0/16 + vpc-cidr-assoc-00000000000000001 + + associated + + + + + + Key + Value + + + default + false + + + ", + {ok, [ + [ + {vpc_id, "vpc-00000000000000001"}, + {state, "available"}, + {cidr_block, "10.0.0.0/16"}, + {dhcp_options_id, "dopt-00000001"}, + {instance_tenancy, "default"}, + {is_default, false}, + {cidr_block_association_set, [ + [ + {cidr_block, "10.0.0.0/16"}, + {association_id, "vpc-cidr-assoc-00000000000000001"}, + {cidr_block_state, [{state, "associated"}]} + ] + ]}, + {tag_set, [ + [{key, "Key"}, {value, "Value"}] + ]} + ] + ]} + }) + ], + output_tests(?_f(erlcloud_ec2:describe_vpcs()), Tests). %% DescribeTags test based on the API examples: %% http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeTags.html From faf34ab8eecdebf0259ddf67321f19430a8461a2 Mon Sep 17 00:00:00 2001 From: alexo-spb Date: Wed, 4 Mar 2020 10:01:18 +0000 Subject: [PATCH 118/310] Extend example with test statusMessage --- test/erlcloud_ec2_tests.erl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 9809cf6ef..35246887e 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -203,6 +203,14 @@ describe_vpcs_tests(_) -> associated + + 10.1.0.0/16 + vpc-cidr-assoc-00000000000000002 + + test state + test status message + + @@ -228,6 +236,11 @@ describe_vpcs_tests(_) -> {cidr_block, "10.0.0.0/16"}, {association_id, "vpc-cidr-assoc-00000000000000001"}, {cidr_block_state, [{state, "associated"}]} + ], + [ + {cidr_block, "10.1.0.0/16"}, + {association_id, "vpc-cidr-assoc-00000000000000002"}, + {cidr_block_state, [{state, "test state"}, {status_message, "test status message"}]} ] ]}, {tag_set, [ From 707c16906dc493986d3ff34cc659fb657805d5de Mon Sep 17 00:00:00 2001 From: Mike Benza Date: Thu, 19 Mar 2020 23:49:11 -0500 Subject: [PATCH 119/310] Clean up some phrasing in README, erlcloud_aws.erl --- README.md | 25 ++++++++++++------------- src/erlcloud_aws.erl | 8 ++++---- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 439f1dcc6..faecd23ac 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,13 @@ Service APIs implemented: - AWS Cost and Usage Report API - and more to come -Majority of API functions have been implemented. +The majority of API functions have been implemented. Not all functions have been thoroughly tested, so exercise care when integrating this library into production code. Please send issues and patches. -The libraries can be used two ways: -- either you can specify configuration parameters in the process dictionary. Useful for simple tasks -- you can create a configuration object and pass that to each request as the final parameter. Useful for Cross AWS Account access +The libraries can be used two ways. You can +- specify configuration parameters in the process dictionary. Useful for simple tasks, or +- create a configuration object and pass that to each request as the final parameter. Useful for Cross AWS Account access ## Roadmap ## @@ -94,10 +94,9 @@ application:ensure_all_started(erlcloud). ``` ### Using Temporary Security Credentials -The access to AWS resource might be managed through [third-party identity provider](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp.html). -The access is managed using [temporary security credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). +When access to AWS resources is managed through [third-party identity providers](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp.html) it is performed using [temporary security credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). -You can provide your amazon credentials in OS environmental variables +You can provide your AWS credentials in OS environment variables ``` export AWS_ACCESS_KEY_ID= @@ -105,7 +104,7 @@ export AWS_SECRET_ACCESS_KEY= export AWS_SESSION_TOKEN= export AWS_DEFAULT_REGION= ``` -If you did not provide your amazon credentials in the OS environmental variables, then you need to provide configuration read from your profile: +If you did not provide your AWS credentials in the OS environment variables, then you need to provide configuration read from your profile: ``` {ok, Conf} = erlcloud_aws:profile(). erlcloud_s3:list_buckets(Conf). @@ -118,12 +117,12 @@ application:set_env(erlcloud, aws_security_token, "your token"), application:set_env(erlcloud, aws_region, "your region"), ``` ### Using Access Key ### -You can provide your amazon credentials in environmental variables. +You can provide your AWS credentials in environmental variables. ``` export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= ``` -If you did not provide your amazon credentials in the environmental variables, then you need to provide the per-process configuration: +If you did not provide your AWS credentials in the environment variables, then you need to provide the per-process configuration: ``` erlcloud_ec2:configure(AccessKeyId, SecretAccessKey [, Hostname]). ``` @@ -137,12 +136,12 @@ erlcloud_ec2:describe_images(EC2). ``` ### aws_config -[aws_config](https://github.com/erlcloud/erlcloud/blob/master/include/erlcloud_aws.hrl) record contains many valuable defaults, +The [aws_config](https://github.com/erlcloud/erlcloud/blob/master/include/erlcloud_aws.hrl) record contains many valuable defaults, such as protocols and ports for AWS services. You can always redefine them by making new `#aws_config{}` record and changing particular fields, then passing the result to any erlcloud function. But if you want to change something in runtime this might be tedious and/or not flexible enough. -Alternative approach is to set default fields within the `app.config -> erlcloud -> aws_config` section and +An alternative approach is to set default fields within the `app.config -> erlcloud -> aws_config` section and rely on the config, used by all functions by default. Example of such app.config: @@ -160,7 +159,7 @@ Example of such app.config: ### Basic use ### -Then you can start making api calls, like: +Then you can start making API calls, like: ``` erlcloud_ec2:describe_images(). % list buckets of Account stored in config in process dict diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index bca4160c5..36b8e7d37 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -1203,8 +1203,8 @@ profile( Name ) -> %% source_profile = default %% %% -%% and finally, will supports the role_arn specification, and will -%% assume the role indicated using the credentials current when interpreting +%% Finally, it supports the role_arn specification, and will +%% assume the role indicated using the current credentials when interpreting %% the profile in which they it is declared: %% %% 
@@ -1213,7 +1213,7 @@ profile( Name ) ->
 %%  source_profile = default
 %%
%% -%% When using the the role_arn specification, you may supply the +%% When using the role_arn specification, you may supply the %% following two options to control the way in which the assume_role request %% is made via AWS STS service: %% @@ -1230,7 +1230,7 @@ profile( Name ) -> %% %%
  • 'external_id' %%

    The identifier that is used in the ExternalId -%% parameter. If this option is not specified, then it will default to +%% parameter. If this option isn't specified, then it will default to %% 'undefined', which will work for normal in-account roles, but will %% need to be specified for roles in external accounts.

    %%
    • From 90b7ee21ee4db9d7fe5ba8920689aa4e16e8a274 Mon Sep 17 00:00:00 2001 From: motobob Date: Mon, 13 Apr 2020 11:33:00 +0100 Subject: [PATCH 120/310] allow using configured VPCe make is ENV configurable --- src/erlcloud.app.src | 6 ++++- src/erlcloud_aws.erl | 52 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/erlcloud.app.src b/src/erlcloud.app.src index 677803889..ac3403bea 100644 --- a/src/erlcloud.app.src +++ b/src/erlcloud.app.src @@ -18,7 +18,11 @@ %% hackney, lhttpc]}, {modules, []}, - {env, []}, + {env, [ + % Example [{<<"kinesis">>, [<<"myAZ1.amazonaws.com">>, <<"myAZ2.amazonaws.com">>]}] + % or vi ENV [{<<"kinesis">>, {env, "KINESIS_VPC_ENDPOINTS"}] + {services_vpc_endpoints, []} + ]}, {licenses, ["MIT"]}, {links, [{"Github", "https://github.com/erlcloud/erlcloud"}]} ] diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 36b8e7d37..721f433d6 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -14,6 +14,7 @@ default_config_region/2, default_config_override/1, update_config/1,clear_config/1, clear_expired_configs/0, service_config/3, service_host/2, + get_host_vpc_endpoint/2, get_vpc_endpoints/0, configure/1, format_timestamp/1, http_headers_body/1, http_body/1, @@ -796,14 +797,55 @@ service_host( <<"s3">>, Region ) -> service_host( <<"iam">>, <<"cn-north-1">> ) -> "iam.amazonaws.com.cn"; service_host( <<"iam">>, <<"cn-northwest-1">> ) -> "iam.amazonaws.com.cn"; service_host( <<"sdb">>, <<"us-east-1">> ) -> "sdb.amazonaws.com"; -service_host( <<"states">>, Region ) -> - binary_to_list( <<"states.", Region/binary, ".amazonaws.com">> ); service_host( Service, <<"cn-north-1">> = Region ) when is_binary(Service) -> binary_to_list( <> ); -service_host( Service, <<"cn-northwest-1">> =Region ) when is_binary(Service) -> +service_host( Service, <<"cn-northwest-1">> = Region ) when is_binary(Service) -> binary_to_list( <> ); -service_host( Service, Region ) when is_binary(Service) -> - binary_to_list( <> ). +% services which can have VPCe configured to mitigate cross-AZ traffic. +% It's application level decision to use VPCe and configure those +% magic can be done vi EC2 DescribeVpcEndpoints/filter by VPC/filter by AZ, +% however permissions and describe* API throttling is not what we want to deal with here. +service_host( Service, Region ) when is_binary(Service) andalso is_binary(Region) -> + Default = binary_to_list( <> ), + get_host_vpc_endpoint(Service, Default). + +-spec get_host_vpc_endpoint(binary(), binary()) -> binary(). +%% take the list of possibly configured endpoints +%% and pick the one which suites from our AZ. +get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> + ConfiguredEndpoints = proplists:get_value(Service, + application:get_env(erlcloud, services_vpc_endpoints, [])), + %% resolve through ENV if any + Endpoints = case ConfiguredEndpoints of + {env, EnvVarName} when is_list(EnvVarName) -> + Es = string:split(os:getenv(EnvVarName, ""), ",", all), + [list_to_binary(E) || E <- Es]; + EndpointsList when is_list(EndpointsList) -> + EndpointsList + end, + % now get our AZ and match + case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", default_config()) of + {ok, AZ} -> + lists:foldl( + fun (E , Acc) -> + case {binary:match(E, AZ), Acc == Default} of + {nomatch, _} -> Acc; + % take only the first one if smb provided duplicates + {_, true} -> E; + % was previously set + {_, false} -> Acc + end + end, + Default, + Endpoints + ); + {error, _} -> + Default + end. + +-spec get_vpc_endpoints () -> list({binary(), binary()}). +get_vpc_endpoints() -> + application:get_env(erlcloud, services_vpc_endpoints, []). -spec configure(aws_config()) -> {ok, aws_config()}. From dbafa2b30ae98ddd58e25a7577d53b6319735431 Mon Sep 17 00:00:00 2001 From: motobob Date: Mon, 13 Apr 2020 17:54:31 +0100 Subject: [PATCH 121/310] do not call hostmeta when nothing configured --- src/erlcloud.app.src | 2 +- src/erlcloud_aws.erl | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/erlcloud.app.src b/src/erlcloud.app.src index ac3403bea..6a73400e4 100644 --- a/src/erlcloud.app.src +++ b/src/erlcloud.app.src @@ -20,7 +20,7 @@ {modules, []}, {env, [ % Example [{<<"kinesis">>, [<<"myAZ1.amazonaws.com">>, <<"myAZ2.amazonaws.com">>]}] - % or vi ENV [{<<"kinesis">>, {env, "KINESIS_VPC_ENDPOINTS"}] + % or via ENV [{<<"kinesis">>, {env, "KINESIS_VPC_ENDPOINTS"}] {services_vpc_endpoints, []} ]}, {licenses, ["MIT"]}, diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 721f433d6..4e86c63d0 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -819,11 +819,15 @@ get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> Endpoints = case ConfiguredEndpoints of {env, EnvVarName} when is_list(EnvVarName) -> Es = string:split(os:getenv(EnvVarName, ""), ",", all), - [list_to_binary(E) || E <- Es]; + [list_to_binary(E) || E <- Es, E /= ""]; EndpointsList when is_list(EndpointsList) -> EndpointsList end, - % now get our AZ and match + % now match our AZ to configured ones + pick_vpc_endpoint(Endpoints, Default). + +pick_vpc_endpoint([], Default) -> Default; +pick_vpc_endpoint(Endpoints, Default) -> case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", default_config()) of {ok, AZ} -> lists:foldl( From 3a849de7ab9b521822735cd7585be86892b7df93 Mon Sep 17 00:00:00 2001 From: motobob Date: Tue, 14 Apr 2020 12:08:52 +0100 Subject: [PATCH 122/310] review --- README.md | 18 +++++++++++++++++- src/erlcloud.app.src | 2 +- src/erlcloud_aws.erl | 15 +++++++-------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index faecd23ac..0f80010c9 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,24 @@ Example of such app.config: ]. ``` +### VPC endpoints +If you want to utilise AZ affinity for VPC endpoints you can configure those in application config via: +```erlang +{erlcloud, [ + {services_vpc_endpoints, [ + {<<"sqs">>, [<<"myAZ1.sqs-dns.amazonaws.com">>, <<"myAZ2.sqs-dns.amazonaws.com">>]}, + {<<"kinesis">>, {env, "KINESIS_VPC_ENDPOINTS"}} + ]} +]} +``` +Two options supported: + - explicit lists of route53 AZ endpoints + - OS environment variable, comes handy for ECS deployments. + +Upon config generation, `erlcloud` will check the AZ of the deployment and match it to one of the pre-configured DNS records to use. + -### Basic use ### +## Basic use ## Then you can start making API calls, like: ``` erlcloud_ec2:describe_images(). diff --git a/src/erlcloud.app.src b/src/erlcloud.app.src index 6a73400e4..6d834d26f 100644 --- a/src/erlcloud.app.src +++ b/src/erlcloud.app.src @@ -19,7 +19,7 @@ lhttpc]}, {modules, []}, {env, [ - % Example [{<<"kinesis">>, [<<"myAZ1.amazonaws.com">>, <<"myAZ2.amazonaws.com">>]}] + % Example [{<<"kinesis">>, [<<"myAZ1.amazonaws.com">>, <<"myAZ2.amazonaws.com">>]}] % or via ENV [{<<"kinesis">>, {env, "KINESIS_VPC_ENDPOINTS"}] {services_vpc_endpoints, []} ]}, diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 4e86c63d0..f0f378997 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -801,24 +801,23 @@ service_host( Service, <<"cn-north-1">> = Region ) when is_binary(Service) -> binary_to_list( <> ); service_host( Service, <<"cn-northwest-1">> = Region ) when is_binary(Service) -> binary_to_list( <> ); -% services which can have VPCe configured to mitigate cross-AZ traffic. -% It's application level decision to use VPCe and configure those -% magic can be done vi EC2 DescribeVpcEndpoints/filter by VPC/filter by AZ, -% however permissions and describe* API throttling is not what we want to deal with here. service_host( Service, Region ) when is_binary(Service) andalso is_binary(Region) -> Default = binary_to_list( <> ), get_host_vpc_endpoint(Service, Default). -spec get_host_vpc_endpoint(binary(), binary()) -> binary(). -%% take the list of possibly configured endpoints -%% and pick the one which suites from our AZ. +% some services can have VPCe configured and we allow to mitigate cross-AZ traffic. +% It's application level decision to use VPCe and configure those. +% magic can be done via EC2 DescribeVpcEndpoints/filter by VPC/filter by AZ. +% however, permissions and describe* API throttling is not what we want to deal with here. get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> - ConfiguredEndpoints = proplists:get_value(Service, - application:get_env(erlcloud, services_vpc_endpoints, [])), + VPCEndpointsByService = application:get_env(erlcloud, services_vpc_endpoints, []), + ConfiguredEndpoints = proplists:get_value(Service, VPCEndpointsByService, []), %% resolve through ENV if any Endpoints = case ConfiguredEndpoints of {env, EnvVarName} when is_list(EnvVarName) -> Es = string:split(os:getenv(EnvVarName, ""), ",", all), + % ignore "" env var or ",," cases [list_to_binary(E) || E <- Es, E /= ""]; EndpointsList when is_list(EndpointsList) -> EndpointsList From 40fe8d98bc114b9a30534baac70dc8f15b4abf51 Mon Sep 17 00:00:00 2001 From: motobob Date: Tue, 14 Apr 2020 16:24:51 +0100 Subject: [PATCH 123/310] fix readme. DO NOT HAVE INFINITE RECURSION --- README.md | 4 ++-- src/erlcloud_aws.erl | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0f80010c9..76e700e6a 100644 --- a/README.md +++ b/README.md @@ -169,9 +169,9 @@ If you want to utilise AZ affinity for VPC endpoints you can configure those in ``` Two options supported: - explicit lists of route53 AZ endpoints - - OS environment variable, comes handy for ECS deployments. + - OS environment variable, comes handy for ECS deployments. Env should be of comma separated string like:`"myAZ1.sqs-dns.amazonaws.com,myAZ2.sqs-dns.amazonaws.com"` -Upon config generation, `erlcloud` will check the AZ of the deployment and match it to one of the pre-configured DNS records to use. +Upon config generation, `erlcloud` will check the AZ of the deployment and match it to the first one of the pre-configured DNS records to use. ## Basic use ## diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index f0f378997..d1426b943 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -827,7 +827,9 @@ get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> pick_vpc_endpoint([], Default) -> Default; pick_vpc_endpoint(Endpoints, Default) -> - case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", default_config()) of + % it fine to use default here - no IAM is used, only for http client + % one cannot use auto_config()/default_cfg() as it creates an infinite recursion. + case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", #aws_config{}) of {ok, AZ} -> lists:foldl( fun (E , Acc) -> From ab769bfd2725d39a20610ce4f28d6c230e49ed21 Mon Sep 17 00:00:00 2001 From: motobob Date: Tue, 14 Apr 2020 16:30:41 +0100 Subject: [PATCH 124/310] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 76e700e6a..7294bc6b6 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,8 @@ Two options supported: - explicit lists of route53 AZ endpoints - OS environment variable, comes handy for ECS deployments. Env should be of comma separated string like:`"myAZ1.sqs-dns.amazonaws.com,myAZ2.sqs-dns.amazonaws.com"` -Upon config generation, `erlcloud` will check the AZ of the deployment and match it to the first one of the pre-configured DNS records to use. +Upon config generation, `erlcloud` will check the AZ of the deployment +and match it to one of the pre-configured DNS records. First match is used and if not match found default is used. ## Basic use ## From d3943b731d4be93a50a4ec265ecbf4785c54e2c0 Mon Sep 17 00:00:00 2001 From: motobob Date: Tue, 14 Apr 2020 18:16:47 +0100 Subject: [PATCH 125/310] Update erlcloud_aws.erl --- src/erlcloud_aws.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index d1426b943..8a9504ff0 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -831,7 +831,7 @@ pick_vpc_endpoint(Endpoints, Default) -> % one cannot use auto_config()/default_cfg() as it creates an infinite recursion. case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", #aws_config{}) of {ok, AZ} -> - lists:foldl( + AZEndpoint = lists:foldl( fun (E , Acc) -> case {binary:match(E, AZ), Acc == Default} of {nomatch, _} -> Acc; @@ -843,7 +843,8 @@ pick_vpc_endpoint(Endpoints, Default) -> end, Default, Endpoints - ); + ), + binary_to_list(AZEndpoint); {error, _} -> Default end. From cc705a59f46421f5a115c76eaf55583efaa3ba27 Mon Sep 17 00:00:00 2001 From: motobob Date: Tue, 14 Apr 2020 18:39:31 +0100 Subject: [PATCH 126/310] Update erlcloud_aws.erl --- src/erlcloud_aws.erl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 8a9504ff0..b6e5ab64c 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -802,8 +802,8 @@ service_host( Service, <<"cn-north-1">> = Region ) when is_binary(Service) -> service_host( Service, <<"cn-northwest-1">> = Region ) when is_binary(Service) -> binary_to_list( <> ); service_host( Service, Region ) when is_binary(Service) andalso is_binary(Region) -> - Default = binary_to_list( <> ), - get_host_vpc_endpoint(Service, Default). + Default = <>, + binary_to_list(get_host_vpc_endpoint(Service, Default)). -spec get_host_vpc_endpoint(binary(), binary()) -> binary(). % some services can have VPCe configured and we allow to mitigate cross-AZ traffic. @@ -831,7 +831,7 @@ pick_vpc_endpoint(Endpoints, Default) -> % one cannot use auto_config()/default_cfg() as it creates an infinite recursion. case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", #aws_config{}) of {ok, AZ} -> - AZEndpoint = lists:foldl( + lists:foldl( fun (E , Acc) -> case {binary:match(E, AZ), Acc == Default} of {nomatch, _} -> Acc; @@ -843,8 +843,7 @@ pick_vpc_endpoint(Endpoints, Default) -> end, Default, Endpoints - ), - binary_to_list(AZEndpoint); + ); {error, _} -> Default end. From 5355c54ffbec96eecbe85ea2f99bda094e666a46 Mon Sep 17 00:00:00 2001 From: motobob Date: Tue, 14 Apr 2020 18:50:26 +0100 Subject: [PATCH 127/310] one more place for VPC endpoints handling --- src/erlcloud_aws.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index b6e5ab64c..6d90fc36a 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -161,11 +161,14 @@ aws_region_from_host(Host) -> %% the second is the region identifier, the rest is ignored %% the exception (of course) is the dynamodb streams and the marketplace which follows a %% different format + %% another exception is VPC endpoints ["streams", "dynamodb", Value | _Rest] -> Value; [Prefix, "marketplace", Value | _Rest] - when Prefix =:= "metering"; Prefix =:= "entitlement" -> - Value; + when Prefix =:= "metering"; Prefix =:= "entitlement" -> + Value; + [_, _, Value, "vpce" | _Rest] -> + Value; [_, Value, _, _ | _Rest] -> Value; _ -> From 90cbd0365cf8dd2089426a53402ac742d888c30a Mon Sep 17 00:00:00 2001 From: Mike Benza Date: Wed, 15 Apr 2020 09:50:58 -0500 Subject: [PATCH 128/310] Fix VPC endpoint documentation minor grammar fixes --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7294bc6b6..62f72881a 100644 --- a/README.md +++ b/README.md @@ -167,9 +167,9 @@ If you want to utilise AZ affinity for VPC endpoints you can configure those in ]} ]} ``` -Two options supported: - - explicit lists of route53 AZ endpoints - - OS environment variable, comes handy for ECS deployments. Env should be of comma separated string like:`"myAZ1.sqs-dns.amazonaws.com,myAZ2.sqs-dns.amazonaws.com"` +Two options are supported: + - explicit list of Route53 AZ endpoints + - OS environment variable (handy for ECS deployments). The value of the variable should be of comma-separated string like `"myAZ1.sqs-dns.amazonaws.com,myAZ2.sqs-dns.amazonaws.com"` Upon config generation, `erlcloud` will check the AZ of the deployment and match it to one of the pre-configured DNS records. First match is used and if not match found default is used. From 882537f5b1fc0e46cb7d1ce97bb56da7ba79d481 Mon Sep 17 00:00:00 2001 From: Luis Rascao Date: Thu, 14 May 2020 16:16:37 +0100 Subject: [PATCH 129/310] Add record-lifecycle-action-heartbeat autoscaling function As described in the doc: https://docs.aws.amazon.com/cli/latest/reference/autoscaling/record-lifecycle-action-heartbeat.html --- src/erlcloud_as.erl | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_as.erl b/src/erlcloud_as.erl index 05734e436..c0012416b 100644 --- a/src/erlcloud_as.erl +++ b/src/erlcloud_as.erl @@ -28,7 +28,8 @@ detach_instances/2, detach_instances/3, detach_instances/4, describe_lifecycle_hooks/1, describe_lifecycle_hooks/2, describe_lifecycle_hooks/3, - complete_lifecycle_action/4, complete_lifecycle_action/5 + complete_lifecycle_action/4, complete_lifecycle_action/5, + record_lifecycle_action_heartbeat/3, record_lifecycle_action_heartbeat/4 ]). -define(API_VERSION, "2011-01-01"). @@ -78,6 +79,9 @@ -define(COMPLETE_LIFECYCLE_ACTION_ACTIVITY, "/CompleteLifecycleActionResponse/ResponseMetadata/RequestId"). +-define(RECORD_LIFECYCLE_ACTION_HEARTBEAT_ACTIVITY, + "/RecordLifecycleActionHeartbeatResponse/ResponseMetadata/RequestId"). + %% -------------------------------------------------------------------- %% @doc Calls describe_groups([], default_configuration()) @@ -695,6 +699,29 @@ complete_lifecycle_action(GroupName, LifecycleActionResult, LifecycleHookName, I {error, Reason} end. +-spec record_lifecycle_action_heartbeat(string(), string(), {instance_id | token, string()}) -> {ok, string()} | {error, term()}. +record_lifecycle_action_heartbeat(GroupName, LifecycleHookName, InstanceIdOrLifecycleActionToken) -> + record_lifecycle_action_heartbeat(GroupName, LifecycleHookName, InstanceIdOrLifecycleActionToken, erlcloud_aws:default_config()). + +-spec record_lifecycle_action_heartbeat(string(), string(), {instance_id | token, string()}, aws_config()) -> {ok, string()} | {error, term()}. +record_lifecycle_action_heartbeat(GroupName, LifecycleHookName, InstanceIdOrLifecycleActionToken, Config) -> + InstanceIdOrLifecycleActionTokenParam = case InstanceIdOrLifecycleActionToken of + {instance_id,InstanceId} -> + {"InstanceId", InstanceId}; + {token,LifecycleActionToken} -> + {"LifecycleActionToken", LifecycleActionToken} + end, + Params = [{"AutoScalingGroupName", GroupName}, + {"LifecycleHookName", LifecycleHookName}, + InstanceIdOrLifecycleActionTokenParam], + case as_query(Config, "RecordLifecycleActionHeartbeat", Params, ?API_VERSION) of + {ok, Doc} -> + RequestId = erlcloud_xml:get_text(?RECORD_LIFECYCLE_ACTION_HEARTBEAT_ACTIVITY, Doc), + {ok, RequestId}; + {error, Reason} -> + {error, Reason} + end. + %% given a list of member identifiers, return a list of %% {key with prefix, member identifier} for use in autoscaling calls. %% Example pair that could be returned in a list is From db720edec8b682079fd3b04159dd6bc260e834b0 Mon Sep 17 00:00:00 2001 From: Matteo Cafasso Date: Fri, 26 Jun 2020 20:02:55 +0300 Subject: [PATCH 130/310] Add CloudWatch Log Groups and Streams creation and deletion functions (#643) * Add erlcloud_cloudwatch_logs:create_log_stream function The `erlcloud_cloudwatch_logs:create_log_stream` allows to create a new Log Stream within a CloudWatch Log Group. Signed-off-by: Matteo Cafasso * Add erlcloud_cloudwatch_logs:create_log_stream function tests Signed-off-by: Matteo Cafasso * Add erlcloud_cloudwatch_logs:delete_log_stream function The `erlcloud_cloudwatch_logs:delete_log_stream` allows to delete a Log Stream from a CloudWatch Log Group. Signed-off-by: Matteo Cafasso * Add erlcloud_cloudwatch_logs:delete_log_stream function tests Signed-off-by: Matteo Cafasso * Add erlcloud_cloudwatch_logs:create_log_group function The `erlcloud_cloudwatch_logs:create_log_group` allows to create a new AWS CloudWatch Log Group. Signed-off-by: Matteo Cafasso * Add erlcloud_cloudwatch_logs:create_log_group function tests Signed-off-by: Matteo Cafasso * Add erlcloud_cloudwatch_logs:delete_log_group function The `erlcloud_cloudwatch_logs:delete_log_group` allows to delete a Log Group from AWS CloudWatch. Signed-off-by: Matteo Cafasso * Add erlcloud_cloudwatch_logs:delete_log_group function tests Signed-off-by: Matteo Cafasso --- src/erlcloud_cloudwatch_logs.erl | 168 +++++++++++++++++++++++- test/erlcloud_cloudwatch_logs_tests.erl | 147 ++++++++++++++++++++- 2 files changed, 302 insertions(+), 13 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 42086860b..bbbed1b45 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -30,6 +30,7 @@ -type metric_namespace() :: string() | binary() | undefined. -type log_stream_order() :: log_stream_name | last_event_time | undefined. -type events() :: [#{message => binary(), timestamp => pos_integer()}]. +-type kms_key_id() :: string() | binary() | undefined. -type success_result_paged(ObjectType) :: {ok, [ObjectType], paging_token()}. @@ -59,6 +60,20 @@ %% CloudWatch API -export([ + create_log_group/1, + create_log_group/2, + create_log_group/3, + create_log_group/4, + + create_log_stream/2, + create_log_stream/3, + + delete_log_group/1, + delete_log_group/2, + + delete_log_stream/2, + delete_log_stream/3, + describe_log_groups/0, describe_log_groups/1, describe_log_groups/2, @@ -130,6 +145,145 @@ new(AccessKeyID, SecretAccessKey, Host) -> %%============================================================================== +%%------------------------------------------------------------------------------ +%% @doc +%% +%% CreateLogGroup action +%% http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogGroup.html +%% +%% @end +%%------------------------------------------------------------------------------ +-spec create_log_group(log_group_name()) -> ok | error_result(). +create_log_group(LogGroupName) -> + create_log_group(LogGroupName, default_config()). + + +-spec create_log_group( + log_group_name(), + aws_config() +) -> ok | error_result(). +create_log_group(LogGroupName, Config) -> + create_log_group(LogGroupName, undefined, undefined, Config). + + +-spec create_log_group( + log_group_name(), + list(tag()), + aws_config() +) -> ok | error_result(). +create_log_group(LogGroupName, Tags, Config) when is_list(Tags) -> + create_log_group(LogGroupName, Tags, undefined, Config). + + +-spec create_log_group( + log_group_name(), + list(tag()), + kms_key_id(), + aws_config() +) -> ok | error_result(). +create_log_group(LogGroupName, Tags, KmsKeyId, Config) -> + case cw_request(Config, "CreateLogGroup", [ + {<<"logGroupName">>, LogGroupName}, + {<<"tags">>, Tags}, + {<<"kmsKeyId">>, KmsKeyId} + ]) + of + {ok, []} -> ok; + {error, _} = Error -> Error + end. + + +%%------------------------------------------------------------------------------ +%% @doc +%% +%% CreateLogStream action +%% http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_CreateLogStream.html +%% +%% @end +%%------------------------------------------------------------------------------ +-spec create_log_stream( + log_group_name(), + log_stream_name() +) -> ok | error_result(). +create_log_stream(LogGroupName, LogStreamName) -> + create_log_stream(LogGroupName, LogStreamName, default_config()). + + +-spec create_log_stream( + log_group_name(), + log_stream_name(), + aws_config() +) -> ok | error_result(). +create_log_stream(LogGroupName, LogStreamName, Config) -> + case cw_request(Config, "CreateLogStream", [ + {<<"logGroupName">>, LogGroupName}, + {<<"logStreamName">>, LogStreamName} + ]) + of + {ok, []} -> ok; + {error, _} = Error -> Error + end. + + +%%------------------------------------------------------------------------------ +%% @doc +%% +%% DeleteLogGroup action +%% http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_DeleteLogGroup.html +%% +%% @end +%%------------------------------------------------------------------------------ +-spec delete_log_group(log_group_name()) -> ok | error_result(). +delete_log_group(LogGroupName) -> + delete_log_group(LogGroupName, default_config()). + + +-spec delete_log_group( + log_group_name(), + aws_config() +) -> ok | error_result(). +delete_log_group(LogGroupName, Config) -> + case cw_request(Config, "DeleteLogGroup", [ + {<<"logGroupName">>, LogGroupName} + ]) + of + {ok, []} -> ok; + {error, _} = Error -> Error + end. + + +%%------------------------------------------------------------------------------ +%% @doc +%% +%% DeleteLogStream action +%% http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_DeleteLogStream.html +%% +%% @end +%%------------------------------------------------------------------------------ +-spec delete_log_stream( + log_group_name(), + log_stream_name() +) -> ok | error_result(). +delete_log_stream(LogGroupName, LogStreamName) -> + delete_log_stream(LogGroupName, LogStreamName, default_config()). + + +-spec delete_log_stream( + log_group_name(), + log_stream_name(), + aws_config() +) -> ok | error_result(). +delete_log_stream(LogGroupName, LogStreamName, Config) -> + case cw_request(Config, "DeleteLogStream", [ + {<<"logGroupName">>, LogGroupName}, + {<<"logStreamName">>, LogStreamName} + ]) + of + {ok, []} -> ok; + {error, _} = Error -> Error + end. + + %%------------------------------------------------------------------------------ %% @doc %% @@ -176,14 +330,14 @@ describe_log_groups(LogGroupNamePrefix, Limit, Config) -> aws_config() ) -> result_paged(log_group()). describe_log_groups(LogGroupNamePrefix, Limit, Token, Config) -> - case + case cw_request(Config, "DescribeLogGroups", req_log_groups(LogGroupNamePrefix, Limit, Token) ) of {ok, Json} -> LogGroups = proplists:get_value(<<"logGroups">>, Json, []), - NextToken = proplists:get_value(<<"nextToken">>, Json, undefined), + NextToken = proplists:get_value(<<"nextToken">>, Json, undefined), {ok, LogGroups, NextToken}; {error, _} = Error -> Error @@ -263,7 +417,7 @@ describe_log_streams(LogGroupName, LogStreamPrefix, OrderBy, Desc, Limit, Token, of {ok, Json} -> LogStream = proplists:get_value(<<"logStreams">>, Json, []), - NextToken = proplists:get_value(<<"nextToken">>, Json, undefined), + NextToken = proplists:get_value(<<"nextToken">>, Json, undefined), {ok, LogStream, NextToken}; {error, _} = Error -> Error @@ -290,14 +444,14 @@ log_stream_order_by(last_event_time) -> <<"LastEventTime">>. %% https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html %% %% ===Example=== -%% +%% %% Put log events requires a Upload Sequence Token, it is available via DescribeLogStreams %% %% ` -%% application:ensure_all_started(erlcloud). +%% application:ensure_all_started(erlcloud). %% {ok, Config} = erlcloud_aws:auto_config(). %% {ok, Streams, _} = erlcloud_cloudwatch_logs:describe_log_streams(GroupName, StreamName, Config). -%% {_, Seq} = lists:keyfind(<<"uploadSequenceToken">>, 1, hd(Streams)). +%% {_, Seq} = lists:keyfind(<<"uploadSequenceToken">>, 1, hd(Streams)). %% %% Batch = [#{timestamp => 1526233086694, message => <<"Example Message">>}]. %% erlcloud_cloudwatch_logs:put_logs_events(GroupName, StreamName, Seq, Batch, Config). @@ -337,7 +491,7 @@ put_logs_events(LogGroup, LogStream, SeqToken, Events, Config) -> {error, _} = Error -> Error end. - + req_logs_events(LogGroup, LogStream, SeqToken, Events) -> [ {<<"logEvents">>, log_events(Events)}, diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index f017c5ee9..aa2a1a41a 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -89,6 +89,14 @@ erlcloud_cloudwatch_test_() -> {foreach, fun start/0, fun stop/1, [ + fun create_log_group_input_test/1, + + fun create_log_stream_input_test/1, + + fun delete_log_group_input_test/1, + + fun delete_log_stream_input_test/1, + fun describe_log_groups_input_tests/1, fun describe_log_groups_output_tests/1, @@ -98,7 +106,7 @@ erlcloud_cloudwatch_test_() -> fun describe_log_streams_input_tests/1, fun describe_log_streams_output_tests/1, - fun put_logs_events_input_tests/1 + fun put_logs_events_input_tests/1 ]}. @@ -120,6 +128,133 @@ stop(_) -> %%============================================================================== +create_log_group_input_test(_) -> + input_tests(jsx:encode([]), [ + ?_cloudwatch_test( + {"Tests creating log group", + ?_f(erlcloud_cloudwatch_logs:create_log_group(?LOG_GROUP_NAME)), + [{<<"Action">>, <<"CreateLogGroup">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests creating log group with custom AWS config provided", + ?_f(erlcloud_cloudwatch_logs:create_log_group( + ?LOG_GROUP_NAME, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"CreateLogGroup">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests creating log group with AWS Tags and KMS key", + ?_f(erlcloud_cloudwatch_logs:create_log_group( + ?LOG_GROUP_NAME, + [{<<"tag_name">>, <<"tag_value">>}], + "alias/example", + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"CreateLogGroup">>}, + {<<"tags">>, [{<<"tag_name">>, <<"tag_value">>}]}, + {<<"kmsKeyId">>, <<"alias/example">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests creating log group without AWS Tags and with KMS key", + ?_f(erlcloud_cloudwatch_logs:create_log_group( + ?LOG_GROUP_NAME, + undefined, + "alias/example", + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"CreateLogGroup">>}, + {<<"kmsKeyId">>, <<"alias/example">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ) + ]). + + +delete_log_group_input_test(_) -> + input_tests(jsx:encode([]), [ + ?_cloudwatch_test( + {"Tests creating log group", + ?_f(erlcloud_cloudwatch_logs:delete_log_group(?LOG_GROUP_NAME)), + [{<<"Action">>, <<"DeleteLogGroup">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ), + ?_cloudwatch_test( + {"Tests creating log group with custom AWS config provided", + ?_f(erlcloud_cloudwatch_logs:delete_log_group( + ?LOG_GROUP_NAME, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"DeleteLogGroup">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}]} + ) + ]). + + +create_log_stream_input_test(_) -> + input_tests(jsx:encode([]), [ + ?_cloudwatch_test( + {"Tests creating log stream", + ?_f(erlcloud_cloudwatch_logs:create_log_stream( + ?LOG_GROUP_NAME, + ?LOG_STREAM_NAME + )), + [{<<"Action">>, <<"CreateLogStream">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}, + {<<"logStreamName">>, ?LOG_STREAM_NAME}]} + ), + ?_cloudwatch_test( + {"Tests creating log stream with custom AWS config provided", + ?_f(erlcloud_cloudwatch_logs:create_log_stream( + ?LOG_GROUP_NAME, + ?LOG_STREAM_NAME, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"CreateLogStream">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}, + {<<"logStreamName">>, ?LOG_STREAM_NAME}]} + ) + ]). + + +delete_log_stream_input_test(_) -> + input_tests(jsx:encode([]), [ + ?_cloudwatch_test( + {"Tests creating log stream", + ?_f(erlcloud_cloudwatch_logs:delete_log_stream( + ?LOG_GROUP_NAME, + ?LOG_STREAM_NAME + )), + [{<<"Action">>, <<"DeleteLogStream">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}, + {<<"logStreamName">>, ?LOG_STREAM_NAME}]} + ), + ?_cloudwatch_test( + {"Tests creating log stream with custom AWS config provided", + ?_f(erlcloud_cloudwatch_logs:delete_log_stream( + ?LOG_GROUP_NAME, + ?LOG_STREAM_NAME, + erlcloud_aws:default_config() + )), + [{<<"Action">>, <<"DeleteLogStream">>}, + {<<"Version">>, ?API_VERSION}, + {<<"logGroupName">>, ?LOG_GROUP_NAME}, + {<<"logStreamName">>, ?LOG_STREAM_NAME}]} + ) + ]). + + describe_log_groups_input_tests(_) -> input_tests(jsx:encode([{<<"logGroups">>, []}]), [ ?_cloudwatch_test( @@ -337,7 +472,7 @@ describe_log_streams_input_tests(_) -> ?_cloudwatch_test( {"Tests describing log streams with with log group name and stream name prefix", ?_f(erlcloud_cloudwatch_logs:describe_log_streams( - ?LOG_GROUP_NAME, + ?LOG_GROUP_NAME, ?LOG_STREAM_NAME_PREFIX, erlcloud_aws:default_config() )), @@ -353,7 +488,7 @@ describe_log_streams_input_tests(_) -> {"Tests describing log streams with with log group name, stream name prefix" "and stream sorting", ?_f(erlcloud_cloudwatch_logs:describe_log_streams( - ?LOG_GROUP_NAME, + ?LOG_GROUP_NAME, ?LOG_STREAM_NAME_PREFIX, last_event_time, true, @@ -371,7 +506,7 @@ describe_log_streams_input_tests(_) -> {"Tests describing log streams with with log group name, stream name prefix," "stream sorting and limits", ?_f(erlcloud_cloudwatch_logs:describe_log_streams( - ?LOG_GROUP_NAME, + ?LOG_GROUP_NAME, ?LOG_STREAM_NAME_PREFIX, last_event_time, true, @@ -390,7 +525,7 @@ describe_log_streams_input_tests(_) -> {"Tests describing log streams with with log group name, stream name prefix," "stream sorting, limits and page token", ?_f(erlcloud_cloudwatch_logs:describe_log_streams( - ?LOG_GROUP_NAME, + ?LOG_GROUP_NAME, ?LOG_STREAM_NAME_PREFIX, last_event_time, true, @@ -406,7 +541,7 @@ describe_log_streams_input_tests(_) -> {<<"logStreamNamePrefix">>, ?LOG_STREAM_NAME_PREFIX}, {<<"nextToken">>, ?PAGING_TOKEN}, {<<"orderBy">>,<<"LastEventTime">>}]} - ) + ) ]). From e4d903c993b41d8338cce0f973a0353414ee2bfc Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Mon, 29 Jun 2020 11:25:58 -0500 Subject: [PATCH 131/310] Make use of jsx compatibe with 3.x [jsx](https://github.com/talentdeficit/jsx) moved to [v3.0.0](https://github.com/talentdeficit/jsx/releases/tag/v3.0.0) on Friday, 26 June 2020. See [jsx#139](https://github.com/talentdeficit/jsx/pull/139). The big change is that the default option is that `return_maps` now defaults to `true`, meaning that you will get maps output **unless** you explicitly pass `[{return_maps, false}]` when decoding JSON. This change updates internal use of jsx to *always* pass `return_maps` explicitly, true or false, so that it is compatible with jsx [v3.0.0](https://github.com/talentdeficit/jsx/releases/tag/v3.0.0). I also updated the pin of jsx to [v2.11.0](https://github.com/talentdeficit/jsx/releases/tag/v2.11.0), since I don't want to force 3.x onto clients just yet. --- rebar.config | 2 +- rebar.config.script | 2 +- rebar.lock | 4 ++-- src/erlcloud_application_autoscaler.erl | 2 +- src/erlcloud_aws.erl | 6 +++--- src/erlcloud_cloudsearch.erl | 4 ++-- src/erlcloud_cloudtrail.erl | 2 +- src/erlcloud_cloudwatch_logs.erl | 2 +- src/erlcloud_ddb_impl.erl | 4 ++-- src/erlcloud_ddb_streams.erl | 4 ++-- src/erlcloud_directconnect.erl | 2 +- src/erlcloud_ecs.erl | 2 +- src/erlcloud_emr.erl | 4 ++-- src/erlcloud_guardduty.erl | 2 +- src/erlcloud_inspector.erl | 2 +- src/erlcloud_kinesis_impl.erl | 4 ++-- src/erlcloud_kms.erl | 2 +- src/erlcloud_lambda.erl | 2 +- src/erlcloud_mes.erl | 2 +- src/erlcloud_mms.erl | 2 +- src/erlcloud_sns.erl | 4 ++-- src/erlcloud_waf.erl | 2 +- test/erlcloud_cloudtrail_tests.erl | 22 ++++++++++++---------- test/erlcloud_cloudwatch_logs_tests.erl | 2 +- test/erlcloud_ddb2_tests.erl | 4 ++-- test/erlcloud_ddb_streams_tests.erl | 4 ++-- test/erlcloud_ddb_tests.erl | 4 ++-- test/erlcloud_directconnect_tests.erl | 4 ++-- test/erlcloud_ecs_tests.erl | 4 ++-- test/erlcloud_emr_tests.erl | 2 +- test/erlcloud_inspector_tests.erl | 6 +++--- test/erlcloud_kinesis_tests.erl | 17 ++++++++++------- test/erlcloud_kms_tests.erl | 4 ++-- test/erlcloud_mes_tests.erl | 11 +++++++---- test/erlcloud_mms_tests.erl | 13 ++++++++----- test/erlcloud_waf_tests.erl | 4 ++-- 36 files changed, 87 insertions(+), 76 deletions(-) diff --git a/rebar.config b/rebar.config index bed306cef..34480cff9 100644 --- a/rebar.config +++ b/rebar.config @@ -17,7 +17,7 @@ warn_unused_vars]}. {deps, [ - {jsx, "2.9.0"}, + {jsx, "2.11.0"}, {lhttpc, "1.6.2"}, {eini, "1.2.7"}, {base16, "1.0.0"} diff --git a/rebar.config.script b/rebar.config.script index d729da63c..5eaa2fc34 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -6,7 +6,7 @@ case erlang:function_exported(rebar3, main, 1) of %% Use git-based deps %% profiles [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.13"}}}, - {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "2.9.0"}}}, + {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "v2.11.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.7"}}}, {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, diff --git a/rebar.lock b/rebar.lock index ec2248c8e..134d2c922 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,12 +1,12 @@ {"1.1.0", [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.7">>},0}, - {<<"jsx">>,{pkg,<<"jsx">>,<<"2.9.0">>},0}, + {<<"jsx">>,{pkg,<<"jsx">>,<<"2.11.0">>},0}, {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.6.2">>},0}]}. [ {pkg_hash,[ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, {<<"eini">>, <<"EFC9D836E88591A47550BD34CE964E21CA1369F8716B24F73CFEA513FA99F666">>}, - {<<"jsx">>, <<"D2F6E5F069C00266CAD52FB15D87C428579EA4D7D73A33669E12679E203329DD">>}, + {<<"jsx">>, <<"08154624050333919B4AC1B789667D5F4DB166DC50E190C4D778D1587F102EE0">>}, {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} ]. diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 3679a6568..c22f0c4f1 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -702,7 +702,7 @@ request_with_action(Configuration, BodyConfiguration, Action) -> Headers = [{"content-type", "application/x-amz-json-1.1"} | HeadersPrev], Request = prepare_record(Config, post, Headers, Body), Response = erlcloud_retry:request(Config, Request, fun aas_result_fun/1), - {ok, jsx:decode(Response#aws_request.response_body)}; + {ok, jsx:decode(Response#aws_request.response_body, [{return_maps, false}])}; {error, Reason} -> {error, Reason} end. diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 6d90fc36a..91dd8343c 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -900,7 +900,7 @@ get_credentials_from_metadata(Config) -> {error, Reason} -> {error, Reason}; {ok, Json} -> - Creds = jsx:decode(Json), + Creds = jsx:decode(Json, [{return_maps, false}]), get_credentials_from_metadata_xform( Creds ) end end. @@ -913,7 +913,7 @@ get_credentials_from_task_metadata(Config) -> {error, Reason} -> {error, Reason}; {ok, Json} -> - Creds = jsx:decode(Json), + Creds = jsx:decode(Json, [{return_maps, false}]), get_credentials_from_metadata_xform( Creds ) end. @@ -1159,7 +1159,7 @@ get_service_status(ServiceNames) when is_list(ServiceNames) -> "/data.json", "", [], default_config()), case get_filtered_statuses(ServiceNames, - proplists:get_value(<<"current">>, jsx:decode(Json))) + proplists:get_value(<<"current">>, jsx:decode(Json, [{return_maps, false}]))) of [] -> ok; ReturnStatuses -> ReturnStatuses diff --git a/src/erlcloud_cloudsearch.erl b/src/erlcloud_cloudsearch.erl index 9ed597885..a01b8aba9 100644 --- a/src/erlcloud_cloudsearch.erl +++ b/src/erlcloud_cloudsearch.erl @@ -721,7 +721,7 @@ cloudsearch_query(Config, Action, Params, ApiVersion) -> [{"Accept", "application/json"}], Config) of {ok, Response} -> - {ok, jsx:decode(Response)}; + {ok, jsx:decode(Response, [{return_maps, false}])}; {error, Reason} -> {error, Reason} end. @@ -736,7 +736,7 @@ cloudsearch_post_json(Host, Path, Body, [{"content-type", "application/json"} | Headers], Config) of {ok, RespBody} -> - {ok, jsx:decode(RespBody)}; + {ok, jsx:decode(RespBody, [{return_maps, false}])}; {error, Reason} -> {error, Reason} end. diff --git a/src/erlcloud_cloudtrail.erl b/src/erlcloud_cloudtrail.erl index 6f4327b7a..33cdbf8a1 100644 --- a/src/erlcloud_cloudtrail.erl +++ b/src/erlcloud_cloudtrail.erl @@ -205,7 +205,7 @@ request_impl(Method, Scheme, Host, Port, Path, Operation, Params, Body, #aws_con {ok, RespBody} -> case Config#aws_config.cloudtrail_raw_result of true -> {ok, RespBody}; - _ -> {ok, jsx:decode(RespBody)} + _ -> {ok, jsx:decode(RespBody, [{return_maps, false}])} end; {error, Reason} -> {error, Reason} diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index bbbed1b45..43153fe46 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -695,7 +695,7 @@ maybe_cw_request({error, _} = Error, _Action, _Params) -> maybe_json({ok, <<>>}) -> {ok, []}; maybe_json({ok, Response}) -> - {ok, jsx:decode(Response)}; + {ok, jsx:decode(Response, [{return_maps, false}])}; maybe_json({error, _} = Error) -> Error. diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 79bc30150..1e3f9b02f 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -193,7 +193,7 @@ request_and_retry(Config, Headers, Body, {attempt, Attempt}) -> {ok, {{200, _}, _, RespBody}} -> %% TODO check crc - {ok, jsx:decode(RespBody)}; + {ok, jsx:decode(RespBody, [{return_maps, false}])}; Error -> DDBError = #ddb2_error{attempt = Attempt, @@ -230,7 +230,7 @@ client_error(Body, DDBError) -> false -> DDBError#ddb2_error{error_type = http, should_retry = false}; true -> - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), case proplists:get_value(<<"__type">>, Json) of undefined -> DDBError#ddb2_error{error_type = http, should_retry = false}; diff --git a/src/erlcloud_ddb_streams.erl b/src/erlcloud_ddb_streams.erl index d7cdd73a8..e61c0f041 100644 --- a/src/erlcloud_ddb_streams.erl +++ b/src/erlcloud_ddb_streams.erl @@ -829,7 +829,7 @@ request2(Config, Operation, Json) -> request_to_return(#aws_request{response_type = ok, response_body = Body}) -> %% TODO check crc - {ok, jsx:decode(Body)}; + {ok, jsx:decode(Body, [{return_maps, false}])}; request_to_return(#aws_request{response_type = error, error_type = aws, httpc_error_reason = undefined, @@ -873,7 +873,7 @@ client_error(#aws_request{response_body = Body} = Request) -> false -> Request#aws_request{should_retry = false}; true -> - Json = jsx:decode(Body), + Json = jsx:decode(Body, [{return_maps, false}]), case proplists:get_value(<<"__type">>, Json) of undefined -> Request#aws_request{should_retry = false}; diff --git a/src/erlcloud_directconnect.erl b/src/erlcloud_directconnect.erl index eb3beaf8f..149df41e1 100644 --- a/src/erlcloud_directconnect.erl +++ b/src/erlcloud_directconnect.erl @@ -416,7 +416,7 @@ dc_query(Operation, Params, Config) -> [{<<"content-type">>, <<"application/x-amz-json-1.1">>} | Headers], Body, 1000, Config)) of {ok, {_RespHeader, RespBody}} -> - {ok, jsx:decode(RespBody)}; + {ok, jsx:decode(RespBody, [{return_maps, false}])}; {error, Reason} -> {error, Reason} end. diff --git a/src/erlcloud_ecs.erl b/src/erlcloud_ecs.erl index f8b048234..a46dbf798 100644 --- a/src/erlcloud_ecs.erl +++ b/src/erlcloud_ecs.erl @@ -2466,7 +2466,7 @@ ecs_request_no_update(Config, Operation, Body) -> request_body = Payload}, case erlcloud_aws:request_to_return(erlcloud_retry:request(Config, Request, fun ecs_result_fun/1)) of {ok, {_RespHeaders, <<>>}} -> {ok, []}; - {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody)}; + {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody, [{return_maps, false}])}; {error, _} = Error-> Error end. diff --git a/src/erlcloud_emr.erl b/src/erlcloud_emr.erl index f874e714a..936d8ffbe 100644 --- a/src/erlcloud_emr.erl +++ b/src/erlcloud_emr.erl @@ -207,11 +207,11 @@ request_no_update(Action, Json, Scheme, Host, Port, Service, Opts, Cfg) -> raw -> {ok, Body}; _ -> case Body of <<>> -> {ok, <<>>}; - _ -> {ok, jsx:decode(Body)} + _ -> {ok, jsx:decode(Body, [{return_maps, false}])} end end; {error, {http_error, _Code, _StatusLine, ErrBody}} -> - {error, {aws_error, jsx:decode(ErrBody)}}; + {error, {aws_error, jsx:decode(ErrBody, [{return_maps, false}])}}; {error, {socket_error, Reason}} -> {error, {socket_error, Reason}} end. diff --git a/src/erlcloud_guardduty.erl b/src/erlcloud_guardduty.erl index f3eb6a75c..834e7e4b8 100644 --- a/src/erlcloud_guardduty.erl +++ b/src/erlcloud_guardduty.erl @@ -117,7 +117,7 @@ guardduty_request_no_update(Config, Method, Path, Body, QParam) -> Method, Config#aws_config.guardduty_scheme, Config#aws_config.guardduty_host, Config#aws_config.guardduty_port, Path, Form, Headers, Config) of {ok, Data} -> - {ok, jsx:decode(Data)}; + {ok, jsx:decode(Data, [{return_maps, false}])}; E -> E end. diff --git a/src/erlcloud_inspector.erl b/src/erlcloud_inspector.erl index abd47e9b2..956d8e983 100644 --- a/src/erlcloud_inspector.erl +++ b/src/erlcloud_inspector.erl @@ -1109,7 +1109,7 @@ inspector_request_no_update(Config, Operation, Body) -> request_body = Payload}, case erlcloud_aws:request_to_return(erlcloud_retry:request(Config, Request, fun inspector_result_fun/1)) of {ok, {_RespHeaders, <<>>}} -> {ok, []}; - {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody)}; + {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody, [{return_maps, false}])}; {error, _} = Error-> Error end. diff --git a/src/erlcloud_kinesis_impl.erl b/src/erlcloud_kinesis_impl.erl index 9cdd2fcd8..e9af4bd1c 100644 --- a/src/erlcloud_kinesis_impl.erl +++ b/src/erlcloud_kinesis_impl.erl @@ -141,7 +141,7 @@ request_and_retry(Config, Headers, Body, ShouldDecode, {attempt, Attempt}) -> -spec client_error(pos_integer(), string(), binary()) -> {retry, term()} | {error, term()}. client_error(Status, StatusLine, Body) -> - try jsx:decode(Body) of + try jsx:decode(Body, [{return_maps, false}]) of Json -> Message = proplists:get_value(<<"message">>, Json, <<>>), case proplists:get_value(<<"__type">>, Json) of @@ -176,4 +176,4 @@ port_spec(#aws_config{kinesis_port=Port}) -> [":", erlang:integer_to_list(Port)]. decode(<<>>) -> []; -decode(JSON) -> jsx:decode(JSON). +decode(JSON) -> jsx:decode(JSON, [{return_maps, false}]). diff --git a/src/erlcloud_kms.erl b/src/erlcloud_kms.erl index a3141b932..6523b45bb 100644 --- a/src/erlcloud_kms.erl +++ b/src/erlcloud_kms.erl @@ -1011,7 +1011,7 @@ kms_request_no_update(Config, Operation, Body) -> request_body = Payload}, case erlcloud_aws:request_to_return(erlcloud_retry:request(Config, Request, fun kms_result_fun/1)) of {ok, {_RespHeaders, <<>>}} -> {ok, []}; - {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody)}; + {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody, [{return_maps, false}])}; {error, _} = Error-> Error end. diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index 0a76fd943..ec7d1cdf7 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -760,7 +760,7 @@ decode_body(Body, true) -> decode_body(<<>>, _RawBody) -> []; decode_body(BinData, _RawBody) -> - jsx:decode(BinData). + jsx:decode(BinData, [{return_maps, false}]). encode_body(Bin) when is_binary(Bin) -> Bin; diff --git a/src/erlcloud_mes.erl b/src/erlcloud_mes.erl index f9d031d1e..a2e8fe4b8 100644 --- a/src/erlcloud_mes.erl +++ b/src/erlcloud_mes.erl @@ -171,7 +171,7 @@ mes_request_no_update(#aws_config{mes_scheme = Scheme, mes_host = Host, mes_port Headers = headers(Config, Operation, Body), case erlcloud_aws:aws_request_form_raw(post, Scheme, Host, Port, "/", Body, Headers, Config) of {ok, Response} -> - {ok, jsx:decode(Response)}; + {ok, jsx:decode(Response, [{return_maps, false}])}; {error, Reason} -> {error, Reason} end. diff --git a/src/erlcloud_mms.erl b/src/erlcloud_mms.erl index 9b087b29f..adc937baf 100644 --- a/src/erlcloud_mms.erl +++ b/src/erlcloud_mms.erl @@ -170,7 +170,7 @@ mms_request_no_update(#aws_config{mms_scheme = Scheme, mms_host = Host, mms_port Headers = headers(Config, Operation, Body), case erlcloud_aws:aws_request_form_raw(post, Scheme, Host, Port, "/", Body, Headers, Config) of {ok, Response} -> - {ok, jsx:decode(Response)}; + {ok, jsx:decode(Response, [{return_maps, false}])}; {error, Reason} -> {error, Reason} end. diff --git a/src/erlcloud_sns.erl b/src/erlcloud_sns.erl index 7e348bc5b..d073e2830 100644 --- a/src/erlcloud_sns.erl +++ b/src/erlcloud_sns.erl @@ -525,7 +525,7 @@ publish(Type, RecipientArn, Message, Subject, Attributes, Config) -> -spec parse_event(iodata()) -> sns_event(). parse_event(EventSource) -> - jsx:decode(EventSource). + jsx:decode(EventSource, [{return_maps, false}]). -spec get_event_type(sns_event()) -> sns_event_type(). get_event_type(Event) -> @@ -539,7 +539,7 @@ parse_event_message(Event) -> Message = proplists:get_value(<<"Message">>, Event, <<>>), case get_event_type(Event) of subscription_confirmation -> Message; - notification -> jsx:decode(Message) + notification -> jsx:decode(Message, [{return_maps, false}]) end. -spec get_notification_attribute(binary(), sns_notification()) -> sns_application_attribute() | binary(). diff --git a/src/erlcloud_waf.erl b/src/erlcloud_waf.erl index a52e1cb07..cd5c25094 100644 --- a/src/erlcloud_waf.erl +++ b/src/erlcloud_waf.erl @@ -1186,7 +1186,7 @@ waf_request_no_update(Config, Operation, Body) -> request_body = Payload}, case erlcloud_aws:request_to_return(erlcloud_retry:request(Config, Request, fun waf_result_fun/1)) of {ok, {_RespHeaders, <<>>}} -> {ok, []}; - {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody)}; + {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody, [{return_maps, false}])}; {error, _} = Error-> Error end. diff --git a/test/erlcloud_cloudtrail_tests.erl b/test/erlcloud_cloudtrail_tests.erl index bf5284377..581966fa9 100644 --- a/test/erlcloud_cloudtrail_tests.erl +++ b/test/erlcloud_cloudtrail_tests.erl @@ -71,8 +71,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(list_to_binary(Expected))), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(decode(list_to_binary(Expected))), + Actual = sort_json(decode(Body)), case Want =:= Actual of true -> ok; false -> @@ -189,7 +189,7 @@ create_trail_output_tests(_) -> Tests = [?_cloudtrail_test( {"CreateTrail example response", Response, - {ok, jsx:decode(Response)}}) + {ok, decode(Response)}}) ], output_tests(?_f(erlcloud_cloudtrail:create_trail("test", "test_bucket", "test_prefix", "test_topic", true, erlcloud_aws:default_config())), Tests). @@ -216,7 +216,7 @@ delete_trail_output_tests(_) -> Tests = [?_cloudtrail_test( {"DeleteTrail example response", Response, - {ok, jsx:decode(Response)}}) + {ok, decode(Response)}}) ], output_tests(?_f(erlcloud_cloudtrail:delete_trail("test", erlcloud_aws:default_config())), Tests). @@ -242,7 +242,7 @@ start_logging_output_tests(_) -> Tests = [?_cloudtrail_test( {"StartLogging example response", Response, - {ok, jsx:decode(Response)}}) + {ok, decode(Response)}}) ], output_tests(?_f(erlcloud_cloudtrail:start_logging("test", erlcloud_aws:default_config())), Tests). @@ -268,7 +268,7 @@ stop_logging_output_tests(_) -> Tests = [?_cloudtrail_test( {"StopLogging example response", Response, - {ok, jsx:decode(Response)}}) + {ok, decode(Response)}}) ], output_tests(?_f(erlcloud_cloudtrail:stop_logging("test", erlcloud_aws:default_config())), Tests). @@ -303,7 +303,7 @@ describe_trails_output_tests(_) -> Tests = [?_cloudtrail_test( {"DescribeTrails example response", Response, - {ok, jsx:decode(Response)}}) + {ok, decode(Response)}}) ], output_tests(?_f(erlcloud_cloudtrail:describe_trails(["test"], erlcloud_aws:default_config())), Tests). @@ -341,7 +341,7 @@ get_event_selectors_output_tests(_) -> Tests = [?_cloudtrail_test( {"GetEventSelectors example response", Response, - {ok, jsx:decode(Response)}}) + {ok, decode(Response)}}) ], output_tests(?_f(erlcloud_cloudtrail:get_event_selectors("test", erlcloud_aws:default_config())), Tests). @@ -376,7 +376,7 @@ get_trail_status_output_tests(_) -> Tests = [?_cloudtrail_test( {"GetTrailStatus example response", Response, - {ok, jsx:decode(Response)}}) + {ok, decode(Response)}}) ], output_tests(?_f(erlcloud_cloudtrail:get_trail_status("test", erlcloud_aws:default_config())), Tests). @@ -412,8 +412,10 @@ update_trail_output_tests(_) -> Tests = [?_cloudtrail_test( {"UpdateTrail example response", Response, - {ok, jsx:decode(Response)}}) + {ok, decode(Response)}}) ], output_tests(?_f(erlcloud_cloudtrail:update_trail("test", "test_bucket", "test_prefix", "test_topic", true, erlcloud_aws:default_config())), Tests). +decode(S) -> + jsx:decode(S, [{return_maps, false}]). diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index aa2a1a41a..6599842d2 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -592,7 +592,7 @@ input_test(ResponseBody, {Line, {Description, Fun, ExpectedParams}}) -> erlcloud_httpc, request, fun(_Url, post, _Headers, RequestBody, _Timeout, _Config) -> - ActualParams = jsx:decode(RequestBody), + ActualParams = jsx:decode(RequestBody, [{return_maps, false}]), ?assertEqual(sort_json(ExpectedParams), sort_json(ActualParams)), {ok, {{200, "OK"}, [], ResponseBody}} end diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index acd39097f..a1af991c9 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -136,8 +136,8 @@ validate_body(<<>> = Actual, Want) -> ?debugFmt("~nEXPECTED~n~p~nACTUAL~n~p~n", [Want, Actual]), ?assertEqual(Want, Actual); validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(list_to_binary(Expected))), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(jsx:decode(list_to_binary(Expected), [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), case Want =:= Actual of true -> ok; false -> diff --git a/test/erlcloud_ddb_streams_tests.erl b/test/erlcloud_ddb_streams_tests.erl index 6b86cb040..98150d2e3 100644 --- a/test/erlcloud_ddb_streams_tests.erl +++ b/test/erlcloud_ddb_streams_tests.erl @@ -56,8 +56,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(list_to_binary(Expected))), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(jsx:decode(list_to_binary(Expected), [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), case Want =:= Actual of true -> ok; false -> diff --git a/test/erlcloud_ddb_tests.erl b/test/erlcloud_ddb_tests.erl index 15fd70a5a..dd194d0d3 100644 --- a/test/erlcloud_ddb_tests.erl +++ b/test/erlcloud_ddb_tests.erl @@ -73,8 +73,8 @@ stop(_) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = jsx:decode(list_to_binary(Expected)), - Actual = jsx:decode(Body), + Want = jsx:decode(list_to_binary(Expected), [{return_maps, false}]), + Actual = jsx:decode(Body, [{return_maps, false}]), case Want =:= Actual of true -> ok; false -> diff --git a/test/erlcloud_directconnect_tests.erl b/test/erlcloud_directconnect_tests.erl index 1ee4fcb8d..a57c4d9a3 100644 --- a/test/erlcloud_directconnect_tests.erl +++ b/test/erlcloud_directconnect_tests.erl @@ -59,8 +59,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(list_to_binary(Expected))), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(jsx:decode(list_to_binary(Expected), [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), case Want =:= Actual of true -> ok; false -> diff --git a/test/erlcloud_ecs_tests.erl b/test/erlcloud_ecs_tests.erl index 93a06538e..ef03b7c18 100644 --- a/test/erlcloud_ecs_tests.erl +++ b/test/erlcloud_ecs_tests.erl @@ -3240,8 +3240,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(list_to_binary(Expected))), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(jsx:decode(list_to_binary(Expected), [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), case Want =:= Actual of true -> ok; false -> diff --git a/test/erlcloud_emr_tests.erl b/test/erlcloud_emr_tests.erl index 30f76d0d7..3354158e3 100644 --- a/test/erlcloud_emr_tests.erl +++ b/test/erlcloud_emr_tests.erl @@ -189,7 +189,7 @@ input_test(ResponseBody, {Line, {Description, Fun, ExpectedParams}}) -> erlcloud_httpc, request, fun(_Url, post, _Headers, RequestBody, _Timeout, _Config) -> - ActualParams = jsx:decode(RequestBody), + ActualParams = jsx:decode(RequestBody, [{return_maps, false}]), ?assertEqual(sort_json(ExpectedParams), sort_json(ActualParams)), {ok, {{200, "OK"}, [], ResponseBody}} end diff --git a/test/erlcloud_inspector_tests.erl b/test/erlcloud_inspector_tests.erl index 653ded4b8..e438d6949 100644 --- a/test/erlcloud_inspector_tests.erl +++ b/test/erlcloud_inspector_tests.erl @@ -514,8 +514,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(Expected)), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(jsx:decode(Expected, [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), case Want =:= Actual of true -> ok; false -> @@ -602,4 +602,4 @@ all_tests(Action, Function, PostData, Response) -> )], input_tests(<<>>, InputTests) ++ - output_tests(Function, OutputTests). \ No newline at end of file + output_tests(Function, OutputTests). diff --git a/test/erlcloud_kinesis_tests.erl b/test/erlcloud_kinesis_tests.erl index dbed58eb3..c317dfb9d 100644 --- a/test/erlcloud_kinesis_tests.erl +++ b/test/erlcloud_kinesis_tests.erl @@ -99,8 +99,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(list_to_binary(Expected))), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(decode(list_to_binary(Expected))), + Actual = sort_json(decode(Body)), case Want =:= Actual of true -> ok; false -> @@ -195,7 +195,7 @@ create_stream_output_tests(_) -> Tests = [?_kinesis_test( {"CreateStream example response", "{}", - {ok, jsx:decode(<<"{}">>)}}) + {ok, decode(<<"{}">>)}}) ], output_tests(?_f(erlcloud_kinesis:create_stream(<<"streamName">>, 2)), Tests). @@ -220,7 +220,7 @@ delete_stream_output_tests(_) -> Tests = [?_kinesis_test( {"DeleteStream example response", "{}", - {ok, jsx:decode(<<"{}">>)}}) + {ok, decode(<<"{}">>)}}) ], output_tests(?_f(erlcloud_kinesis:delete_stream(<<"streamName">>)), Tests). @@ -954,7 +954,7 @@ merge_shards_output_tests(_) -> Tests = [?_kinesis_test( {"MergeShards example response", "{}", - {ok, jsx:decode(<<"{}">>)}}) + {ok, decode(<<"{}">>)}}) ], output_tests(?_f(erlcloud_kinesis:merge_shards(<<"test">>, <<"shardId-000000000001">>, <<"shardId-000000000003">>)), Tests). @@ -981,7 +981,7 @@ split_shards_output_tests(_) -> Tests = [?_kinesis_test( {"SplitShard example response", "{}", - {ok, jsx:decode(<<"{}">>)}}) + {ok, decode(<<"{}">>)}}) ], output_tests(?_f(erlcloud_kinesis:split_shards(<<"test">>, <<"shardId-000000000000">>, <<"10">>)), Tests). @@ -1028,7 +1028,7 @@ list_tags_for_stream_output_tests(_) -> \"Tags\": [{\"Key\":\"key1\",\"Value\":\"val1\"}]}", Tests = [?_kinesis_test({"List tags response test", Response, - {ok, jsx:decode(list_to_binary(Response))}})], + {ok, decode(list_to_binary(Response))}})], output_tests( ?_f(erlcloud_kinesis:list_tags_for_stream(<<"stream">>, <<"key1">>, 1)), Tests @@ -1076,3 +1076,6 @@ remove_tags_from_stream_output_tests(_) -> [<<"key">>])), Tests ). + +decode(S) -> + jsx:decode(S, [{return_maps, false}]). diff --git a/test/erlcloud_kms_tests.erl b/test/erlcloud_kms_tests.erl index abbe41ee4..76fce69f7 100644 --- a/test/erlcloud_kms_tests.erl +++ b/test/erlcloud_kms_tests.erl @@ -116,8 +116,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(Expected)), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(jsx:decode(Expected, [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), case Want =:= Actual of true -> ok; false -> diff --git a/test/erlcloud_mes_tests.erl b/test/erlcloud_mes_tests.erl index 73863ecaa..d8203cfd2 100644 --- a/test/erlcloud_mes_tests.erl +++ b/test/erlcloud_mes_tests.erl @@ -57,8 +57,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(list_to_binary(Expected))), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(decode(list_to_binary(Expected))), + Actual = sort_json(decode(Body)), case Want =:= Actual of true -> ok; false -> @@ -250,7 +250,7 @@ get_entitlement_output_tests(_) -> } ] }", - {ok, jsx:decode(<<" + {ok, decode(<<" { \"Entitlements\": [ { @@ -283,7 +283,7 @@ get_entitlement_output_tests(_) -> } ] }", - {ok, jsx:decode(<<" + {ok, decode(<<" { \"Entitlements\": [ { @@ -308,3 +308,6 @@ get_entitlement_output_tests(_) -> <<"string">>, [{customer_identifier, [<<"string">>]}, {dimension, [<<"string">>]}], [{next_token, <<"token">>}])), Tests). + +decode(S) -> + jsx:decode(S, [{return_maps, false}]). diff --git a/test/erlcloud_mms_tests.erl b/test/erlcloud_mms_tests.erl index e1ead2941..e0eb8d1c1 100644 --- a/test/erlcloud_mms_tests.erl +++ b/test/erlcloud_mms_tests.erl @@ -61,8 +61,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(list_to_binary(Expected))), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(decode(list_to_binary(Expected))), + Actual = sort_json(decode(Body)), case Want =:= Actual of true -> ok; false -> @@ -213,7 +213,7 @@ batch_meter_usage_output_tests(_) -> \"Timestamp\": 1471959107 } ] }", - {ok, jsx:decode(<<" + {ok, decode(<<" { \"Results\": [ { @@ -281,7 +281,7 @@ meter_usage_output_tests(_) -> Tests = [?_mms_test( {"MeterUsage example response", "{\"MeteringRecordId\": \"string\"}", - {ok, jsx:decode(<<"{\"MeteringRecordId\": \"string\"}">>)}}) + {ok, decode(<<"{\"MeteringRecordId\": \"string\"}">>)}}) ], output_tests(?_f(erlcloud_mms:meter_usage( @@ -319,7 +319,10 @@ resolve_customer_output_tests(_) -> \"CustomerIdentifier\": \"string\", \"ProductCode\": \"string\" }", - {ok,jsx:decode(<<"{\"CustomerIdentifier\": \"string\",\"ProductCode\": \"string\"}">>)}}) + {ok, decode(<<"{\"CustomerIdentifier\": \"string\",\"ProductCode\": \"string\"}">>)}}) ], output_tests(?_f(erlcloud_mms:resolve_customer(<<"string">>)), Tests). + +decode(S) -> + jsx:decode(S, [{return_maps, false}]). diff --git a/test/erlcloud_waf_tests.erl b/test/erlcloud_waf_tests.erl index 533096566..5e81ecebf 100644 --- a/test/erlcloud_waf_tests.erl +++ b/test/erlcloud_waf_tests.erl @@ -608,8 +608,8 @@ sort_json(V) -> %% verifies that the parameters in the body match the expected parameters -spec validate_body(binary(), expected_body()) -> ok. validate_body(Body, Expected) -> - Want = sort_json(jsx:decode(Expected)), - Actual = sort_json(jsx:decode(Body)), + Want = sort_json(jsx:decode(Expected, [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), case Want =:= Actual of true -> ok; false -> From 4f967ac0759c7f8e106a83cb2943f1dbd1ad0fbe Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Mon, 29 Jun 2020 13:21:14 -0500 Subject: [PATCH 132/310] Add OTP 22 to supported releases * And bump OTP 21 support from 21.1 to 21.3 --- .travis.yml | 3 ++- README.md | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index c8bca2286..61d8f9eef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ language: erlang otp_release: - 19.3 - 20.3 - - 21.1.3 + - 21.3 + - 22.3 env: global: - MAIN_OTP=20.3 diff --git a/README.md b/README.md index 62f72881a..6683bcd91 100644 --- a/README.md +++ b/README.md @@ -68,9 +68,10 @@ Below is the library roadmap update along with regular features and fixes. At the moment we support the following OTP releases: - 19.3 - 20.3 - - 21.1 + - 21.3 + - 22.3 -it might still work on 17+ (primariliy due to Erlang maps) but we do not guarantee that. +it might still work on 17+ (primarily due to Erlang maps) but we do not guarantee that. ## Getting started ## You need to clone the repository and download rebar/rebar3 (if it's not already available in your path). From 63aae4399550535eb1540004936e5f9bbbd1fe77 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 1 Jul 2020 00:26:56 +0100 Subject: [PATCH 133/310] Fix dialyzer-issued warning --- src/erlcloud_cloudwatch_logs.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 43153fe46..b14202ffb 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -177,8 +177,8 @@ create_log_group(LogGroupName, Tags, Config) when is_list(Tags) -> -spec create_log_group( log_group_name(), - list(tag()), - kms_key_id(), + undefined | list(tag()), + undefined | kms_key_id(), aws_config() ) -> ok | error_result(). create_log_group(LogGroupName, Tags, KmsKeyId, Config) -> From ead30d0ab10d19514c2e3e2fbdcf723ce5a7440c Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 1 Jul 2020 00:27:34 +0100 Subject: [PATCH 134/310] "Prevent" further dialyzer -related issues from creeping into the project --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 61d8f9eef..d49c254ad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,7 @@ install: - make travis-install script: - make check_warnings + - make check - make eunit deploy: skip_cleanup: true From 508eb21a1546a266ff5a684e16be060f29f09823 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 1 Jul 2020 01:16:29 +0100 Subject: [PATCH 135/310] Allow `make check` to run with rebar 2 (since this is part of the .travis.yml matrix) Target .dialyzer_plt prevents the existing PLT from being constantly rebuilt --- .gitignore | 1 + Makefile | 21 +++++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 920c52d3f..758520476 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ eunit.coverage.xml ercloud.iml *[#]*[#] *[#]* +.dialyzer_plt diff --git a/Makefile b/Makefile index 9399206a4..17dd0fcdf 100644 --- a/Makefile +++ b/Makefile @@ -57,23 +57,32 @@ else @$(REBAR) eunit endif +.dialyzer_plt: + dialyzer --build_plt -r deps \ + --apps erts kernel stdlib inets crypto public_key ssl xmerl \ + --fullpath \ + --output_plt .dialyzer_plt + check: ifeq ($(REBAR_VSN),2) @$(REBAR) compile - dialyzer --verbose --no_check_plt --no_native --fullpath \ + $(MAKE) .dialyzer_plt + dialyzer --no_check_plt --fullpath \ $(CHECK_FILES) \ - -Wunmatched_returns \ - -Werror_handling + -I include \ + --plt .dialyzer_plt else @$(REBAR) dialyzer endif check-eunit: eunit ifeq ($(REBAR_VSN),2) - dialyzer --verbose --no_check_plt --no_native --fullpath \ + @$(REBAR) compile + $(MAKE) .dialyzer_plt + dialyzer --no_check_plt --fullpath \ $(CHECK_EUNIT_FILES) \ - -Wunmatched_returns \ - -Werror_handling + -I include \ + --plt .dialyzer_plt else @$(REBAR) dialyzer endif From 3595174d5754c1dc0309dcba88f5d3d8251310eb Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 1 Jul 2020 01:32:32 +0100 Subject: [PATCH 136/310] Fix "unknown function" issue (`string:split/3` is OTP 20+ only) --- src/erlcloud_aws.erl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 91dd8343c..9577977f9 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -819,7 +819,7 @@ get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> %% resolve through ENV if any Endpoints = case ConfiguredEndpoints of {env, EnvVarName} when is_list(EnvVarName) -> - Es = string:split(os:getenv(EnvVarName, ""), ",", all), + Es = string_split(os:getenv(EnvVarName, ""), ","), % ignore "" env var or ",," cases [list_to_binary(E) || E <- Es, E /= ""]; EndpointsList when is_list(EndpointsList) -> @@ -828,6 +828,17 @@ get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> % now match our AZ to configured ones pick_vpc_endpoint(Endpoints, Default). +-ifdef(AT_LEAST_20). +string_split(String, Char) -> + string:split(String, Char, all). +-else. +string_split(String, Char) -> + Subject = list_to_binary(String), + Pattern = list_to_binary(Char), + Options = [global], + [binary_to_list(Elem) || Elem <- binary:split(Subject, Pattern, Options)]. +-endif. + pick_vpc_endpoint([], Default) -> Default; pick_vpc_endpoint(Endpoints, Default) -> % it fine to use default here - no IAM is used, only for http client From 4f94d98010a44729125f8c8c80f1bc187b72d7b0 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 26 May 2020 14:04:45 +0100 Subject: [PATCH 137/310] Prepare for the future I ended up moving the handler to erlcloud_util since it's used elsewhere, thus the fix is centralized I added minimal testing according to erlcloud_s3.erl's example comment --- src/erlcloud_s3.erl | 2 +- src/erlcloud_util.erl | 25 ++++++++++++++++++++++++- test/erlcloud_aws_tests.erl | 2 +- test/erlcloud_s3_tests.erl | 7 ++++++- 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index d0d05228d..1a32742e6 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -1738,7 +1738,7 @@ create_notification_param_xml({cloud_function, CF}, Acc) -> [{'CloudFunction', [ -spec get_bucket_and_key(string()) -> {string(), string()}. get_bucket_and_key(Uri) -> - {ok, Parsed} = http_uri:parse(Uri), + {ok, Parsed} = erlcloud_util:uri_parse(Uri), {Host, Path} = extract_host_and_path(Parsed), extract_location_fields(Host, Path). diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index b33d3b38d..400ee8c00 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -16,7 +16,8 @@ encode_object/2, encode_object_list/2, next_token/2, - filter_undef/1 + filter_undef/1, + uri_parse/1 ]). -define(MAX_ITEMS, 1000). @@ -174,3 +175,25 @@ next_token(Path, XML) -> -spec filter_undef(proplists:proplist()) -> proplists:proplist(). filter_undef(List) -> lists:filter(fun({_Name, Value}) -> Value =/= undefined end, List). + +-ifdef(OTP_RELEASE). +-if(?OTP_RELEASE >= 23). +uri_parse(Uri) -> + URIMap = uri_string:parse(Uri), + DefaultScheme = "https", + DefaultPort = 443, + Scheme = list_to_atom(maps:get(scheme, URIMap, DefaultScheme)), + UserInfo = maps:get(userinfo, URIMap, ""), + Host = maps:get(host, URIMap, ""), + Port = maps:get(port, URIMap, DefaultPort), + Path = maps:get(path, URIMap, ""), + Query = maps:get(query, URIMap, ""), + {ok, {Scheme, UserInfo, Host, Port, Path, Query}}. +-else. +uri_parse(Uri) -> + http_uri:parse(Uri). +-endif. +-else. +uri_parse(Uri) -> + http_uri:parse(Uri). +-endif. diff --git a/test/erlcloud_aws_tests.erl b/test/erlcloud_aws_tests.erl index ba2058fba..9f5e88625 100644 --- a/test/erlcloud_aws_tests.erl +++ b/test/erlcloud_aws_tests.erl @@ -189,7 +189,7 @@ get_url_from_history([{_, {erlcloud_httpc, request, [Url, _, _, _, _, _]}, _}]) Url. test_url(ExpScheme, ExpHost, ExpPort, ExpPath, Url) -> - {ok, {Scheme, _UserInfo, Host, Port, Path, _Query}} = http_uri:parse(Url), + {ok, {Scheme, _UserInfo, Host, Port, Path, _Query}} = erlcloud_util:uri_parse(Url), [?_assertEqual(ExpScheme, Scheme), ?_assertEqual(ExpHost, Host), ?_assertEqual(ExpPort, Port), diff --git a/test/erlcloud_s3_tests.erl b/test/erlcloud_s3_tests.erl index 2d28982b3..95b624859 100755 --- a/test/erlcloud_s3_tests.erl +++ b/test/erlcloud_s3_tests.erl @@ -41,7 +41,8 @@ operation_test_() -> fun get_bucket_encryption_test/1, fun get_bucket_encryption_not_found_test/1, fun delete_bucket_encryption_test/1, - fun hackney_proxy_put_validation_test/1 + fun hackney_proxy_put_validation_test/1, + fun get_bucket_and_key/1 ]}. start() -> @@ -823,3 +824,7 @@ hackney_proxy_put_validation_test(_) -> ,{"x-amz-version-id", "version_id"} ], Result). +get_bucket_and_key(_) -> + ErlcloudS3ExportExample = "https://s3.amazonaws.com/some_bucket/path_to_file", + Result = erlcloud_s3:get_bucket_and_key(ErlcloudS3ExportExample), + ?_assertEqual({"some_bucket","path_to_file"}, Result). From 6b2445827ff4e4f49b06e876217426db38b1bd68 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 1 Jul 2020 03:06:36 +0100 Subject: [PATCH 138/310] Preemptively adapt to future OTP 24 compilation/static analysis issues (Warning: crypto:hmac/3 is deprecated and will be removed in OTP 24; use use crypto:mac/4 instead) --- src/erlcloud_util.erl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index b33d3b38d..7f35647d3 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -21,12 +21,32 @@ -define(MAX_ITEMS, 1000). +-ifdef(OTP_RELEASE). +-if(?OTP_RELEASE >= 23). +sha_mac(K, S) -> + crypto:mac(hmac, sha, K, S). +-else. sha_mac(K, S) -> crypto:hmac(sha, K, S). +-endif. +-else. +sha_mac(K, S) -> + crypto:hmac(sha, K, S). +-endif. +-ifdef(OTP_RELEASE). +-if(?OTP_RELEASE >= 23). +sha256_mac(K, S) -> + crypto:mac(hmac, sha256, K, S). +-else. sha256_mac(K, S) -> crypto:hmac(sha256, K, S). +-endif. +-else. +sha256_mac(K, S) -> + crypto:hmac(sha256, K, S). +-endif. sha256(V) -> crypto:hash(sha256, V). From a678594a513b4fd717c663dbb9fbb3aca106f48c Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 2 Jul 2020 01:12:18 +0100 Subject: [PATCH 139/310] Remove .dialyzer_plt when cleaning from rebar It works for rebar 2 and rebar3, though rebar3 has its own model of saving the PLT elsewhere --- rebar.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rebar.config b/rebar.config index 34480cff9..de23d270e 100644 --- a/rebar.config +++ b/rebar.config @@ -35,3 +35,5 @@ {test, [{deps, [{meck, "0.8.13"}]}]} ,{warnings, [{erl_opts, [warnings_as_errors]}]} ]}. + +{post_hooks, [{clean, "rm -f .dialyzer_plt"}]}. From 1243387f6545a7fbc397f51a1c117ee28b98ead0 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 2 Jul 2020 19:15:45 +0100 Subject: [PATCH 140/310] Prepare for the future - OTP 25 (http_uri:decode/1 + http_uri:encode/1) (#649) * Adapt to upcoming OTP 25 changes where the "replaced" functions will be removed (Warning: http_uri:decode/1 is deprecated and will be removed in OTP 25; use use uri_string functions instead) + (Warning: http_uri:encode/1 is deprecated and will be removed in OTP 25; use use uri_string functions instead) --- src/erlcloud_iam.erl | 2 +- src/erlcloud_util.erl | 31 ++++++++++++++++++++++++++++++- test/erlcloud_iam_tests.erl | 22 +++++++++++----------- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index 6a6f8296e..4ef8e94f4 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -1055,7 +1055,7 @@ data_fun("Boolean") -> {erlcloud_xml, get_bool}; data_fun("Uri") -> {?MODULE, get_uri}. get_uri(Key, Item) -> - http_uri:decode(erlcloud_xml:get_text(Key, Item)). + erlcloud_util:http_uri_decode(erlcloud_xml:get_text(Key, Item)). make_list_virtual_mfa_devices_params(undefined, undefined, undefined) -> []; diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index a47f4583c..2aa9fe7de 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -17,7 +17,9 @@ encode_object_list/2, next_token/2, filter_undef/1, - uri_parse/1 + uri_parse/1, + http_uri_decode/1, + http_uri_encode/1 ]). -define(MAX_ITEMS, 1000). @@ -217,3 +219,30 @@ uri_parse(Uri) -> uri_parse(Uri) -> http_uri:parse(Uri). -endif. + +-ifdef(OTP_RELEASE). +-if(?OTP_RELEASE >= 23). +http_uri_decode(HexEncodedURI) -> + [{URI, true}] = uri_string:dissect_query(HexEncodedURI), + URI. +-else. +http_uri_decode(HexEncodedURI) -> + http_uri:decode(HexEncodedURI). +-endif. +-else. +http_uri_decode(HexEncodedURI) -> + http_uri:decode(HexEncodedURI). +-endif. + +-ifdef(OTP_RELEASE). +-if(?OTP_RELEASE >= 23). +http_uri_encode(URI) -> + uri_string:compose_query([{URI, true}]). +-else. +http_uri_encode(URI) -> + http_uri:encode(URI). +-endif. +-else. +http_uri_encode(URI) -> + http_uri:encode(URI). +-endif. diff --git a/test/erlcloud_iam_tests.erl b/test/erlcloud_iam_tests.erl index cc5fc005f..aaaa4fb5d 100644 --- a/test/erlcloud_iam_tests.erl +++ b/test/erlcloud_iam_tests.erl @@ -2197,7 +2197,7 @@ list_attached_user_policies_input_tests(_) -> [ {"Action", "ListAttachedUserPolicies"}, {"UserName", "Alice"}, - {"PathPrefix", http_uri:encode("/")} + {"PathPrefix", erlcloud_util:http_uri_encode("/")} ]}) ], @@ -2266,7 +2266,7 @@ list_attached_group_policies_input_tests(_) -> [ {"Action", "ListAttachedGroupPolicies"}, {"GroupName", "ReadOnlyUsers"}, - {"PathPrefix", http_uri:encode("/")} + {"PathPrefix", erlcloud_util:http_uri_encode("/")} ]}) ], @@ -2335,7 +2335,7 @@ list_attached_role_policies_input_tests(_) -> [ {"Action", "ListAttachedRolePolicies"}, {"RoleName", "ReadOnlyRole"}, - {"PathPrefix", http_uri:encode("/")} + {"PathPrefix", erlcloud_util:http_uri_encode("/")} ]}) ], @@ -2534,7 +2534,7 @@ list_entities_for_policy_input_tests(_) -> ?_f(erlcloud_iam:list_entities_for_policy("test")), [ {"Action", "ListEntitiesForPolicy"}, - {"PathPrefix", http_uri:encode("/")}, + {"PathPrefix", erlcloud_util:http_uri_encode("/")}, {"PolicyArn", "test"} ]}) ], @@ -2658,7 +2658,7 @@ get_policy_input_tests(_) -> ?_f(erlcloud_iam:get_policy("arn:aws:iam::123456789012:policy/S3-read-only-example-bucket")), [ {"Action", "GetPolicy"}, - {"PolicyArn", http_uri:encode("arn:aws:iam::123456789012:policy/S3-read-only-example-bucket")} + {"PolicyArn", erlcloud_util:http_uri_encode("arn:aws:iam::123456789012:policy/S3-read-only-example-bucket")} ]}) ], @@ -2706,7 +2706,7 @@ get_policy_version_input_tests(_) -> ?_f(erlcloud_iam:get_policy_version("arn:aws:iam::123456789012:policy/S3-read-only-example-bucket", "v1")), [ {"Action", "GetPolicyVersion"}, - {"PolicyArn", http_uri:encode("arn:aws:iam::123456789012:policy/S3-read-only-example-bucket")}, + {"PolicyArn", erlcloud_util:http_uri_encode("arn:aws:iam::123456789012:policy/S3-read-only-example-bucket")}, {"VersionId", "v1"} ]}) ], @@ -2805,7 +2805,7 @@ simulate_custom_policy_input_test(_) -> PolicyDoc2])), [ {"Action", "SimulateCustomPolicy"}, - {"ActionNames.member.1", http_uri:encode(Action)}, + {"ActionNames.member.1", erlcloud_util:http_uri_encode(Action)}, {"PolicyInputList.member.1", PolicyDoc1}, {"PolicyInputList.member.2", PolicyDoc2}, {"MaxItems", "1000"} @@ -2816,9 +2816,9 @@ simulate_custom_policy_input_test(_) -> [PolicyDoc1], ContextEntries)), [{"Action","SimulateCustomPolicy"}, - {"ActionNames.member.1", http_uri:encode(Action)}, + {"ActionNames.member.1", erlcloud_util:http_uri_encode(Action)}, {"PolicyInputList.member.1","policy_doc1"}, - {"ContextEntries.member.1.ContextKeyName",http_uri:encode("aws:MultiFactorAuthPresent")}, + {"ContextEntries.member.1.ContextKeyName",erlcloud_util:http_uri_encode("aws:MultiFactorAuthPresent")}, {"ContextEntries.member.1.ContextKeyType","boolean"}, {"ContextEntries.member.1.ContextKeyValues.member.1","true"}, {"MaxItems","1000"}]}) @@ -2863,8 +2863,8 @@ simulate_principal_policy_input_test(_) -> [Action])), [ {"Action", "SimulatePrincipalPolicy"}, - {"ActionNames.member.1", http_uri:encode(Action)}, - {"PolicySourceArn", http_uri:encode(Principal)}, + {"ActionNames.member.1", erlcloud_util:http_uri_encode(Action)}, + {"PolicySourceArn", erlcloud_util:http_uri_encode(Principal)}, {"MaxItems", "1000"} ]}) ], From 0f0540041ad6b55700715d1d3629853bf3158791 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 14 Jul 2020 20:20:24 +0100 Subject: [PATCH 141/310] Be stricter on eunit compilation-based warnings (#650) * Be stricter in our approach to test compilation We include this in .travis.yml so that from now on the applied strictness is maintained * Fix compilation issue and approach eunit's function export standard * Prevent compilation issue on case clause already matched `?_assertNotException` (also `?assertNotException`) is to be used in cases where the Class and Term of the exception are previously known to expect not matching against. If used generically (with `_`, as-was) the Erlang pre-processor will expand that macro in such a way that a `case ... of _ -> ...; _ -> ... end` will occur thus causing an unexpected compilation issue * Fix dialyzer issues in erlcloud_waf_tests.erl * Fix dialyzer issues in erlcloud_sns_tests.erl * Fix dialyzer issues in erlcloud_s3_tests.erl * Fix dialyzer issues in erlcloud_route53_tests.erl * Fix dialyzer issues in erlcloud_lambda_tests.erl (derived from _tests) * Fix dialyzer issues in erlcloud_kms_tests.erl * Fix dialyzer issues in erlcloud_kinesis.erl (derived from _tests) * Fix dialyzer issues in erlcloud_inspector_tests.erl * Fix dialyzer issues in erlcloud_iam_tests.erl * Fix dialyzer issues in erlcloud_ecs_tests.erl * Fix dialyzer issues in erlcloud_ec2_tests.erl * Fix dialyzer issues in erlcloud_ddb_util_tests.erl * Fix dialyzer issues in erlcloud_ddb_tests.erl * Fix dialyzer issues in erlcloud_ddb2_tests.erl * Fix dialyzer issues in erlcloud_config_tests.erl * Fix dialyzer issues in erlcloud_cloudtrail_tests.erl * Fix dialyzer issues in erlcloud_cloudsearch.erl (derived from _tests) * Fix dialyzer issues in erlcloud_cloudfront.erl (derived from _tests) * Fix dialyzer issues in erlcloud_cloudformation.hrl (derived from _tests) * Fix dialyzer issues in erlcloud_aws_tests.erl * Fix dialyzer issues in erlcloud_autoscaling_tests.erl * Fix dialyzer issues in erlcloud_as.hrl (derived from _tests) We remove the use of `lists:concat` as, according to Erlang/OTP doc.s this is a function to use in the concatenation of elements that we want to result in a string; otherwise it'll make dialyzer complain, in any case (even when it works). * Further fix dialyzer issues spawned from our recent changes We're not yet being very restrictive with the dialyzer options, but once tried out these results surfaced * Try to approach rebar 2's static analysis check flow to rebar3's * Fix tests that mislead us into changing the public API * Prevent dialyzer'ing of tests to not be affected by non-opaque arguments when using `meck:expect` * Fix meck's bump dialyzer -related issue * Fix tests that mislead us into changing the public API * Use pre-defined types where available * Try to avoid potential eunit internal shortcircuit * Allow for test reference to be easier to find * Make rebar 2 behave just as much as rebar3 * Fix tests that mislead us into changing the public API * Make for alphabetically ordered API * Fix tests that mislead us into changing the public API According to the doc.s (https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Service.html) createdAt is "The Unix timestamp for when the service was created." which is a pos_integer(), not a float() I also update updated_at for the same reason (it's not interface breaking, it's a bugfix) * Make for faster (but as complete as before) Travis CI analysis 1. we remove check-unit (added by us) and concentrate the analysis in check (already present) 2. we rename eunit_warnigs (added by us) as warnings and concentrate the analysis there 3. we change rebar.config.script so we can accept an environment variable to control erl_opts (as a kind of approach to a rebar3 profile) * Tweak API to AWS' idiosyncrasies * Start testing on top of Erlang/OTP 23 --- .travis.yml | 3 +- Makefile | 38 ++- include/erlcloud.hrl | 2 +- include/erlcloud_as.hrl | 10 +- include/erlcloud_cloudformation.hrl | 48 +-- include/erlcloud_ddb2.hrl | 26 +- include/erlcloud_ec2.hrl | 28 +- include/erlcloud_ecs.hrl | 40 +-- rebar.config | 2 +- rebar.config.script | 14 +- src/erlcloud_application_autoscaler.erl | 22 +- src/erlcloud_as.erl | 60 ++-- src/erlcloud_cloudfront.erl | 2 +- src/erlcloud_cloudsearch.erl | 2 +- src/erlcloud_cloudtrail.erl | 16 +- src/erlcloud_cloudwatch_logs.erl | 4 +- src/erlcloud_ddb2.erl | 16 +- src/erlcloud_ec2.erl | 2 +- src/erlcloud_ecs.erl | 63 ++-- src/erlcloud_iam.erl | 20 +- src/erlcloud_inspector.erl | 176 +++++------ src/erlcloud_kinesis.erl | 16 +- src/erlcloud_lambda.erl | 12 +- src/erlcloud_route53.erl | 10 +- src/erlcloud_s3.erl | 6 +- src/erlcloud_waf.erl | 5 +- test/erlcloud_autoscaling_tests.erl | 2 +- test/erlcloud_aws_tests.erl | 40 +-- test/erlcloud_cloudtrail_tests.erl | 2 +- test/erlcloud_ddb2_tests.erl | 371 ++++++++++++------------ test/erlcloud_ddb_tests.erl | 10 +- test/erlcloud_ddb_util_tests.erl | 25 +- test/erlcloud_ec2_tests.erl | 1 + test/erlcloud_ecs_tests.erl | 4 +- test/erlcloud_iam_tests.erl | 6 +- test/erlcloud_inspector_tests.erl | 10 +- test/erlcloud_kms_tests.erl | 10 +- test/erlcloud_s3_tests.erl | 8 +- test/erlcloud_sdb_tests.erl | 16 +- test/erlcloud_sns_tests.erl | 9 +- test/erlcloud_waf_tests.erl | 12 +- 41 files changed, 609 insertions(+), 560 deletions(-) diff --git a/.travis.yml b/.travis.yml index d49c254ad..db346c397 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ otp_release: - 20.3 - 21.3 - 22.3 + - 23.0 env: global: - MAIN_OTP=20.3 @@ -18,7 +19,7 @@ branches: install: - make travis-install script: - - make check_warnings + - make warnings - make check - make eunit deploy: diff --git a/Makefile b/Makefile index 17dd0fcdf..d8304bd53 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,6 @@ CHECK_FILES=\ ebin/*.beam CHECK_EUNIT_FILES=\ - $(CHECK_FILES) \ .eunit/*.beam @@ -42,16 +41,26 @@ else $(REBAR) shell endif -check_warnings: +deps: get-deps + +check_warnings: deps ifeq ($(REBAR_VSN),2) - @echo skip checking warnings + @$(REBAR) compile else @$(REBAR) as warnings compile endif -eunit: +warnings: deps ifeq ($(REBAR_VSN),2) - @$(REBAR) compile + @WARNINGS_AS_ERRORS=true $(REBAR) compile + @WARNINGS_AS_ERRORS=true $(REBAR) compile_only=true eunit +else + @$(REBAR) as test compile +endif + +eunit: deps +ifeq ($(REBAR_VSN),2) + $(MAKE) compile @$(REBAR) eunit skip_deps=true else @$(REBAR) eunit @@ -63,28 +72,17 @@ endif --fullpath \ --output_plt .dialyzer_plt -check: +check: deps ifeq ($(REBAR_VSN),2) - @$(REBAR) compile - $(MAKE) .dialyzer_plt - dialyzer --no_check_plt --fullpath \ - $(CHECK_FILES) \ - -I include \ - --plt .dialyzer_plt -else - @$(REBAR) dialyzer -endif - -check-eunit: eunit -ifeq ($(REBAR_VSN),2) - @$(REBAR) compile + $(MAKE) compile + @$(REBAR) compile_only=true eunit $(MAKE) .dialyzer_plt dialyzer --no_check_plt --fullpath \ $(CHECK_EUNIT_FILES) \ -I include \ --plt .dialyzer_plt else - @$(REBAR) dialyzer + @$(REBAR) as test dialyzer endif doc: diff --git a/include/erlcloud.hrl b/include/erlcloud.hrl index 9872fa5fa..a924b8063 100644 --- a/include/erlcloud.hrl +++ b/include/erlcloud.hrl @@ -5,7 +5,7 @@ -type datetime() :: {{pos_integer(), 1..12, 1..31}, {0..23, 0..59, 0..60}}. -type paging_token() :: binary() | undefined. --type success_result_paged(ObjectType) :: {ok, [ObjectType]}. +-type success_result_paged(ObjectType) :: {ok, ObjectType}. -type error_result() :: {error, Reason :: term()}. -type result_paged(ObjectType) :: success_result_paged(ObjectType) | error_result(). -type result(ObjectType) :: {ok, [ObjectType]} | error_result(). diff --git a/include/erlcloud_as.hrl b/include/erlcloud_as.hrl index 91b1075aa..47a78a50c 100644 --- a/include/erlcloud_as.hrl +++ b/include/erlcloud_as.hrl @@ -26,15 +26,15 @@ -record(aws_autoscaling_group, { group_name :: string(), availability_zones :: undefined | list(string()), - load_balancer_names :: list(string()), + load_balancer_names :: undefined | list(string()), tags :: list(aws_autoscaling_tag()), desired_capacity :: undefined | integer(), min_size :: undefined | integer(), max_size :: undefined | integer(), launch_configuration_name :: undefined | string(), vpc_zone_id :: undefined | list(string()), - instances :: list(aws_autoscaling_instance()), - status :: string() + instances :: undefined | list(aws_autoscaling_instance()), + status :: undefined | string() }). -type(aws_autoscaling_group() :: #aws_autoscaling_group{}). @@ -42,7 +42,7 @@ name :: string(), image_id :: string(), instance_type :: string(), - tenancy :: string(), + tenancy :: undefined | string(), user_data :: undefined | string(), security_groups = [] :: list(string()), public_ip_address = false :: undefined | boolean(), @@ -61,7 +61,7 @@ status_code :: string(), status_msg :: string(), start_time :: datetime(), - end_time :: datetime(), + end_time :: undefined | datetime(), progress :: integer() }). -type(aws_autoscaling_activity() :: #aws_autoscaling_activity{}). diff --git a/include/erlcloud_cloudformation.hrl b/include/erlcloud_cloudformation.hrl index da1ef8373..40d70e1b5 100644 --- a/include/erlcloud_cloudformation.hrl +++ b/include/erlcloud_cloudformation.hrl @@ -5,55 +5,55 @@ -record(cloudformation_create_stack_input, { capabilities = [] :: [string()], %% list containing CAPABILITY_IAM | CAPABILITY_NAMED_IAM | CAPABILITY_AUTO_EXPAND - client_request_token :: string(), - disable_rollback :: boolean(), - enable_termination_protection :: boolean(), + client_request_token :: undefined | string(), + disable_rollback :: undefined | boolean(), + enable_termination_protection :: undefined | boolean(), notification_arns = [] :: [string()], on_failure = "ROLLBACK" :: string(), %% DO_NOTHING | ROLLBACK | DELETE parameters = [] :: [cloudformation_parameter()], resource_types = [] :: [string()], - role_arn :: string(), + role_arn :: undefined | string(), rollback_configuration :: undefined | cloudformation_rollback_configuration(), stack_name :: string(), - stack_policy_body :: string(), - stack_policy_url :: string(), + stack_policy_body :: undefined | string(), + stack_policy_url :: undefined | string(), tags = []:: [cloudformation_tag()], - template_body :: string(), - template_url :: string(), - timeout_in_minutes :: integer() + template_body :: undefined | string(), + template_url :: undefined | string(), + timeout_in_minutes :: undefined | integer() }). -record(cloudformation_update_stack_input, { capabilities = [] :: [string()], %% list containing CAPABILITY_IAM | CAPABILITY_NAMED_IAM | CAPABILITY_AUTO_EXPAND - client_request_token :: string(), + client_request_token :: undefined | string(), notification_arns = [] :: [string()], parameters = [] :: [cloudformation_parameter()], resource_types = [] :: [string()], - role_arn :: string(), - rollback_configuration :: cloudformation_rollback_configuration(), + role_arn :: undefined | string(), + rollback_configuration :: undefined | cloudformation_rollback_configuration(), stack_name :: string(), - stack_policy_body :: string(), - stack_policy_during_update_body :: string(), - stack_policy_during_update_url :: string(), - stack_policy_url :: string(), + stack_policy_body :: undefined | string(), + stack_policy_during_update_body :: undefined | string(), + stack_policy_during_update_url :: undefined | string(), + stack_policy_url :: undefined | string(), tags = []:: [cloudformation_tag()], - template_body :: string(), - template_url :: string(), - use_previous_template :: boolean() + template_body :: undefined | string(), + template_url :: undefined | string(), + use_previous_template :: undefined | boolean() }). -record(cloudformation_delete_stack_input, { - client_request_token :: string(), + client_request_token :: undefined | string(), retain_resources = [] :: [string()], - role_arn :: string(), - stack_name :: string() + role_arn :: undefined | string(), + stack_name :: undefined | string() }). -record(cloudformation_parameter, { parameter_key :: string(), parameter_value :: string(), - resolved_value :: string(), - use_previous_value :: boolean() + resolved_value :: undefined | string(), + use_previous_value :: undefined | boolean() }). -record(cloudformation_rollback_configuration, { diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index 1dbf84b16..e1c0c986a 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -47,8 +47,14 @@ {index_name :: undefined | binary(), provisioned_throughput_override :: undefined | #ddb2_provisioned_throughput_override{}}). +-record(ddb2_replica_global_secondary_index_auto_scaling_description, + {index_name :: undefined | binary(), + index_status :: undefined | index_status(), + provisioned_read_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, + provisioned_write_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}}). + -record(ddb2_replica_auto_scaling_description, - {global_secondary_indexes :: undefined | [#ddb2_replica_global_secondary_index_description{}], + {global_secondary_indexes :: undefined | [#ddb2_replica_global_secondary_index_auto_scaling_description{}], region_name :: undefined | binary(), replica_provisioned_read_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, replica_provisioned_write_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, @@ -72,15 +78,13 @@ provisioned_write_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, provisioned_write_capacity_units :: undefined | pos_integer()}). --record(ddb2_replica_global_secondary_index_auto_scaling_description, - {index_name :: undefined | binary(), - index_status :: undefined | binary(), - provisioned_read_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, - provisioned_write_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}}). +-record(ddb2_billing_mode_summary, + {billing_mode :: undefined | erlcloud_ddb2:billing_mode(), + last_update_to_pay_per_request_date_time :: undefined | date_time()}). -record(ddb2_replica_settings_description, {region_name :: undefined | binary(), - replica_billing_mode_summary :: undefined | binary(), + replica_billing_mode_summary :: undefined | #ddb2_billing_mode_summary{}, replica_global_secondary_index_settings :: undefined | [#ddb2_replica_global_secondary_index_settings_description{}], replica_provisioned_read_capacity_auto_scaling_settings :: undefined | #ddb2_auto_scaling_settings_description{}, replica_provisioned_read_capacity_units :: undefined | number(), @@ -105,10 +109,6 @@ replication_group :: undefined | [#ddb2_replica{}] }). --record(ddb2_billing_mode_summary, - {billing_mode :: undefined | erlcloud_ddb2:billing_mode(), - last_update_to_pay_per_request_date_time :: undefined | date_time()}). - -record(ddb2_provisioned_throughput_description, {last_decrease_date_time :: undefined | date_time(), last_increase_date_time :: undefined | date_time(), @@ -361,7 +361,7 @@ provisioned_throughput :: undefined | #ddb2_provisioned_throughput{}, table_arn :: undefined | binary(), table_creation_date_time :: undefined | number(), - table_id :: undefined | number(), + table_id :: undefined | binary(), table_name :: undefined | binary(), table_size_bytes :: undefined | integer() }). @@ -388,7 +388,7 @@ {global_secondary_indexes :: undefined | [#ddb2_global_secondary_index_info{}], local_secondary_indexes :: undefined | [#ddb2_local_secondary_index_info{}], sse_description :: undefined | erlcloud_ddb2:sse_description(), - stream_description :: undefined | #ddb2_stream_description{}, + stream_description :: undefined | #ddb2_stream_description{} | {boolean(), erlcloud_ddb2:stream_view_type()}, time_to_live_description :: undefined | #ddb2_time_to_live_description{} }). diff --git a/include/erlcloud_ec2.hrl b/include/erlcloud_ec2.hrl index 8da059e32..e7f837d9f 100644 --- a/include/erlcloud_ec2.hrl +++ b/include/erlcloud_ec2.hrl @@ -29,24 +29,24 @@ image_id::string(), min_count=1::pos_integer(), max_count=1::pos_integer(), - key_name::string(), + key_name::undefined | string(), group_set=["default"]::[string()], user_data::undefined|binary(), instance_type::string(), - availability_zone::string(), - placement_group::string(), - kernel_id::string(), - ramdisk_id::string(), + availability_zone::undefined | string(), + placement_group::undefined | string(), + kernel_id::undefined | string(), + ramdisk_id::undefined | string(), block_device_mapping=[]::[ec2_block_device_mapping()], monitoring_enabled=false::boolean(), subnet_id::string(), disable_api_termination=false::boolean(), - instance_initiated_shutdown_behavior::ec2_shutdown_behavior(), + instance_initiated_shutdown_behavior::undefined | ec2_shutdown_behavior(), net_if=[] :: [#ec2_net_if{}], ebs_optimized = false :: boolean(), - iam_instance_profile_name = undefined :: string(), - spot_price::string(), - weighted_capacity::number() + iam_instance_profile_name :: undefined | string(), + spot_price::undefined | string(), + weighted_capacity::undefined | number() }). -record(ec2_image_spec, { image_location::string(), @@ -70,15 +70,15 @@ }). -record(spot_fleet_request_config_spec, { allocation_strategy::undefined|lowest_price|diversified, - client_token::string(), - excess_capacity_termination_policy::no_termination|default, + client_token::undefined|string(), + excess_capacity_termination_policy::undefined|no_termination|default, iam_fleet_role::string(), launch_specification=[]::[#ec2_instance_spec{}], spot_price::string(), target_capacity::pos_integer(), - terminate_instances_with_expiration::true|false, - valid_from::datetime(), - valid_until::datetime() + terminate_instances_with_expiration::undefined|true|false, + valid_from::undefined|datetime(), + valid_until::undefined|datetime() }). -record(ec2_spot_fleet_request, { spot_fleet_request_config::#spot_fleet_request_config_spec{} diff --git a/include/erlcloud_ecs.hrl b/include/erlcloud_ecs.hrl index 327bdb575..fc89f854a 100644 --- a/include/erlcloud_ecs.hrl +++ b/include/erlcloud_ecs.hrl @@ -38,18 +38,18 @@ }). -record(ecs_deployment, { - created_at :: undefined | pos_integer(), + created_at :: undefined | number(), desired_count :: undefined | pos_integer(), id :: undefined | binary(), - pending_count :: undefined | pos_integer(), - running_count :: undefined | pos_integer(), + pending_count :: undefined | non_neg_integer(), + running_count :: undefined | non_neg_integer(), status :: undefined | binary(), task_definition:: undefined | binary(), - updated_at :: undefined | pos_integer() + updated_at :: undefined | number() }). -record(ecs_event, { - created_at :: undefined | pos_integer(), + created_at :: undefined | number(), id :: undefined | binary(), message :: undefined | binary() }). @@ -62,26 +62,26 @@ }). -record(ecs_cluster, { - active_services_count :: undefined | pos_integer(), + active_services_count :: undefined | non_neg_integer(), cluster_arn :: undefined | binary(), cluster_name :: undefined | binary(), - pending_tasks_count :: undefined | pos_integer(), - registered_container_instances_count :: undefined | pos_integer(), - running_tasks_count :: undefined | pos_integer(), + pending_tasks_count :: undefined | non_neg_integer(), + registered_container_instances_count :: undefined | non_neg_integer(), + running_tasks_count :: undefined | non_neg_integer(), status :: undefined | binary() }). -record(ecs_service, { cluster_arn :: undefined | binary(), - created_at :: undefined | pos_integer(), + created_at :: undefined | number(), deployment_configuration :: undefined | #ecs_deployment_configuration{}, deployments :: undefined | [#ecs_deployment{}], desired_count :: undefined | pos_integer(), events :: undefined | [#ecs_event{}], load_balancers :: undefined | [#ecs_load_balancer{}], - pending_count :: undefined | pos_integer(), + pending_count :: undefined | non_neg_integer(), role_arn :: undefined | binary(), - running_count :: undefined | pos_integer(), + running_count :: undefined | non_neg_integer(), service_arn :: undefined | binary(), service_name :: undefined | binary(), status :: undefined | binary(), @@ -89,11 +89,11 @@ }). -record(ecs_resource, { - double_value :: undefined | pos_integer(), - integer_value :: undefined | pos_integer(), - long_value :: undefined | pos_integer(), + double_value :: undefined | non_neg_integer(), + integer_value :: undefined | non_neg_integer(), + long_value :: undefined | non_neg_integer(), name :: undefined | binary(), - string_set_value :: undefined | binary(), + string_set_value :: undefined | [binary()], type :: undefined | binary() }). @@ -109,10 +109,10 @@ attributes :: undefined | [#ecs_attribute{}], container_instance_arn :: undefined | binary(), ec2_instance_id :: undefined | binary(), - pending_tasks_count :: undefined | pos_integer(), + pending_tasks_count :: undefined | non_neg_integer(), registered_resources :: undefined | [#ecs_resource{}], remaining_resources :: undefined | [#ecs_resource{}], - running_tasks_count :: undefined | pos_integer(), + running_tasks_count :: undefined | non_neg_integer(), status :: undefined | binary(), version_info :: undefined | #ecs_version_info{} }). @@ -210,7 +210,7 @@ -record(ecs_task_definition, { container_definitions :: undefined | [#ecs_container_definition{}], family :: undefined | binary(), - network_mode :: undefined | binary(), + network_mode :: undefined | ecs_network_mode(), requires_attributes :: undefined | [#ecs_attribute{}], revision :: undefined | pos_integer(), status :: undefined | binary(), @@ -251,7 +251,7 @@ cluster_arn :: undefined | binary(), container_instance_arn :: undefined | binary(), containers :: undefined | [#ecs_container{}], - created_at :: undefined | pos_integer(), + created_at :: undefined | number(), desired_status:: undefined | binary(), last_status :: undefined | binary(), overrides :: undefined | #ecs_task_override{}, diff --git a/rebar.config b/rebar.config index de23d270e..753c99996 100644 --- a/rebar.config +++ b/rebar.config @@ -32,7 +32,7 @@ {profiles, [ - {test, [{deps, [{meck, "0.8.13"}]}]} + {test, [{deps, [{meck, "0.9.0"}]}, {erl_opts, [warnings_as_errors]}]} ,{warnings, [{erl_opts, [warnings_as_errors]}]} ]}. diff --git a/rebar.config.script b/rebar.config.script index 5eaa2fc34..1897c844e 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -5,11 +5,21 @@ case erlang:function_exported(rebar3, main, 1) of false -> % rebar 2.x or older %% Use git-based deps %% profiles - [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.8.13"}}}, + CONFIG_DEPS = + [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.9.0"}}}, {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "v2.11.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.7"}}}, {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, {base16, ".*", {git, "https://github.com/goj/base16.git", {tag, "1.0.0"}}}]} - | lists:keydelete(deps, 1, CONFIG)] + | lists:keydelete(deps, 1, CONFIG)], + CONFIG_NEW = + case os:getenv("WARNINGS_AS_ERRORS") of + "true" -> + [{erl_opts, [warnings_as_errors | proplists:get_value(erl_opts, CONFIG_DEPS)]} + | lists:keydelete(erl_opts, 1, CONFIG_DEPS)]; + _ -> + CONFIG_DEPS + end, + CONFIG_NEW end. diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index c22f0c4f1..57b8bded9 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -66,7 +66,7 @@ | container_credentials_unavailable | erlcloud_aws:httpc_result_error()}. --type aws_aas_request_body() :: [proplist:proplist()]. +-type aws_aas_request_body() :: proplists:proplist(). -spec extract_alarm(J :: proplist()) -> response(). extract_alarm(J) -> @@ -317,7 +317,7 @@ delete_scaling_policy(Configuration, BodyConfiguration) -> %%------------------------------------------------------------------------------ -spec delete_scheduled_action( - Configuration :: erlcloud_aws:aws_config(), + Configuration :: aws_config(), ResourceId :: binary(), ScalableDimension :: binary(), ScheduledActionName :: binary(), @@ -345,7 +345,7 @@ delete_scheduled_action(Configuration, BodyConfiguration) -> %%------------------------------------------------------------------------------ -spec deregister_scalable_target( - Configuration :: erlcloud_aws:aws_config(), + Configuration :: aws_config(), ResourceId :: binary(), ScalableDimension :: binary(), ServiceNamespace :: binary() @@ -357,7 +357,7 @@ deregister_scalable_target(Configuration, ResourceId, ScalableDimension, Service deregister_scalable_target(Configuration, BodyProps). -spec deregister_scalable_target( - Configuration :: erlcloud_aws:aws_config(), + Configuration :: aws_config(), BodyConfiguration :: aws_aas_request_body() ) -> ok_error_response(). deregister_scalable_target(Configuration, BodyConfiguration) -> @@ -371,7 +371,7 @@ deregister_scalable_target(Configuration, BodyConfiguration) -> %%------------------------------------------------------------------------------ -spec describe_scalable_targets( - erlcloud_aws:aws_config(), + aws_config(), aws_aas_request_body() | binary() ) -> ok_error_response(). describe_scalable_targets(Configuration, ServiceNamespace) when is_binary(ServiceNamespace)-> @@ -399,7 +399,7 @@ describe_scalable_targets(Configuration, BodyConfiguration) -> %%------------------------------------------------------------------------------ -spec describe_scaling_activities( - erlcloud_aws:aws_config(), + aws_config(), aws_aas_request_body() | binary() ) -> ok_error_response(). describe_scaling_activities(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> @@ -428,7 +428,7 @@ describe_scaling_activities(Configuration, BodyConfiguration) -> %%------------------------------------------------------------------------------ -spec describe_scaling_policies( - erlcloud_aws:aws_config(), + aws_config(), aws_aas_request_body() | binary() ) -> ok_error_response(). describe_scaling_policies(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> @@ -457,7 +457,7 @@ describe_scaling_policies(Configuration, BodyConfiguration) -> %%------------------------------------------------------------------------------ -spec describe_scheduled_actions( - erlcloud_aws:aws_config(), + aws_config(), aws_aas_request_body() | binary() ) -> ok_error_response(). describe_scheduled_actions(Configuration, ServiceNamespace) when is_binary(ServiceNamespace) -> @@ -506,7 +506,7 @@ put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, Ser ScalableDimension :: binary(), ServiceNamespace :: binary(), PolicyType :: binary(), - Policy :: [proplist:proplist()] + Policy :: [proplists:proplist()] ) -> ok_error_response(). put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, ServiceNamespace, PolicyType, Policy) -> BodyProps = [{<<"PolicyName">>, PolicyName}, @@ -522,7 +522,7 @@ put_scaling_policy(Configuration, PolicyName, ResourceId, ScalableDimension, Ser end. -spec put_scaling_policy( - Configuration :: erlcloud_aws:aws_config(), + Configuration :: aws_config(), BodyConfiguration :: aws_aas_request_body() ) -> ok_error_response(). put_scaling_policy(Configuration, BodyConfiguration) -> @@ -668,7 +668,7 @@ register_scalable_target(Configuration, ResourceId, ScalableDimension, ServiceNa register_scalable_target(Configuration, BodyProps ++ MaybeBodyWithMax ++ MaybeBodyWithMin). -spec register_scalable_target( - Configuration :: erlcloud_aws:aws_config(), + Configuration :: aws_config(), BodyConfiguration :: aws_aas_request_body() ) -> ok_error_response(). register_scalable_target(Configuration, BodyConfiguration) -> diff --git a/src/erlcloud_as.erl b/src/erlcloud_as.erl index c0012416b..3ac8886ef 100644 --- a/src/erlcloud_as.erl +++ b/src/erlcloud_as.erl @@ -331,18 +331,16 @@ create_launch_config(#aws_launch_config{ }, Config) -> Params = - lists:concat([ [ {"LaunchConfigurationName", LCName}, {"ImageId", ImageId}, {"InstanceType", Type} - ], - when_defined(UserData, [{"UserData", UserData}], []), - when_defined(PublicIP, [{"AssociatePublicIpAddress", atom_to_list(PublicIP)}], []), - when_defined(Monitoring, [{"InstanceMonitoring.Enabled", atom_to_list(Monitoring)}], []), - member_params("SecurityGroups.member.", SGroups), - when_defined(KeyPair, [{"KeyName", KeyPair}], []) - ]), + ] + ++ when_defined(UserData, [{"UserData", UserData}], []) + ++ when_defined(PublicIP, [{"AssociatePublicIpAddress", atom_to_list(PublicIP)}], []) + ++ when_defined(Monitoring, [{"InstanceMonitoring.Enabled", atom_to_list(Monitoring)}], []) + ++ member_params("SecurityGroups.member.", SGroups) + ++ when_defined(KeyPair, [{"KeyName", KeyPair}], []), io:format("~p ~n", [Params]), create_launch_config(Params, Config); @@ -373,23 +371,21 @@ create_auto_scaling_group(#aws_autoscaling_group{ }, Config) -> ProcessedTags = lists:flatten([tag_to_member_param(T, Idx) || {T, Idx} <- lists:zip(Tags, lists:seq(1, length(Tags)))]), - Params = lists:concat([ - [ + Params = [ {"AutoScalingGroupName", GName}, {"LaunchConfigurationName", LaunchName}, {"MaxSize", integer_to_list(MaxSize)}, {"MinSize", integer_to_list(MinSize)} - ], - case VpcZoneIds of + ] + ++ case VpcZoneIds of undefined -> []; _ ->[{"VPCZoneIdentifier", string:join(VpcZoneIds, ",")}] - end, - case AZones of + end + ++ case AZones of undefined -> []; _ -> member_params("AvailabilityZones.member.", AZones) - end, - ProcessedTags - ]), + end + ++ ProcessedTags, create_auto_scaling_group(Params, Config); create_auto_scaling_group(Params, Config) -> @@ -418,35 +414,33 @@ update_auto_scaling_group(#aws_autoscaling_group{ availability_zones = AZones }, Config) -> - Params = lists:concat([ - [ + Params = [ {"AutoScalingGroupName", GName} - ], - case LaunchName of + ] + ++ case LaunchName of undefined -> []; _ -> [{"LaunchConfigurationName", LaunchName}] - end, - case DesiredCapacity of + end + ++ case DesiredCapacity of undefined -> []; _ -> [{"DesiredCapacity", integer_to_list(DesiredCapacity)}] - end, - case MaxSize of + end + ++ case MaxSize of undefined -> []; _ -> [{"MaxSize", integer_to_list(MaxSize)}] - end, - case MinSize of + end + ++ case MinSize of undefined -> []; _ -> [{"MinSize", integer_to_list(MinSize)}] - end, - case VpcZoneIds of + end + ++ case VpcZoneIds of undefined -> []; _ ->[{"VPCZoneIdentifier", string:join(VpcZoneIds, ",")}] - end, - case AZones of + end + ++ case AZones of undefined -> []; _ -> member_params("AvailabilityZones.member.", AZones) - end - ]), + end, update_auto_scaling_group(Params, Config); update_auto_scaling_group(Params, Config) -> diff --git a/src/erlcloud_cloudfront.erl b/src/erlcloud_cloudfront.erl index d48169cb5..702ce07da 100644 --- a/src/erlcloud_cloudfront.erl +++ b/src/erlcloud_cloudfront.erl @@ -181,7 +181,7 @@ list_distributions(Config) when list_distributions(?MAX_RESULTS, undefined, Config). --spec list_distributions(integer(), string()) -> ok_error(proplist(), string()). +-spec list_distributions(integer(), undefined | string()) -> ok_error(proplist(), string()). list_distributions(MaxResults, Marker) -> list_distributions(MaxResults, Marker, erlcloud_aws:default_config()). diff --git a/src/erlcloud_cloudsearch.erl b/src/erlcloud_cloudsearch.erl index a01b8aba9..5d4ce1e84 100644 --- a/src/erlcloud_cloudsearch.erl +++ b/src/erlcloud_cloudsearch.erl @@ -56,7 +56,7 @@ -type cloudsearch_domain_tag() :: [{TagKey :: binary(), TagValue :: string()}]. -type cloudsearch_index_field_option() :: { OptionName :: binary(), - OptionValue :: boolean() | string() | null | integer() | float() | list() + OptionValue :: boolean() | string() | null | integer() | float() | list() | binary() }. -type cloudsearch_index_field() :: [cloudsearch_index_field_option()]. diff --git a/src/erlcloud_cloudtrail.erl b/src/erlcloud_cloudtrail.erl index 33cdbf8a1..f20dae943 100644 --- a/src/erlcloud_cloudtrail.erl +++ b/src/erlcloud_cloudtrail.erl @@ -118,38 +118,38 @@ describe_trails(Trails, IncludeShadowTrails, Config) -> end, ct_request("DescribeTrails", Json, Config). --spec get_trail_status([string()] ) -> ct_return(). +-spec get_trail_status(string() ) -> ct_return(). get_trail_status(Trail) -> get_trail_status(Trail, default_config()). --spec get_trail_status([string()], aws_config()) -> ct_return(). +-spec get_trail_status(string(), aws_config()) -> ct_return(). get_trail_status(Trail, Config) -> Json = [{<<"Name">>, list_to_binary(Trail)}], ct_request("GetTrailStatus", Json, Config). --spec get_event_selectors([string()]) -> ct_return(). +-spec get_event_selectors(string()) -> ct_return(). get_event_selectors(Trail) -> get_event_selectors(Trail, default_config()). --spec get_event_selectors([string()], aws_config()) -> ct_return(). +-spec get_event_selectors(string(), aws_config()) -> ct_return(). get_event_selectors(Trail, Config) -> Json = [{<<"TrailName">>, list_to_binary(Trail)}], ct_request("GetEventSelectors", Json, Config). --spec start_logging([string()] ) -> ct_return(). +-spec start_logging(string() ) -> ct_return(). start_logging(Trail) -> start_logging(Trail, default_config()). --spec start_logging([string()], aws_config()) -> ct_return(). +-spec start_logging(string(), aws_config()) -> ct_return(). start_logging(Trail, Config) -> Json = [{<<"Name">>, list_to_binary(Trail)}], ct_request("StartLogging", Json, Config). --spec stop_logging([string()] ) -> ct_return(). +-spec stop_logging(string() ) -> ct_return(). stop_logging(Trail) -> stop_logging(Trail, default_config()). --spec stop_logging([string()], aws_config()) -> ct_return(). +-spec stop_logging(string(), aws_config()) -> ct_return(). stop_logging(Trail, Config) -> Json = [{<<"Name">>, list_to_binary(Trail)}], ct_request("StopLogging", Json, Config). diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index b14202ffb..9a74bdd2c 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -465,7 +465,7 @@ log_stream_order_by(last_event_time) -> <<"LastEventTime">>. log_stream_name(), seq_token(), events() -) -> datum:either( seq_token() ). +) -> {ok, seq_token()} | {error, erlcloud_aws:httpc_result_error()}. put_logs_events(LogGroup, LogStream, SeqToken, Events) -> put_logs_events(LogGroup, LogStream, SeqToken, Events, default_config()). @@ -477,7 +477,7 @@ put_logs_events(LogGroup, LogStream, SeqToken, Events) -> seq_token(), events(), aws_config() -) -> datum:either( seq_token() ). +) -> {ok, seq_token()} | {error, erlcloud_aws:httpc_result_error()}. put_logs_events(LogGroup, LogStream, SeqToken, Events, Config) -> case diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 392e7027b..f2130ce2a 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1874,6 +1874,7 @@ dynamize_sse_specification({enabled, Enabled}) when is_boolean(Enabled) -> -type create_table_opt() :: {billing_mode, billing_mode()} | {local_secondary_indexes, local_secondary_indexes()} | {global_secondary_indexes, global_secondary_indexes()} | + {provisioned_throughput, {read_units(), write_units()}} | {sse_specification, sse_specification()} | {stream_specification, stream_specification()}. -type create_table_opts() :: [create_table_opt()]. @@ -3443,7 +3444,9 @@ tag_resource(ResourceArn, Tags, Config) -> %%%----------------------------------------------------------------------------- -type transact_get_items_transact_item_opts() :: expression_attribute_names_opt() | - projection_expression_opt(). + projection_expression_opt() | + return_consumed_capacity_opt() | + out_opt(). -type transact_get_items_opts() :: [transact_get_items_transact_item_opts()]. -type transact_get_items_get_item() :: {table_name(), key()} @@ -3456,7 +3459,7 @@ tag_resource(ResourceArn, Tags, Config) -> -type transact_get_items_return() :: ddb_return(#ddb2_transact_get_items{}, out_item()). --spec dynamize_transact_get_items_transact_items(transact_write_items_transact_items()) +-spec dynamize_transact_get_items_transact_items(transact_get_items_transact_items()) -> [jsx:json_term()]. dynamize_transact_get_items_transact_items(TransactItems) -> dynamize_maybe_list(fun dynamize_transact_get_items_transact_item/1, TransactItems). @@ -3563,9 +3566,12 @@ return_value_on_condition_check_failure_opt() -> expression_attribute_values_opt() | condition_expression_opt() | return_value_on_condition_check_failure_opt(). --type transact_write_items_condition_check_item() :: {table_name(), key(), transact_write_items_transact_item_opt()}. --type transact_write_items_delete_item() :: {table_name(), key(), transact_write_items_transact_item_opts()}. --type transact_write_items_put_item() :: {table_name(), in_item(), transact_write_items_transact_item_opts()}. +-type transact_write_items_condition_check_item() :: {table_name(), key(), transact_write_items_transact_item_opts()} + | {table_name(), key(), binary(), transact_write_items_transact_item_opts()}. +-type transact_write_items_delete_item() :: {table_name(), key()} + | {table_name(), key(), transact_write_items_transact_item_opts()}. +-type transact_write_items_put_item() :: {table_name(), in_item()} + | {table_name(), in_item(), transact_write_items_transact_item_opts()}. -type transact_write_items_update_item() :: {table_name(), key(), expression(), transact_write_items_transact_item_opts()}. -type transact_write_items_condition_check() :: {condition_check, transact_write_items_condition_check_item()}. diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index 3d79ad293..d8f30fc60 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -229,7 +229,7 @@ -define(FLOWS_MR_MIN, 1). -define(FLOWS_MR_MAX, 1000). --type filter_list() :: [{string() | atom(),[string()]}] | none. +-type filter_list() :: [{string() | atom(),[string()] | string()}] | none. -type ec2_param_list() :: [{string(),string()}]. -type ec2_selector() :: proplist(). -type ec2_token() :: string() | undefined. diff --git a/src/erlcloud_ecs.erl b/src/erlcloud_ecs.erl index a46dbf798..6e4f99fe5 100644 --- a/src/erlcloud_ecs.erl +++ b/src/erlcloud_ecs.erl @@ -678,7 +678,8 @@ container_definition_opts() -> {working_directory, <<"workingDirectory">>, fun to_binary/1} ]. --spec encode_container_definitions(Defs :: container_definition_opts()) -> [aws_opts()]. +-spec encode_container_definitions(Defs :: container_definition_opts() + | [container_definition_opts()]) -> [aws_opts()]. encode_container_definitions(Defs) -> encode_maybe_list(fun container_definition_opts/0, Defs). @@ -1177,7 +1178,8 @@ decode_container_overrides_list(V, Opts) -> %%%------------------------------------------------------------------------------ %% CreateCluster %%%------------------------------------------------------------------------------ --type create_cluster_opt() :: {cluster_name, string_param()}. +-type create_cluster_opt() :: {cluster_name, string_param()} | + out_opt(). -type create_cluster_opts() :: [create_cluster_opt()]. -spec create_cluster_opts() -> opt_table(). @@ -1227,7 +1229,8 @@ create_cluster(Opts, #aws_config{} = Config) -> cluster_opt() | deployment_configuration() | load_balancers_opt() | - role_opt(). + role_opt() | + out_opt(). -type create_service_opts() :: [create_service_opt()]. -spec create_service_opts() -> opt_table(). @@ -1329,7 +1332,8 @@ delete_cluster(ClusterName, Opts, #aws_config{} = Config) -> %%%------------------------------------------------------------------------------ %% DeleteService %%%------------------------------------------------------------------------------ --type delete_service_opt() :: {cluster, string_param()}. +-type delete_service_opt() :: {cluster, string_param()} | + out_opt(). -type delete_service_opts() :: [delete_service_opt()]. -spec delete_service_opts() -> opt_table(). @@ -1381,7 +1385,8 @@ delete_service(ServiceName, Opts, #aws_config{} = Config) -> %% DeregisterContainerInstance %%%------------------------------------------------------------------------------ -type deregister_container_instance_opt() :: {cluster, string_param()} | - {force, boolean()}. + {force, boolean()} | + out_opt(). -type deregister_container_instance_opts() :: [deregister_container_instance_opt()]. -spec deregister_container_instance_opts() -> opt_table(). @@ -1483,7 +1488,8 @@ deregister_task_definition(TaskDefinition, Opts, Config) -> %%%------------------------------------------------------------------------------ %% DescribeClusters %%%------------------------------------------------------------------------------ --type describe_clusters_opt() :: {clusters, [string_param()]}. +-type describe_clusters_opt() :: {clusters, [string_param()]} | + out_opt(). -type describe_clusters_opts() :: [describe_clusters_opt()]. -spec describe_clusters_opts() -> opt_table(). @@ -1536,7 +1542,8 @@ describe_clusters(Opts, #aws_config{} = Config) -> %%%------------------------------------------------------------------------------ %% DescribeContainerInstances %%%------------------------------------------------------------------------------ --type describe_container_instances_opt() :: {cluster, string_param()}. +-type describe_container_instances_opt() :: {cluster, string_param()} | + out_opt(). -type describe_container_instances_opts() :: [describe_container_instances_opt()]. -spec describe_container_instances_opts() -> opt_table(). @@ -1594,7 +1601,8 @@ describe_container_instances(Instances, Opts, #aws_config{} = Config) -> %%%------------------------------------------------------------------------------ %% DescribeServices %%%------------------------------------------------------------------------------ --type describe_services_opt() :: {cluster, string_param()}. +-type describe_services_opt() :: {cluster, string_param()} | + out_opt(). -type describe_services_opts() :: [describe_services_opt()]. -spec describe_services_opts() -> opt_table(). @@ -1696,7 +1704,7 @@ describe_task_definition(TaskDefinition, Opts, #aws_config{} = Config) -> %%%------------------------------------------------------------------------------ %% DescribeTasks %%%------------------------------------------------------------------------------ --type describe_tasks_opt() :: {cluster, string_param()}. +-type describe_tasks_opt() :: {cluster, string_param()} | out_opt(). -type describe_tasks_opts() :: [describe_tasks_opt()]. -spec describe_tasks_opts() -> opt_table(). @@ -1756,7 +1764,8 @@ describe_tasks(Tasks, Opts, #aws_config{} = Config) -> %% ListClusters %%%------------------------------------------------------------------------------ -type list_clusters_opt() :: {max_results, 1..100} | - {next_token, binary()}. + {next_token, binary()} | + out_opt(). -type list_clusters_opts() :: [list_clusters_opt()]. @@ -1808,7 +1817,8 @@ list_clusters(Opts, Config) -> %%%------------------------------------------------------------------------------ -type list_container_instances_opt() :: {cluster, string_param()} | {max_results, 1..100} | - {next_token, binary()}. + {next_token, binary()} | + out_opt(). -type list_container_instances_opts() :: [list_container_instances_opt()]. @@ -1861,7 +1871,8 @@ list_container_instances(Opts, Config) -> %%%------------------------------------------------------------------------------ -type list_services_opt() :: {cluster, string_param()} | {max_results, 1..100} | - {next_token, binary()}. + {next_token, binary()} | + out_opt(). -type list_services_opts() :: [list_services_opt()]. @@ -1915,7 +1926,8 @@ list_services(Opts, Config) -> -type list_task_definition_families_opt() :: {family_prefix, string_param()} | {status, active | inactive | all} | {max_results, 1..100} | - {next_token, binary()}. + {next_token, binary()} | + out_opt(). -type list_task_definition_families_opts() :: [list_task_definition_families_opt()]. @@ -1975,7 +1987,8 @@ list_task_definition_families(Opts, Config) -> {status, active | inactive} | {sort, asc | desc} | {max_results, 1..100} | - {next_token, binary()}. + {next_token, binary()} | + out_opt(). -type list_task_definitions_opts() :: [list_task_definitions_opt()]. @@ -2036,7 +2049,8 @@ list_task_definitions(Opts, Config) -> {family, string_param()} | {sort, asc | desc} | {max_results, 1..100} | - {next_token, binary()}. + {next_token, binary()} | + out_opt(). -type list_tasks_opts() :: [list_tasks_opt()]. @@ -2097,7 +2111,8 @@ list_tasks(Opts, Config) -> %%%------------------------------------------------------------------------------ -type register_task_definition_opt() :: {network_mode, ecs_network_mode()} | {task_role_arn, string_param()} | - volumes(). + volumes() | + out_opt(). -type register_task_definition_opts() :: [register_task_definition_opt()]. -spec register_task_definition_opts() -> opt_table(). @@ -2163,8 +2178,9 @@ register_task_definition(ContainerDefinitions, Family, Opts, Config) -> -type run_task_opt() :: {cluster, string_param()} | {count, pos_integer()} | task_overrides_opt() | - placement_strategy_opt() | - {started_by, string_param()}. + placement_strategy() | + {started_by, string_param()} | + out_opt(). -type run_task_opts() :: [run_task_opt()]. -spec run_task_opts() -> opt_table(). @@ -2227,7 +2243,8 @@ run_task(TaskDefinition, Opts, Config) -> %%%------------------------------------------------------------------------------ -type start_task_opt() :: {cluster, string_param()} | task_overrides_opt() | - {started_by, string_param()}. + {started_by, string_param()} | + out_opt(). -type start_task_opts() :: [start_task_opt()]. -spec start_task_opts() -> opt_table(). @@ -2295,7 +2312,8 @@ start_task(TaskDefinition, ContainerInstances, Opts, Config) -> %% StopTask %%%------------------------------------------------------------------------------ -type stop_task_opt() :: {cluster, string_param()} | - {reason, string_param()}. + {reason, string_param()} | + out_opt(). -type stop_task_opts() :: [stop_task_opt()]. -spec stop_task_opts() -> opt_table(). stop_task_opts() -> @@ -2342,7 +2360,7 @@ stop_task(Task, Opts, #aws_config{} = Config) -> %%%------------------------------------------------------------------------------ %% UpdateContainerAgent %%%------------------------------------------------------------------------------ --type update_container_agent_opts() :: [{cluster, string_param()}]. +-type update_container_agent_opts() :: [{cluster, string_param()} | out_opt()]. -spec update_container_agent_opts() -> opt_table(). update_container_agent_opts() -> [{cluster, <<"cluster">>, fun to_binary/1}]. @@ -2393,7 +2411,8 @@ update_container_agent(ContainerInstance, Opts, Config) -> -type update_service_opt() :: {cluster, string_param()} | deployment_configuration() | {desired_count, pos_integer()} | - {task_definition, string_param()}. + {task_definition, string_param()} | + out_opt(). -type update_service_opts() :: [update_service_opt()]. -spec update_service_opts() -> opt_table(). update_service_opts() -> diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index 4ef8e94f4..db09def4a 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -678,11 +678,11 @@ get_account_authorization_details(#aws_config{} = Config) -> {error, _} = Error -> Error end. --spec get_account_summary() -> {ok, proplist()} | {error, any()}. +-spec get_account_summary() -> {ok, [proplist()]} | {error, any()}. get_account_summary() -> get_account_summary(default_config()). --spec get_account_summary(aws_config()) -> {ok, proplist()} | {error, any()}. +-spec get_account_summary(aws_config()) -> {ok, [proplist()]} | {error, any()}. get_account_summary(#aws_config{} = Config) -> case iam_query(Config, "GetAccountSummary", []) of {ok, Doc} -> @@ -764,46 +764,46 @@ simulate_custom_policy(ActionNames, PolicyInputList, ContextEntries, #aws_config iam_query_all(Config, "SimulateCustomPolicy", Params, ItemPath, data_type("EvaluationResult")). --spec list_virtual_mfa_devices() -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +-spec list_virtual_mfa_devices() -> {ok, [proplist()]} | {ok, [proplist()], string()} | {error, any()}. list_virtual_mfa_devices() -> list_virtual_mfa_devices(default_config()). --spec list_virtual_mfa_devices(string() | aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +-spec list_virtual_mfa_devices(string() | aws_config()) -> {ok, [proplist()]} | {ok, [proplist()], string()} | {error, any()}. list_virtual_mfa_devices(#aws_config{} = Config) -> list_virtual_mfa_devices(undefined, undefined, undefined, Config); list_virtual_mfa_devices(AssignmentStatus) -> list_virtual_mfa_devices(AssignmentStatus, undefined, undefined, default_config()). --spec list_virtual_mfa_devices(string(), string() | aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +-spec list_virtual_mfa_devices(string(), string() | aws_config()) -> {ok, [proplist()]} | {ok, [proplist()], string()} | {error, any()}. list_virtual_mfa_devices(AssignmentStatus, #aws_config{} = Config) -> list_virtual_mfa_devices(AssignmentStatus, undefined, undefined, Config); list_virtual_mfa_devices(AssignmentStatus, Marker) -> list_virtual_mfa_devices(AssignmentStatus, Marker, undefined, default_config()). --spec list_virtual_mfa_devices(string(), string(), string()| aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +-spec list_virtual_mfa_devices(string(), string(), string()| aws_config()) -> {ok, [proplist()]} | {ok, [proplist()], string()} | {error, any()}. list_virtual_mfa_devices(AssignmentStatus, Marker, #aws_config{} = Config) -> list_virtual_mfa_devices(AssignmentStatus, Marker, undefined, Config); list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems) -> list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems, default_config()). --spec list_virtual_mfa_devices(undefined | string(), undefined | string(), undefined | string(), aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +-spec list_virtual_mfa_devices(undefined | string(), undefined | string(), undefined | string(), aws_config()) -> {ok, [proplist()]} | {ok, [proplist()], string()} | {error, any()}. list_virtual_mfa_devices(AssignmentStatus, Marker, MaxItems, #aws_config{} = Config) -> Params = make_list_virtual_mfa_devices_params(AssignmentStatus, Marker, MaxItems), ItemPath = "/ListVirtualMFADevicesResponse/ListVirtualMFADevicesResult/VirtualMFADevices/member", iam_query(Config, "ListVirtualMFADevices", Params, ItemPath, data_type("VirtualMFADeviceMetadata")). --spec list_virtual_mfa_devices_all() -> {ok, proplist()} | {error, any()}. +-spec list_virtual_mfa_devices_all() -> {ok, [proplist()]} | {error, any()}. list_virtual_mfa_devices_all() -> list_virtual_mfa_devices_all(default_config()). --spec list_virtual_mfa_devices_all(string() | aws_config()) -> {ok, proplist()} | {error, any()}. +-spec list_virtual_mfa_devices_all(string() | aws_config()) -> {ok, [proplist()]} | {error, any()}. list_virtual_mfa_devices_all(#aws_config{} = Config) -> list_virtual_mfa_devices_all(undefined, Config); list_virtual_mfa_devices_all(AssignmentStatus) -> list_virtual_mfa_devices_all(AssignmentStatus, default_config()). --spec list_virtual_mfa_devices_all(undefined | string(), aws_config()) -> {ok, proplist()} | {error, any()}. +-spec list_virtual_mfa_devices_all(undefined | string(), aws_config()) -> {ok, [proplist()]} | {error, any()}. list_virtual_mfa_devices_all(AssignmentStatus, #aws_config{} = Config) -> Params = make_list_virtual_mfa_devices_params(AssignmentStatus, undefined, undefined), ItemPath = "/ListVirtualMFADevicesResponse/ListVirtualMFADevicesResult/VirtualMFADevices/member", diff --git a/src/erlcloud_inspector.erl b/src/erlcloud_inspector.erl index 956d8e983..5abd82e61 100644 --- a/src/erlcloud_inspector.erl +++ b/src/erlcloud_inspector.erl @@ -134,16 +134,16 @@ inspector_result_fun(#aws_request{response_type = error, error_type = aws} = Req -type attribute() :: proplists:proplist(). -spec add_attributes_to_findings - (Attributes :: [attribute()], - FindingArns :: [string()]) -> + (Attributes :: attribute(), + FindingArns :: [string() | binary()]) -> inspector_return_val(). add_attributes_to_findings(Attributes, FindingArns) -> add_attributes_to_findings(Attributes, FindingArns, default_config()). -spec add_attributes_to_findings - (Attributes :: [attribute()], - FindingArns :: [string()], + (Attributes :: attribute(), + FindingArns :: [string() | binary()], Config :: aws_config()) -> inspector_return_val(). add_attributes_to_findings(Attributes, FindingArns, Config) -> @@ -159,16 +159,16 @@ add_attributes_to_findings(Attributes, FindingArns, Config) -> %%%------------------------------------------------------------------------------ -spec attach_assessment_and_rules_package - (AssessmentArn :: string(), - RulesPackageArn :: string()) -> + (AssessmentArn :: string() | binary(), + RulesPackageArn :: string() | binary()) -> inspector_return_val(). attach_assessment_and_rules_package(AssessmentArn, RulesPackageArn) -> attach_assessment_and_rules_package(AssessmentArn, RulesPackageArn, default_config()). -spec attach_assessment_and_rules_package - (AssessmentArn :: string(), - RulesPackageArn :: string(), + (AssessmentArn :: string() | binary(), + RulesPackageArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). attach_assessment_and_rules_package(AssessmentArn, RulesPackageArn, Config) -> @@ -184,16 +184,16 @@ attach_assessment_and_rules_package(AssessmentArn, RulesPackageArn, Config) -> %%%------------------------------------------------------------------------------ -spec create_application - (ApplicationName :: string(), - ResourceGroupArn :: string()) -> + (ApplicationName :: string() | binary(), + ResourceGroupArn :: string() | binary()) -> inspector_return_val(). create_application(ApplicationName, ResourceGroupArn) -> create_application(ApplicationName, ResourceGroupArn, default_config()). -spec create_application - (ApplicationName :: string(), - ResourceGroupArn :: string(), + (ApplicationName :: string() | binary(), + ResourceGroupArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). create_application(ApplicationName, ResourceGroupArn, Config) -> @@ -209,8 +209,8 @@ create_application(ApplicationName, ResourceGroupArn, Config) -> %%%------------------------------------------------------------------------------ -spec create_assessment - (ApplicationArn :: string(), - AssessmentName :: string(), + (ApplicationArn :: string() | binary(), + AssessmentName :: string() | binary(), DurationInSeconds :: integer()) -> inspector_return_val(). create_assessment(ApplicationArn, AssessmentName, DurationInSeconds) -> @@ -218,8 +218,8 @@ create_assessment(ApplicationArn, AssessmentName, DurationInSeconds) -> -spec create_assessment - (ApplicationArn :: string(), - AssessmentName :: string(), + (ApplicationArn :: string() | binary(), + AssessmentName :: string() | binary(), DurationInSeconds :: integer(), Options :: inspector_opts()) -> inspector_return_val(). @@ -228,8 +228,8 @@ create_assessment(ApplicationArn, AssessmentName, DurationInSeconds, Options) -> -spec create_assessment - (ApplicationArn :: string(), - AssessmentName :: string(), + (ApplicationArn :: string() | binary(), + AssessmentName :: string() | binary(), DurationInSeconds :: integer(), Options :: inspector_opts(), Config :: aws_config()) -> @@ -271,14 +271,14 @@ create_resource_group(ResourceGroupTags, Config) -> %%%------------------------------------------------------------------------------ -spec delete_application - (ApplicationArn :: string()) -> + (ApplicationArn :: string() | binary()) -> inspector_return_val(). delete_application(ApplicationArn) -> delete_application(ApplicationArn, default_config()). -spec delete_application - (ApplicationArn :: string(), + (ApplicationArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). delete_application(ApplicationArn, Config) -> @@ -293,14 +293,14 @@ delete_application(ApplicationArn, Config) -> %%%------------------------------------------------------------------------------ -spec delete_assessment - (AssessmentArn :: string()) -> + (AssessmentArn :: string() | binary()) -> inspector_return_val(). delete_assessment(AssessmentArn) -> delete_assessment(AssessmentArn, default_config()). -spec delete_assessment - (AssessmentArn :: string(), + (AssessmentArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). delete_assessment(AssessmentArn, Config) -> @@ -315,14 +315,14 @@ delete_assessment(AssessmentArn, Config) -> %%%------------------------------------------------------------------------------ -spec delete_run - (RunArn :: string()) -> + (RunArn :: string() | binary()) -> inspector_return_val(). delete_run(RunArn) -> delete_run(RunArn, default_config()). -spec delete_run - (RunArn :: string(), + (RunArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). delete_run(RunArn, Config) -> @@ -337,14 +337,14 @@ delete_run(RunArn, Config) -> %%%------------------------------------------------------------------------------ -spec describe_application - (ApplicationArn :: string()) -> + (ApplicationArn :: string() | binary()) -> inspector_return_val(). describe_application(ApplicationArn) -> describe_application(ApplicationArn, default_config()). -spec describe_application - (ApplicationArn :: string(), + (ApplicationArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). describe_application(ApplicationArn, Config) -> @@ -359,14 +359,14 @@ describe_application(ApplicationArn, Config) -> %%%------------------------------------------------------------------------------ -spec describe_assessment - (AssessmentArn :: string()) -> + (AssessmentArn :: string() | binary()) -> inspector_return_val(). describe_assessment(AssessmentArn) -> describe_assessment(AssessmentArn, default_config()). -spec describe_assessment - (AssessmentArn :: string(), + (AssessmentArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). describe_assessment(AssessmentArn, Config) -> @@ -401,14 +401,14 @@ describe_cross_account_access_role(Config) -> %%%------------------------------------------------------------------------------ -spec describe_finding - (FindingArn :: string()) -> + (FindingArn :: string() | binary()) -> inspector_return_val(). describe_finding(FindingArn) -> describe_finding(FindingArn, default_config()). -spec describe_finding - (FindingArn :: string(), + (FindingArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). describe_finding(FindingArn, Config) -> @@ -423,14 +423,14 @@ describe_finding(FindingArn, Config) -> %%%------------------------------------------------------------------------------ -spec describe_resource_group - (ResourceGroupArn :: string()) -> + (ResourceGroupArn :: string() | binary()) -> inspector_return_val(). describe_resource_group(ResourceGroupArn) -> describe_resource_group(ResourceGroupArn, default_config()). -spec describe_resource_group - (ResourceGroupArn :: string(), + (ResourceGroupArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). describe_resource_group(ResourceGroupArn, Config) -> @@ -445,14 +445,14 @@ describe_resource_group(ResourceGroupArn, Config) -> %%%------------------------------------------------------------------------------ -spec describe_rules_package - (RulesPackageArn :: string()) -> + (RulesPackageArn :: string() | binary()) -> inspector_return_val(). describe_rules_package(RulesPackageArn) -> describe_rules_package(RulesPackageArn, default_config()). -spec describe_rules_package - (RulesPackageArn :: string(), + (RulesPackageArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). describe_rules_package(RulesPackageArn, Config) -> @@ -467,14 +467,14 @@ describe_rules_package(RulesPackageArn, Config) -> %%%------------------------------------------------------------------------------ -spec describe_run - (RunArn :: string()) -> + (RunArn :: string() | binary()) -> inspector_return_val(). describe_run(RunArn) -> describe_run(RunArn, default_config()). -spec describe_run - (RunArn :: string(), + (RunArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). describe_run(RunArn, Config) -> @@ -489,16 +489,16 @@ describe_run(RunArn, Config) -> %%%------------------------------------------------------------------------------ -spec detach_assessment_and_rules_package - (AssessmentArn :: string(), - RulesPackageArn :: string()) -> + (AssessmentArn :: string() | binary(), + RulesPackageArn :: string() | binary()) -> inspector_return_val(). detach_assessment_and_rules_package(AssessmentArn, RulesPackageArn) -> detach_assessment_and_rules_package(AssessmentArn, RulesPackageArn, default_config()). -spec detach_assessment_and_rules_package - (AssessmentArn :: string(), - RulesPackageArn :: string(), + (AssessmentArn :: string() | binary(), + RulesPackageArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). detach_assessment_and_rules_package(AssessmentArn, RulesPackageArn, Config) -> @@ -514,14 +514,14 @@ detach_assessment_and_rules_package(AssessmentArn, RulesPackageArn, Config) -> %%%------------------------------------------------------------------------------ -spec get_assessment_telemetry - (AssessmentArn :: string()) -> + (AssessmentArn :: string() | binary()) -> inspector_return_val(). get_assessment_telemetry(AssessmentArn) -> get_assessment_telemetry(AssessmentArn, default_config()). -spec get_assessment_telemetry - (AssessmentArn :: string(), + (AssessmentArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). get_assessment_telemetry(AssessmentArn, Config) -> @@ -565,14 +565,14 @@ list_applications(Options, Config) -> %%%------------------------------------------------------------------------------ -spec list_assessment_agents - (AssessmentArn :: string()) -> + (AssessmentArn :: string() | binary()) -> inspector_return_val(). list_assessment_agents(AssessmentArn) -> list_assessment_agents(AssessmentArn, []). -spec list_assessment_agents - (AssessmentArn :: string(), + (AssessmentArn :: string() | binary(), Options :: inspector_opts()) -> inspector_return_val(). list_assessment_agents(AssessmentArn, Options) -> @@ -580,7 +580,7 @@ list_assessment_agents(AssessmentArn, Options) -> -spec list_assessment_agents - (AssessmentArn :: string(), + (AssessmentArn :: string() | binary(), Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). @@ -626,14 +626,14 @@ list_assessments(Options, Config) -> %%%------------------------------------------------------------------------------ -spec list_attached_assessments - (RulesPackageArn :: string()) -> + (RulesPackageArn :: string() | binary()) -> inspector_return_val(). list_attached_assessments(RulesPackageArn) -> list_attached_assessments(RulesPackageArn, []). -spec list_attached_assessments - (RulesPackageArn :: string(), + (RulesPackageArn :: string() | binary(), Options :: inspector_opts()) -> inspector_return_val(). list_attached_assessments(RulesPackageArn, Options) -> @@ -641,7 +641,7 @@ list_attached_assessments(RulesPackageArn, Options) -> -spec list_attached_assessments - (RulesPackageArn :: string(), + (RulesPackageArn :: string() | binary(), Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). @@ -658,14 +658,14 @@ list_attached_assessments(RulesPackageArn, Options, Config) -> %%%------------------------------------------------------------------------------ -spec list_attached_rules_packages - (AssessmentArn :: string()) -> + (AssessmentArn :: string() | binary()) -> inspector_return_val(). list_attached_rules_packages(AssessmentArn) -> list_attached_rules_packages(AssessmentArn, []). -spec list_attached_rules_packages - (AssessmentArn :: string(), + (AssessmentArn :: string() | binary(), Options :: inspector_opts()) -> inspector_return_val(). list_attached_rules_packages(AssessmentArn, Options) -> @@ -673,7 +673,7 @@ list_attached_rules_packages(AssessmentArn, Options) -> -spec list_attached_rules_packages - (AssessmentArn :: string(), + (AssessmentArn :: string() | binary(), Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). @@ -704,7 +704,7 @@ list_findings(Options) -> -spec list_findings - (Options :: inspector_opts(), + (Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). list_findings(Options, Config) -> @@ -733,7 +733,7 @@ list_rules_packages(Options) -> -spec list_rules_packages - (Options :: inspector_opts(), + (Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). list_rules_packages(Options, Config) -> @@ -762,7 +762,7 @@ list_runs(Options) -> -spec list_runs - (Options :: inspector_opts(), + (Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). list_runs(Options, Config) -> @@ -777,14 +777,14 @@ list_runs(Options, Config) -> %%%------------------------------------------------------------------------------ -spec list_tags_for_resource - (ResourceArn :: string()) -> + (ResourceArn :: string() | binary()) -> inspector_return_val(). list_tags_for_resource(ResourceArn) -> list_tags_for_resource(ResourceArn, []). -spec list_tags_for_resource - (ResourceArn :: string(), + (ResourceArn :: string() | binary(), Options :: inspector_opts()) -> inspector_return_val(). list_tags_for_resource(ResourceArn, Options) -> @@ -792,7 +792,7 @@ list_tags_for_resource(ResourceArn, Options) -> -spec list_tags_for_resource - (ResourceArn :: string(), + (ResourceArn :: string() | binary(), Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). @@ -826,7 +826,7 @@ localize_text(LocalizedTexts) -> -spec localize_text (LocalizedTexts :: [localized_text()], - Locale :: string()) -> + Locale :: string() | binary()) -> inspector_return_val(). localize_text(LocalizedTexts, Locale) -> localize_text(LocalizedTexts, Locale, default_config()). @@ -834,7 +834,7 @@ localize_text(LocalizedTexts, Locale) -> -spec localize_text (LocalizedTexts :: [localized_text()], - Locale :: string(), + Locale :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). localize_text(LocalizedTexts, Locale, Config) -> @@ -850,14 +850,14 @@ localize_text(LocalizedTexts, Locale, Config) -> %%%------------------------------------------------------------------------------ -spec preview_agents_for_resource_group - (ResourceGroupArn :: string()) -> + (ResourceGroupArn :: string() | binary()) -> inspector_return_val(). preview_agents_for_resource_group(ResourceGroupArn) -> preview_agents_for_resource_group(ResourceGroupArn, []). -spec preview_agents_for_resource_group - (ResourceGroupArn :: string(), + (ResourceGroupArn :: string() | binary(), Options :: inspector_opts()) -> inspector_return_val(). preview_agents_for_resource_group(ResourceGroupArn, Options) -> @@ -865,7 +865,7 @@ preview_agents_for_resource_group(ResourceGroupArn, Options) -> -spec preview_agents_for_resource_group - (ResourceGroupArn :: string(), + (ResourceGroupArn :: string() | binary(), Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). @@ -882,14 +882,14 @@ preview_agents_for_resource_group(ResourceGroupArn, Options, Config) -> %%%------------------------------------------------------------------------------ -spec register_cross_account_access_role - (RoleArn :: string()) -> + (RoleArn :: string() | binary()) -> inspector_return_val(). register_cross_account_access_role(RoleArn) -> register_cross_account_access_role(RoleArn, []). -spec register_cross_account_access_role - (RoleArn :: string(), + (RoleArn :: string() | binary(), Options :: inspector_opts()) -> inspector_return_val(). register_cross_account_access_role(RoleArn, Options) -> @@ -897,7 +897,7 @@ register_cross_account_access_role(RoleArn, Options) -> -spec register_cross_account_access_role - (RoleArn :: string(), + (RoleArn :: string() | binary(), Options :: inspector_opts(), Config :: aws_config()) -> inspector_return_val(). @@ -914,16 +914,16 @@ register_cross_account_access_role(RoleArn, Options, Config) -> %%%------------------------------------------------------------------------------ -spec remove_attributes_from_findings - (AttributeKeys :: [string()], - FindingArns :: [string()]) -> + (AttributeKeys :: [string() | binary()], + FindingArns :: [string() | binary()]) -> inspector_return_val(). remove_attributes_from_findings(AttributeKeys, FindingArns) -> remove_attributes_from_findings(AttributeKeys, FindingArns, default_config()). -spec remove_attributes_from_findings - (AttributeKeys :: [string()], - FindingArns :: [string()], + (AttributeKeys :: [string() | binary()], + FindingArns :: [string() | binary()], Config :: aws_config()) -> inspector_return_val(). remove_attributes_from_findings(AttributeKeys, FindingArns, Config) -> @@ -939,16 +939,16 @@ remove_attributes_from_findings(AttributeKeys, FindingArns, Config) -> %%%------------------------------------------------------------------------------ -spec run_assessment - (AssessmentArn :: string(), - RunName :: string()) -> + (AssessmentArn :: string() | binary(), + RunName :: string() | binary()) -> inspector_return_val(). run_assessment(AssessmentArn, RunName) -> run_assessment(AssessmentArn, RunName, default_config()). -spec run_assessment - (AssessmentArn :: string(), - RunName :: string(), + (AssessmentArn :: string() | binary(), + RunName :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). run_assessment(AssessmentArn, RunName, Config) -> @@ -964,7 +964,7 @@ run_assessment(AssessmentArn, RunName, Config) -> %%%------------------------------------------------------------------------------ -spec set_tags_for_resource - (ResourceArn :: string(), + (ResourceArn :: string() | binary(), Tags :: term()) -> inspector_return_val(). set_tags_for_resource(ResourceArn, Tags) -> @@ -972,8 +972,8 @@ set_tags_for_resource(ResourceArn, Tags) -> -spec set_tags_for_resource - (ResourceArn :: string(), - Tags :: string(), + (ResourceArn :: string() | binary(), + Tags :: term(), Config :: aws_config()) -> inspector_return_val(). set_tags_for_resource(ResourceArn, Tags, Config) -> @@ -989,7 +989,7 @@ set_tags_for_resource(ResourceArn, Tags, Config) -> %%%------------------------------------------------------------------------------ -spec start_data_collection - (AssessmentArn :: string()) -> + (AssessmentArn :: string() | binary()) -> inspector_return_val(). start_data_collection(RunArn) -> start_data_collection(RunArn, default_config()). @@ -1011,7 +1011,7 @@ start_data_collection(AssessmentArn, Config) -> %%%------------------------------------------------------------------------------ -spec stop_data_collection - (AssessmentArn :: string()) -> + (AssessmentArn :: string() | binary()) -> inspector_return_val(). stop_data_collection(RunArn) -> stop_data_collection(RunArn, default_config()). @@ -1033,18 +1033,18 @@ stop_data_collection(AssessmentArn, Config) -> %%%------------------------------------------------------------------------------ -spec update_application - (ApplicationArn :: string(), - ApplicationName :: string(), - ResourceGroupArn :: string()) -> + (ApplicationArn :: string() | binary(), + ApplicationName :: string() | binary(), + ResourceGroupArn :: string() | binary()) -> inspector_return_val(). update_application(ApplicationArn, ApplicationName, ResourceGroupArn) -> update_application(ApplicationArn, ApplicationName, ResourceGroupArn, default_config()). -spec update_application - (ApplicationArn :: string(), - ApplicationName :: string(), - ResourceGroupArn :: string(), + (ApplicationArn :: string() | binary(), + ApplicationName :: string() | binary(), + ResourceGroupArn :: string() | binary(), Config :: aws_config()) -> inspector_return_val(). update_application(ApplicationArn, ApplicationName, ResourceGroupArn, Config) -> @@ -1061,8 +1061,8 @@ update_application(ApplicationArn, ApplicationName, ResourceGroupArn, Config) -> %%%------------------------------------------------------------------------------ -spec update_assessment - (AssessmentArn :: string(), - AssessmentName :: string(), + (AssessmentArn :: string() | binary(), + AssessmentName :: string() | binary(), DurationInSeconds :: integer()) -> inspector_return_val(). update_assessment(AssessmentArn, AssessmentName, DurationInSeconds) -> @@ -1070,8 +1070,8 @@ update_assessment(AssessmentArn, AssessmentName, DurationInSeconds) -> -spec update_assessment - (AssessmentArn :: string(), - AssessmentName :: string(), + (AssessmentArn :: string() | binary(), + AssessmentName :: string() | binary(), DurationInSeconds :: integer(), Config :: aws_config()) -> inspector_return_val(). diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index 09818d8cc..15f1c9aa0 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -555,12 +555,12 @@ get_shard_iterator_request(Config, Json) -> %% @end %%------------------------------------------------------------------------------ --spec get_records(string()) -> erlcloud_kinesis_impl:json_return(). +-spec get_records(binary()) -> erlcloud_kinesis_impl:json_return(). get_records(ShardIterator) -> Json = [{<<"ShardIterator">>, ShardIterator}], get_normalized_records(default_config(), Json). --spec get_records(string(), get_records_limit()| aws_config()) -> erlcloud_kinesis_impl:json_return(). +-spec get_records(binary(), get_records_limit()| aws_config()) -> erlcloud_kinesis_impl:json_return(). get_records(ShardIterator, Config) when is_record(Config, aws_config) -> Json = [{<<"ShardIterator">>, ShardIterator}], get_normalized_records(Config, Json); @@ -712,8 +712,8 @@ put_record(StreamName, PartitionKey, Data, ExplicitHashKey, Ordering, Options, C %% @end %%------------------------------------------------------------------------------ --type put_records_item() :: {Data :: string(), PartitionKey :: string()} - | {Data :: string(), ExplicitHashKey :: string(), PartitionKey :: string()}. +-type put_records_item() :: {Data :: binary(), PartitionKey :: binary()} + | {Data :: binary(), ExplicitHashKey :: binary(), PartitionKey :: binary()}. -type put_records_items() :: [put_records_item()]. -spec put_records(binary(), put_records_items()) -> erlcloud_kinesis_impl:json_return(). @@ -771,13 +771,13 @@ prepare_put_records_item({Data, ExplicitHashKey, PartitionKey}, Fun) -> %% @end %%------------------------------------------------------------------------------ --spec merge_shards(binary(), string(), string()) -> erlcloud_kinesis_impl:json_return(). +-spec merge_shards(binary(), binary(), binary()) -> erlcloud_kinesis_impl:json_return(). merge_shards(StreamName, AdjacentShardToMerge, ShardToMerge) -> Json = [{<<"StreamName">>, StreamName}, {<<"AdjacentShardToMerge">>, AdjacentShardToMerge}, {<<"ShardToMerge">>, ShardToMerge}], erlcloud_kinesis_impl:request(default_config(), "Kinesis_20131202.MergeShards", Json). --spec merge_shards(binary(), string(), string(), aws_config()) -> erlcloud_kinesis_impl:json_return(). +-spec merge_shards(binary(), binary(), binary(), aws_config()) -> erlcloud_kinesis_impl:json_return(). merge_shards(StreamName, AdjacentShardToMerge, ShardToMerge, Config) when is_record(Config, aws_config) -> Json = [{<<"StreamName">>, StreamName}, {<<"AdjacentShardToMerge">>, AdjacentShardToMerge}, {<<"ShardToMerge">>, ShardToMerge}], @@ -800,13 +800,13 @@ merge_shards(StreamName, AdjacentShardToMerge, ShardToMerge, Config) when is_rec %% @end %%------------------------------------------------------------------------------ --spec split_shards(binary(), string(), string()) -> erlcloud_kinesis_impl:json_return(). +-spec split_shards(binary(), binary(), binary()) -> erlcloud_kinesis_impl:json_return(). split_shards(StreamName, ShardToSplit, NewStartingHashKey) -> Json = [{<<"StreamName">>, StreamName}, {<<"ShardToSplit">>, ShardToSplit}, {<<"NewStartingHashKey">>, NewStartingHashKey}], erlcloud_kinesis_impl:request(default_config(), "Kinesis_20131202.SplitShard", Json). --spec split_shards(binary(), string(), string(), aws_config()) -> erlcloud_kinesis_impl:json_return(). +-spec split_shards(binary(), binary(), binary(), aws_config()) -> erlcloud_kinesis_impl:json_return(). split_shards(StreamName, ShardToSplit, NewStartingHashKey, Config) when is_record(Config, aws_config) -> Json = [{<<"StreamName">>, StreamName}, {<<"ShardToSplit">>, ShardToSplit}, {<<"NewStartingHashKey">>, NewStartingHashKey}], diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index ec7d1cdf7..570060ce2 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -164,9 +164,9 @@ create_event_source_mapping(EventSourceArn, FunctionName, %% %%------------------------------------------------------------------------------ -spec create_function(Code :: erlcloud_lambda_code(), - FunctionName :: string(), - Handler :: string(), - Role :: string(), + FunctionName :: binary(), + Handler :: binary(), + Role :: binary(), Runtime :: runtime(), Options :: proplist()) -> return_val(). create_function(#erlcloud_lambda_code{} = Code, @@ -175,9 +175,9 @@ create_function(#erlcloud_lambda_code{} = Code, Runtime, Options, default_config()). -spec create_function(Code :: erlcloud_lambda_code(), - FunctionName :: string(), - Handler :: string(), - Role :: string(), + FunctionName :: binary(), + Handler :: binary(), + Role :: binary(), Runtime :: runtime(), Options :: proplist(), Config :: aws_config()) -> return_val(). diff --git a/src/erlcloud_route53.erl b/src/erlcloud_route53.erl index f540bc884..51cf8ade9 100644 --- a/src/erlcloud_route53.erl +++ b/src/erlcloud_route53.erl @@ -77,7 +77,7 @@ configure(AccessKeyID, SecretAccessKey, Host) -> %% @end %% -------------------------------------------------------------------- -spec describe_zone(ZoneId :: string()) -> - {ok, list(aws_route53_zone())} | + {ok, aws_route53_zone()} | {error, term()}. describe_zone(ZoneId) -> describe_zone(ZoneId, erlcloud_aws:default_config()). @@ -88,7 +88,7 @@ describe_zone(ZoneId) -> %% -------------------------------------------------------------------- -spec describe_zone(ZoneId :: string(), AwsConfig :: aws_config()) -> - {ok, list(aws_route53_zone())} | + {ok, aws_route53_zone()} | {error, term()}. describe_zone(ZoneId, AwsConfig) -> describe_zone(ZoneId, [], AwsConfig). @@ -100,7 +100,7 @@ describe_zone(ZoneId, AwsConfig) -> -spec describe_zone(ZoneId :: string(), Options :: list({string(), string()}), AwsConfig :: aws_config()) -> - {ok, list(aws_route53_zone())} | + {ok, aws_route53_zone()} | {error, term()}. describe_zone(ZoneId, Options, Config) when is_list(Options), is_record(Config, aws_config) -> @@ -132,7 +132,7 @@ describe_zones(AwsConfig) -> %% @doc Describes all zones using provided config + AWS options %% @end %% -------------------------------------------------------------------- --spec describe_zones(Options :: list({string(), string()}), +-spec describe_zones(Options :: list({string(), string() | integer()}), AwsConfig :: aws_config()) -> {ok, list(aws_route53_zone())} | {ok, list(aws_route53_zone()), string()} | @@ -498,7 +498,7 @@ key_replace_or_add(Key, Value, List) -> end. -spec route53_query(get | post, aws_config(), string(), string(), - list({string(), string()}), string()) -> + list({string(), string() | integer()}), string()) -> {ok, term()} | {error, term()}. route53_query(Method, Config, Action, Path, Params, ApiVersion) -> QParams = [{"Action", Action}, {"Version", ApiVersion} | Params], diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 1a32742e6..ff4599480 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -470,11 +470,11 @@ put_bucket_policy(BucketName, Policy, Config) when is_list(BucketName), is_binary(Policy), is_record(Config, aws_config) -> s3_simple_request(Config, put, BucketName, "/", "policy", [], Policy, []). --spec get_bucket_lifecycle(BucketName::string()) -> ok | {error, Reason::term()}. +-spec get_bucket_lifecycle(BucketName::string()) -> {ok, list(proplist())} | {error, Reason::term()}. get_bucket_lifecycle(BucketName) -> get_bucket_lifecycle(BucketName, default_config()). --spec get_bucket_lifecycle(BucketName::string(), Config::aws_config()) -> {ok, Policy::string()} | {error, Reason::term()}. +-spec get_bucket_lifecycle(BucketName::string(), Config::aws_config()) -> {ok, list(proplist())} | {error, Reason::term()}. get_bucket_lifecycle(BucketName, Config) when is_record(Config, aws_config) -> case s3_request2(Config, get, BucketName, "/", "lifecycle", [], <<>>, []) of @@ -1701,7 +1701,7 @@ delete_bucket_encryption(BucketName, Config) -> %% takes an S3 bucket notification configuration and creates an xmerl simple %% form out of it. %% for the examples of input / output of this function, see tests. --spec create_notification_xml(proplist()) -> tuple(). +-spec create_notification_xml(list(proplist())) -> tuple(). create_notification_xml(Confs) -> {'NotificationConfiguration', [create_notification_xml(ConfName, Params) || [{ConfName, Params}] <- Confs]}. diff --git a/src/erlcloud_waf.erl b/src/erlcloud_waf.erl index cd5c25094..22875c395 100644 --- a/src/erlcloud_waf.erl +++ b/src/erlcloud_waf.erl @@ -899,7 +899,7 @@ update_ip_set(ChangeToken, IPSetId, Updates, Config) -> %% http://docs.aws.amazon.com/waf/latest/APIReference/API_UpdateRule.html %%%------------------------------------------------------------------------------ -spec update_rule(ChangeToken :: string() | binary(), - RuleId :: string(), + RuleId :: string() | binary(), Updates :: [waf_rule_update()]) -> waf_return_val(). update_rule(ChangeToken, RuleId, Updates) -> @@ -1034,7 +1034,8 @@ update_xss_match_set(ChangeToken, XssMatchSetId, Updates, Config) -> waf_rule_update() | waf_size_constraint_update() | waf_sql_injection_match_set_update() | - waf_web_acl_update()) -> + waf_web_acl_update() | + waf_xss_match_set_update()) -> proplists:proplist(). transform_to_proplist(#waf_byte_match_set_update{action = Action, byte_match_tuple = ByteMatchTuple}) -> [{<<"Action">>, get_update_action(Action)}, diff --git a/test/erlcloud_autoscaling_tests.erl b/test/erlcloud_autoscaling_tests.erl index 833268573..1e5846110 100644 --- a/test/erlcloud_autoscaling_tests.erl +++ b/test/erlcloud_autoscaling_tests.erl @@ -140,7 +140,7 @@ output_expect_seq(Responses) -> %% output_test converts an output_test specifier into an eunit test generator --type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string(), term()}}. +-type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string() | [string()], term()}}. -spec output_test(fun(), output_test_spec(), fun()) -> tuple(). output_test(Fun, {Line, {Description, Response, Result}}, OutputFun) -> {Description, diff --git a/test/erlcloud_aws_tests.erl b/test/erlcloud_aws_tests.erl index 9f5e88625..3d584b495 100644 --- a/test/erlcloud_aws_tests.erl +++ b/test/erlcloud_aws_tests.erl @@ -7,16 +7,16 @@ request_test_() -> {foreach, fun start/0, fun stop/1, - [fun request_default_test/1, - fun request_retry_test/1, - fun request_prot_host_port_str_test/1, - fun request_prot_host_port_int_test/1, - fun get_service_status_test/1, - fun auto_config_with_env/1]}. + [fun test_request_default/1, + fun test_request_retry/1, + fun test_request_prot_host_port_str/1, + fun test_request_prot_host_port_int/1, + fun test_get_service_status/1, + fun test_auto_config_with_env/1]}. start() -> meck:new(erlcloud_httpc), - meck:expect(erlcloud_httpc, request, fun(_,_,_,_,_,_) -> {ok, {{200, "OK"}, [], ok}} end), + meck:expect(erlcloud_httpc, request, fun(_,_,_,_,_,_) -> {ok, {{200, "OK"}, [], <<"OkBody">>}} end), ok. stop(_) -> @@ -28,12 +28,12 @@ config() -> retry = fun erlcloud_retry:default_retry/1, retry_num = 3}. -request_default_test(_) -> - ok = erlcloud_aws:aws_request(get, "host", "/", [], "id", "key"), +test_request_default(_) -> + <<"OkBody">> = erlcloud_aws:aws_request(get, "host", "/", [], "id", "key"), Url = get_url_from_history(meck:history(erlcloud_httpc)), test_url(https, "host", 443, "/", Url). -request_retry_test(_) -> +test_request_retry(_) -> Response400 = {ok, {{400, "Bad Request"}, [], <<"\n" " \n" @@ -66,27 +66,27 @@ request_retry_test(_) -> erlcloud_aws:aws_request_xml4(get, "host", "/", [], "any", config()); (ResponseSeq) -> meck:sequence(erlcloud_httpc, request, 6, ResponseSeq), - erlcloud_aws:aws_request(get, "host", "/", [], config()) + <<"OkBody">> = erlcloud_aws:aws_request(get, "host", "/", [], config()) end, - [?_assertNotException(_, _, <<"OkBody">> = MeckAndRequest([Response400, Response200])), - ?_assertNotException(_, _, <<"OkBody">> = MeckAndRequest([Response400, Response500, Response200])), - ?_assertNotException(_, _, <<"OkBody">> = MeckAndRequest([Response429, Response200])), + [?_assertMatch(<<"OkBody">>, MeckAndRequest([Response400, Response200])), + ?_assertMatch(<<"OkBody">>, MeckAndRequest([Response400, Response500, Response200])), + ?_assertMatch(<<"OkBody">>, MeckAndRequest([Response429, Response200])), ?_assertMatch({error, {http_error, 400, "Bad Request", _ErrorMsg}}, MeckAndRequest({[Response400, Response500, Response400, Response200], xml4})) ]. -request_prot_host_port_str_test(_) -> - ok = erlcloud_aws:aws_request(get, "http", "host1", "9999", "/path1", [], "id", "key"), +test_request_prot_host_port_str(_) -> + <<"OkBody">> = erlcloud_aws:aws_request(get, "http", "host1", "9999", "/path1", [], "id", "key"), Url = get_url_from_history(meck:history(erlcloud_httpc)), test_url(http, "host1", 9999, "/path1", Url). -request_prot_host_port_int_test(_) -> - ok = erlcloud_aws:aws_request(get, "http", "host1", 9999, "/path1", [], "id", "key"), +test_request_prot_host_port_int(_) -> + <<"OkBody">> = erlcloud_aws:aws_request(get, "http", "host1", 9999, "/path1", [], "id", "key"), Url = get_url_from_history(meck:history(erlcloud_httpc)), test_url(http, "host1", 9999, "/path1", Url). -get_service_status_test(_) -> +test_get_service_status(_) -> StatusJsonS3 = jsx:encode( [{<<"archive">>, [[{<<"service_name">>, @@ -157,7 +157,7 @@ get_service_status_test(_) -> ]. -auto_config_with_env(_) -> +test_auto_config_with_env(_) -> % Note: meck do not support os module X_AWS_ACCESS = os:getenv("AWS_ACCESS_KEY_ID"), X_AWS_SECRET = os:getenv("AWS_SECRET_ACCESS_KEY"), diff --git a/test/erlcloud_cloudtrail_tests.erl b/test/erlcloud_cloudtrail_tests.erl index 581966fa9..57e2e6acd 100644 --- a/test/erlcloud_cloudtrail_tests.erl +++ b/test/erlcloud_cloudtrail_tests.erl @@ -125,7 +125,7 @@ output_expect(Response) -> end. %% output_test converts an output_test specifier into an eunit test generator --type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string(), term()}}. +-type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string() | binary(), term()}}. -spec output_test(fun(), output_test_spec()) -> tuple(). output_test(Fun, {Line, {Description, Response, Result}}) -> {Description, diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index a1af991c9..c82047f99 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -19,7 +19,7 @@ -define(_f(F), fun() -> F end). -export([validate_body/2]). - + %%%=================================================================== %%% Test entry points %%%=================================================================== @@ -149,7 +149,7 @@ validate_body(Body, Expected) -> %% Validates the request body and responds with the provided response. -spec input_expect(string(), expected_body()) -> fun(). input_expect(Response, Expected) -> - fun(_Url, post, _Headers, Body, _Timeout, _Config) -> + fun(_Url, post, _Headers, Body, _Timeout, _Config) -> validate_body(Body, Expected), {ok, {{200, "OK"}, [], list_to_binary(Response)}} end. @@ -159,7 +159,7 @@ input_expect(Response, Expected) -> -spec input_test(string(), input_test_spec()) -> tuple(). input_test(Response, {Line, {Description, Fun, Expected}}) when is_list(Description) -> - {Description, + {Description, {Line, fun() -> meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), @@ -181,8 +181,8 @@ input_tests(Response, Tests) -> %% returns the mock of the erlcloud_httpc function output tests expect to be called. -spec output_expect(string()) -> fun(). output_expect(Response) -> - fun(_Url, post, _Headers, _Body, _Timeout, _Config) -> - {ok, {{200, "OK"}, [], list_to_binary(Response)}} + fun(_Url, post, _Headers, _Body, _Timeout, _Config) -> + {ok, {{200, "OK"}, [], list_to_binary(Response)}} end. %% output_test converts an output_test specifier into an eunit test generator @@ -204,9 +204,9 @@ output_test(Fun, {Line, {Description, Response, Result}}) -> end}}. %% output_test(Fun, {Line, {Response, Result}}) -> %% output_test(Fun, {Line, {"", Response, Result}}). - + %% output_tests converts a list of output_test specifiers into an eunit test generator --spec output_tests(fun(), [output_test_spec()]) -> [term()]. +-spec output_tests(fun(), [output_test_spec()]) -> [term()]. output_tests(Fun, Tests) -> [output_test(Fun, Test) || Test <- Tests]. @@ -218,7 +218,7 @@ output_tests(Fun, Tests) -> -spec httpc_response(pos_integer(), string()) -> tuple(). httpc_response(Code, Body) -> {ok, {{Code, ""}, [], list_to_binary(Body)}}. - + -type error_test_spec() :: {pos_integer(), {string(), list(), term()}}. -spec error_test(fun(), error_test_spec()) -> tuple(). error_test(Fun, {Line, {Description, Responses, Result}}) -> @@ -232,7 +232,7 @@ error_test(Fun, {Line, {Description, Responses, Result}}) -> Actual = Fun(), ?assertEqual(Result, Actual) end}}. - + -spec error_tests(fun(), [error_test_spec()]) -> [term()]. error_tests(Fun, Tests) -> [error_test(Fun, Test) || Test <- Tests]. @@ -243,13 +243,16 @@ error_tests(Fun, Tests) -> input_exception_test_() -> [?_assertError({erlcloud_ddb, {invalid_attr_value, {n, "string"}}}, - erlcloud_ddb2:get_item(<<"Table">>, {<<"K">>, {n, "string"}})), + erlcloud_ddb2:get_item(<<"Table">>, {<<"K">>, {n, "string"}}))] + ++ input_exception_failures_test_(). + +-dialyzer({nowarn_function, input_exception_failures_test_/0}). +input_exception_failures_test_() -> %% This test causes an expected dialyzer error - ?_assertError({erlcloud_ddb, {invalid_item, <<"Attr">>}}, + [?_assertError({erlcloud_ddb, {invalid_item, <<"Attr">>}}, erlcloud_ddb2:put_item(<<"Table">>, <<"Attr">>)), ?_assertError({erlcloud_ddb, {invalid_opt, {myopt, myval}}}, - erlcloud_ddb2:list_tables([{myopt, myval}])) - ]. + erlcloud_ddb2:list_tables([{myopt, myval}]))]. %% Error handling tests based on: %% http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ErrorHandling.html @@ -260,12 +263,12 @@ error_handling_tests(_) -> \"status\":{\"S\":\"online\"} }, \"ConsumedCapacityUnits\": 1 -}" +}" ), OkResult = {ok, [{<<"friends">>, [<<"Lynda">>, <<"Aaron">>]}, {<<"status">>, <<"online">>}]}, - Tests = + Tests = [?_ddb_test( {"Test retry after ProvisionedThroughputExceededException", [httpc_response(400, " @@ -287,7 +290,7 @@ error_handling_tests(_) -> OkResponse], OkResult}) ], - + error_tests(?_f(erlcloud_ddb2:get_item(<<"table">>, {<<"k">>, <<"v">>})), Tests), TransactionErrorResult = {error, {<<"TransactionCanceledException">>, @@ -351,13 +354,13 @@ batch_get_item_input_tests(_) -> [?_ddb_test( {"BatchGetItem example request", ?_f(erlcloud_ddb2:batch_get_item( - [{<<"Forum">>, + [{<<"Forum">>, [{<<"Name">>, {s, <<"Amazon DynamoDB">>}}, - {<<"Name">>, {s, <<"Amazon RDS">>}}, + {<<"Name">>, {s, <<"Amazon RDS">>}}, {<<"Name">>, {s, <<"Amazon Redshift">>}}], [{attributes_to_get, [<<"Name">>, <<"Threads">>, <<"Messages">>, <<"Views">>]}]}, - {<<"Thread">>, - [[{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + {<<"Thread">>, + [[{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"Concurrent reads">>}}]], [{attributes_to_get, [<<"Tags">>, <<"Message">>]}]}], [{return_consumed_capacity, total}])), " @@ -512,7 +515,7 @@ batch_get_item_input_tests(_) -> input_tests(Response, Tests). batch_get_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"BatchGetItem example response", " { @@ -586,10 +589,10 @@ batch_get_item_output_tests(_) -> ] }", {ok, #ddb2_batch_get_item - {consumed_capacity = + {consumed_capacity = [#ddb2_consumed_capacity{table_name = <<"Forum">>, capacity_units = 3}, #ddb2_consumed_capacity{table_name = <<"Thread">>, capacity_units = 1}], - responses = + responses = [#ddb2_batch_get_item_response {table = <<"Forum">>, items = [[{<<"Name">>, <<"Amazon DynamoDB">>}, @@ -644,19 +647,19 @@ batch_get_item_output_tests(_) -> } }", {ok, #ddb2_batch_get_item - {responses = [], - unprocessed_keys = - [{<<"Forum">>, + {responses = [], + unprocessed_keys = + [{<<"Forum">>, [[{<<"Name">>, {s, <<"Amazon DynamoDB">>}}], - [{<<"Name">>, {s, <<"Amazon RDS">>}}], + [{<<"Name">>, {s, <<"Amazon RDS">>}}], [{<<"Name">>, {s, <<"Amazon Redshift">>}}]], [{attributes_to_get, [<<"Name">>, <<"Threads">>, <<"Messages">>, <<"Views">>]}]}, - {<<"Thread">>, - [[{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + {<<"Thread">>, + [[{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"Concurrent reads">>}}]], [{attributes_to_get, [<<"Tags">>, <<"Message">>]}]}]}}}) ], - + output_tests(?_f(erlcloud_ddb2:batch_get_item([{<<"table">>, [{<<"k">>, <<"v">>}]}], [{out, record}])), Tests). %% BatchWriteItem test based on the API examples: @@ -799,7 +802,7 @@ batch_write_item_input_tests(_) -> input_tests(Response, Tests). batch_write_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"BatchWriteItem example response", " { @@ -829,7 +832,7 @@ batch_write_item_output_tests(_) -> {ok, #ddb2_batch_write_item {consumed_capacity = [#ddb2_consumed_capacity{table_name = <<"Forum">>, capacity_units = 3}], item_collection_metrics = undefined, - unprocessed_items = [{<<"Forum">>, + unprocessed_items = [{<<"Forum">>, [{put, [{<<"Name">>, {s, <<"Amazon ElastiCache">>}}, {<<"Category">>, {s, <<"Amazon Web Services">>}}]}]}]}}}), ?_ddb_test( @@ -862,7 +865,7 @@ batch_write_item_output_tests(_) -> } }", {ok, #ddb2_batch_write_item - {consumed_capacity = undefined, + {consumed_capacity = undefined, item_collection_metrics = undefined, unprocessed_items = [{<<"Forum">>, [{put, [{<<"Name">>, {s, <<"Amazon DynamoDB">>}}, @@ -913,8 +916,8 @@ batch_write_item_output_tests(_) -> } }", {ok, #ddb2_batch_write_item - {consumed_capacity = undefined, - item_collection_metrics = + {consumed_capacity = undefined, + item_collection_metrics = [{<<"Table1">>, [#ddb2_item_collection_metrics {item_collection_key = <<"value1">>, @@ -930,7 +933,7 @@ batch_write_item_output_tests(_) -> ]}], unprocessed_items = []}}}) ], - + output_tests(?_f(erlcloud_ddb2:batch_write_item([], [{out, record}])), Tests). %% CreateBackup test based on the API examples: @@ -991,10 +994,10 @@ create_global_table_input_tests(_) -> {region_name, <<"us-west-2">>}])), " { \"GlobalTableName\": \"Thread\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" - },{ + },{ \"RegionName\": \"us-west-2\" } ] @@ -1002,11 +1005,11 @@ create_global_table_input_tests(_) -> }), ?_ddb_test( {"CreateGlobalTable example request (1 region)", - ?_f(erlcloud_ddb2:create_global_table(<<"Thread">>, #ddb2_replica{region_name = <<"us-west-2">>})), " + ?_f(erlcloud_ddb2:create_global_table(<<"Thread">>, [#ddb2_replica{region_name = <<"us-west-2">>}])), " { \"GlobalTableName\": \"Thread\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-west-2\" } ] @@ -1014,15 +1017,15 @@ create_global_table_input_tests(_) -> })], Response = " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"CREATING\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" - },{ + },{ \"RegionName\": \"us-west-2\" } ] @@ -1032,17 +1035,17 @@ create_global_table_input_tests(_) -> %% CreateGlobalTable output test: create_global_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"CreateGlobalTable example response with CREATING status ", " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"CREATING\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" } ] @@ -1058,15 +1061,15 @@ create_global_table_output_tests(_) -> ?_ddb_test( {"CreateGlobalTable example response with ACTIVE status ", " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"ACTIVE\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" - },{ + },{ \"RegionName\": \"eu-west-1\" } ] @@ -1079,7 +1082,7 @@ create_global_table_output_tests(_) -> global_table_status = active, replication_group = [#ddb2_replica_description{region_name = <<"us-east-1">>}, #ddb2_replica_description{region_name = <<"eu-west-1">>}]}}})], - output_tests(?_f(erlcloud_ddb2:create_global_table(<<"Thread">>, {region_name, <<"us-east-1">>})), Tests). + output_tests(?_f(erlcloud_ddb2:create_global_table(<<"Thread">>, [{region_name, <<"us-east-1">>}])), Tests). %% CreateTable test based on the API examples: %% http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_CreateTable.html @@ -1093,7 +1096,7 @@ create_table_input_tests(_) -> {<<"Subject">>, s}, {<<"LastPostDateTime">>, s}], {<<"ForumName">>, <<"Subject">>}, - 5, + 5, 5, [{local_secondary_indexes, [{<<"LastPostIndex">>, <<"LastPostDateTime">>, keys_only}]}, @@ -1136,7 +1139,7 @@ create_table_input_tests(_) -> \"WriteCapacityUnits\": 5 } } - ], + ], \"TableName\": \"Thread\", \"KeySchema\": [ { @@ -1180,12 +1183,12 @@ create_table_input_tests(_) -> {<<"Subject">>, s}, {<<"LastPostDateTime">>, s}], {<<"ForumName">>, <<"Subject">>}, - 5, + 5, 5, [{local_secondary_indexes, [{<<"LastPostIndex">>, <<"LastPostDateTime">>, {include, [<<"Author">>, <<"Body">>]}}]}, {global_secondary_indexes, - [{<<"SubjectIndex">>, {<<"Subject">>, <<"LastPostDateTime">>}, {include, [<<"Author">>]}, 10, 5}]}] + [{<<"SubjectIndex">>, {<<"Subject">>, <<"LastPostDateTime">>}, {include, [<<"Author">>]}, 10, 5}]}] )), " { \"AttributeDefinitions\": [ @@ -1226,7 +1229,7 @@ create_table_input_tests(_) -> \"WriteCapacityUnits\": 5 } } - ], + ], \"TableName\": \"Thread\", \"KeySchema\": [ { @@ -1269,7 +1272,7 @@ create_table_input_tests(_) -> <<"Thread">>, {<<"ForumName">>, s}, <<"ForumName">>, - 1, + 1, 1 )), " { @@ -1656,7 +1659,7 @@ delete_backup_output_tests(_) -> output_tests(?_f(erlcloud_ddb2:delete_backup(<<"arn:aws:dynamodb:">>)), Tests). create_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"CreateTable example response", " { @@ -1767,7 +1770,7 @@ create_table_output_tests(_) -> item_count = 0, key_schema = {<<"ForumName">>, <<"LastPostDateTime">>}, projection = keys_only}], - global_secondary_indexes = + global_secondary_indexes = [#ddb2_global_secondary_index_description{ index_name = <<"SubjectIndex">>, index_size_bytes = 2048, @@ -1782,7 +1785,7 @@ create_table_output_tests(_) -> read_capacity_units = 3, write_capacity_units = 4} }], - provisioned_throughput = + provisioned_throughput = #ddb2_provisioned_throughput_description{ last_decrease_date_time = undefined, last_increase_date_time = undefined, @@ -1844,7 +1847,7 @@ create_table_output_tests(_) -> \"WriteCapacityUnits\": 4 } } - ], + ], \"CreationDateTime\": 1.36372808007E9, \"ItemCount\": 0, \"KeySchema\": [ @@ -1906,7 +1909,7 @@ create_table_output_tests(_) -> item_count = 0, key_schema = {<<"ForumName">>, <<"LastPostDateTime">>}, projection = {include, [<<"Author">>, <<"Body">>]}}], - global_secondary_indexes = + global_secondary_indexes = [#ddb2_global_secondary_index_description{ index_name = <<"SubjectIndex">>, index_size_bytes = 2048, @@ -1921,7 +1924,7 @@ create_table_output_tests(_) -> read_capacity_units = 3, write_capacity_units = 4} }], - provisioned_throughput = + provisioned_throughput = #ddb2_provisioned_throughput_description{ last_decrease_date_time = undefined, last_increase_date_time = undefined, @@ -1973,7 +1976,7 @@ create_table_output_tests(_) -> item_count = 0, key_schema = <<"ForumName">>, local_secondary_indexes = undefined, - provisioned_throughput = + provisioned_throughput = #ddb2_provisioned_throughput_description{ last_decrease_date_time = undefined, last_increase_date_time = undefined, @@ -2007,8 +2010,8 @@ create_table_output_tests(_) -> ], \"ProvisionedThroughput\": { \"NumberOfDecreasesToday\": 0, - \"ReadCapacityUnits\": 0, - \"WriteCapacityUnits\": 0 + \"ReadCapacityUnits\": 1, + \"WriteCapacityUnits\": 1 }, \"TableName\": \"Thread\", \"TableSizeBytes\": 0, @@ -2030,13 +2033,13 @@ create_table_output_tests(_) -> last_decrease_date_time = undefined, last_increase_date_time = undefined, number_of_decreases_today = 0, - read_capacity_units = 0, - write_capacity_units = 0}, + read_capacity_units = 1, + write_capacity_units = 1}, table_name = <<"Thread">>, table_size_bytes = 0, table_status = creating}}}) ], - + output_tests(?_f(erlcloud_ddb2:create_table(<<"name">>, [{<<"key">>, s}], <<"key">>, 5, 10)), Tests). %% DeleteItem test based on the API examples: @@ -2045,7 +2048,7 @@ delete_item_input_tests(_) -> Tests = [?_ddb_test( {"DeleteItem example request", - ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, + ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], [{return_values, all_old}, @@ -2091,7 +2094,7 @@ delete_item_input_tests(_) -> }), ?_ddb_test( {"DeleteItem return metrics", - ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, + ?_f(erlcloud_ddb2:delete_item(<<"Thread">>, {<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, [{return_consumed_capacity, total}, {return_item_collection_metrics, size}])), " @@ -2134,7 +2137,7 @@ delete_item_input_tests(_) -> input_tests(Response, Tests). delete_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"DeleteItem example response", " { @@ -2194,7 +2197,7 @@ delete_item_output_tests(_) -> size_estimate_range_gb = {1,2}} }}}) ], - + output_tests(?_f(erlcloud_ddb2:delete_item(<<"table">>, {<<"k">>, <<"v">>}, [{out, record}])), Tests). %% DeleteTable test based on the API examples: @@ -2227,7 +2230,7 @@ delete_table_input_tests(_) -> input_tests(Response, Tests). delete_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"DeleteTable example response", " { @@ -2253,7 +2256,7 @@ delete_table_output_tests(_) -> table_size_bytes = 0, table_status = deleting}}}) ], - + output_tests(?_f(erlcloud_ddb2:delete_table(<<"name">>)), Tests). %% DescribeBackup test based on the API examples: @@ -2582,15 +2585,15 @@ describe_global_table_input_tests(_) -> })], Response = " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"ACTIVE\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" - },{ + },{ \"RegionName\": \"us-west-2\" } ] @@ -2600,17 +2603,17 @@ describe_global_table_input_tests(_) -> %% DescribeGlobalTable output test: describe_global_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"DescribeGlobalTable example response with CREATING status ", " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"CREATING\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" } ] @@ -2626,15 +2629,15 @@ describe_global_table_output_tests(_) -> ?_ddb_test( {"DescribeGlobalTable example response with ACTIVE status ", " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"ACTIVE\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" - },{ + },{ \"RegionName\": \"eu-west-1\" } ] @@ -2983,7 +2986,7 @@ describe_table_input_tests(_) -> input_tests(Response, Tests). describe_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"DescribeTable example response", " { @@ -3032,7 +3035,7 @@ describe_table_output_tests(_) -> \"WriteCapacityUnits\": 4 } } - ], + ], \"CreationDateTime\": 1.363729002358E9, \"GlobalTableVersion\": \"2019.11.21\", \"ItemCount\": 0, @@ -3101,7 +3104,7 @@ describe_table_output_tests(_) -> item_count = 0, key_schema = {<<"ForumName">>, <<"LastPostDateTime">>}, projection = keys_only}], - global_secondary_indexes = + global_secondary_indexes = [#ddb2_global_secondary_index_description{ index_name = <<"SubjectIndex">>, index_size_bytes = 2048, @@ -3115,8 +3118,8 @@ describe_table_output_tests(_) -> number_of_decreases_today = 2, read_capacity_units = 3, write_capacity_units = 4} - }], - provisioned_throughput = + }], + provisioned_throughput = #ddb2_provisioned_throughput_description{ last_decrease_date_time = undefined, last_increase_date_time = undefined, @@ -3131,7 +3134,7 @@ describe_table_output_tests(_) -> table_size_bytes = 0, table_status = active}}}) ], - + output_tests(?_f(erlcloud_ddb2:describe_table(<<"name">>)), Tests). @@ -3356,7 +3359,7 @@ describe_time_to_live_input_tests(_) -> }" })], Response = " -{ +{ \"TimeToLiveDescription\": { \"AttributeName\": \"ExpirationTime\", \"TimeToLiveStatus\": \"ENABLED\" @@ -3366,7 +3369,7 @@ describe_time_to_live_input_tests(_) -> %% DescribeTimeToLive test: describe_time_to_live_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"DescribeTimeToLive example response with enabled TTL", " { @@ -3441,7 +3444,7 @@ get_item_input_tests(_) -> [?_ddb_test( {"GetItem example request, with fully specified keys", ?_f(erlcloud_ddb2:get_item(<<"Thread">>, - [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, + [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, consistent_read, @@ -3452,7 +3455,7 @@ get_item_input_tests(_) -> Example1Response}), ?_ddb_test( {"GetItem example request, with inferred key types", - ?_f(erlcloud_ddb2:get_item(<<"Thread">>, + ?_f(erlcloud_ddb2:get_item(<<"Thread">>, [{<<"ForumName">>, "Amazon DynamoDB"}, {<<"Subject">>, <<"How do I update multiple items?">>}], [{attributes_to_get, [<<"LastPostDateTime">>, <<"Message">>, <<"Tags">>]}, @@ -3514,7 +3517,7 @@ get_item_input_tests(_) -> input_tests(Response, Tests). get_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"GetItem example response", " { @@ -3573,19 +3576,19 @@ get_item_output_tests(_) -> {<<"empty_map">>, []}], consumed_capacity = undefined}}}), ?_ddb_test( - {"GetItem item not found", + {"GetItem item not found", "{}", {ok, #ddb2_get_item{item = undefined}}}), ?_ddb_test( - {"GetItem no attributes returned", + {"GetItem no attributes returned", "{\"Item\":{}}", {ok, #ddb2_get_item{item = []}}}) ], - + output_tests(?_f(erlcloud_ddb2:get_item(<<"table">>, {<<"k">>, <<"v">>}, [{out, record}])), Tests). get_item_output_typed_tests(_) -> - Tests = + Tests = [?_ddb_test( {"GetItem typed test all attribute types", " {\"Item\": @@ -3618,7 +3621,7 @@ get_item_output_typed_tests(_) -> {<<"empty_map">>, {m, []}}], consumed_capacity = undefined}}}) ], - + output_tests(?_f(erlcloud_ddb2:get_item( <<"table">>, {<<"k">>, <<"v">>}, [{out, typed_record}])), Tests). @@ -3705,7 +3708,7 @@ list_global_tables_input_tests(_) -> }), ?_ddb_test( {"ListGlobalTables empty request", - ?_f(erlcloud_ddb2:list_global_tables()), + ?_f(erlcloud_ddb2:list_global_tables()), "{}" }) @@ -3713,22 +3716,22 @@ list_global_tables_input_tests(_) -> Response = " { - \"GlobalTables\": [ - { + \"GlobalTables\": [ + { \"GlobalTableName\": \"Forum\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-west-2\" - },{ + },{ \"RegionName\": \"us-east-1\" } ] - },{ + },{ \"GlobalTableName\": \"Thread\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" - },{ + },{ \"RegionName\": \"eu-west-1\" } ] @@ -3740,26 +3743,26 @@ list_global_tables_input_tests(_) -> %% ListGlobalTables output test: list_global_tables_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"ListGlobalTables example response", " { - \"GlobalTables\": [ - { + \"GlobalTables\": [ + { \"GlobalTableName\": \"Forum\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-west-2\" - },{ + },{ \"RegionName\": \"us-east-1\" } ] - },{ + },{ \"GlobalTableName\": \"Thread\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" - },{ + },{ \"RegionName\": \"eu-west-1\" } ] @@ -3778,7 +3781,7 @@ list_global_tables_output_tests(_) -> replication_group = [#ddb2_replica{region_name = <<"us-east-1">>}, #ddb2_replica{region_name = <<"eu-west-1">>}]}]}}}) ], - + output_tests(?_f(erlcloud_ddb2:list_global_tables([{out, record}])), Tests). %% ListTables test based on the API examples: @@ -3795,7 +3798,7 @@ list_tables_input_tests(_) -> }), ?_ddb_test( {"ListTables empty request", - ?_f(erlcloud_ddb2:list_tables()), + ?_f(erlcloud_ddb2:list_tables()), "{}" }) @@ -3809,7 +3812,7 @@ list_tables_input_tests(_) -> input_tests(Response, Tests). list_tables_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"ListTables example response", " { @@ -3820,7 +3823,7 @@ list_tables_output_tests(_) -> {last_evaluated_table_name = <<"Thread">>, table_names = [<<"Forum">>, <<"Reply">>, <<"Thread">>]}}}) ], - + output_tests(?_f(erlcloud_ddb2:list_tables([{out, record}])), Tests). %% ListTagsOfResource test based on the API: @@ -3853,7 +3856,7 @@ list_tags_of_resource_input_tests(_) -> input_tests(Response, Tests). list_tags_of_resource_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"ListTagsOfResource example response", " { @@ -3874,7 +3877,7 @@ list_tags_of_resource_output_tests(_) -> tags = [{<<"example_key1">>, <<"example_value1">>}, {<<"example_key2">>, <<"example_value2">>}]}}}) ], - + output_tests(?_f(erlcloud_ddb2:list_tags_of_resource(<<"arn:aws:dynamodb:us-east-1:111122223333:table/Forum">>, [{out, record}])), Tests). @@ -3885,7 +3888,7 @@ put_item_input_tests(_) -> Tests = [?_ddb_test( {"PutItem example request", - ?_f(erlcloud_ddb2:put_item(<<"Thread">>, + ?_f(erlcloud_ddb2:put_item(<<"Thread">>, [{<<"LastPostedBy">>, <<"fred@example.com">>}, {<<"ForumName">>, <<"Amazon DynamoDB">>}, {<<"LastPostDateTime">>, <<"201303190422">>}, @@ -3927,7 +3930,7 @@ put_item_input_tests(_) -> }), ?_ddb_test( {"PutItem float inputs", - ?_f(erlcloud_ddb2:put_item(<<"Thread">>, + ?_f(erlcloud_ddb2:put_item(<<"Thread">>, [{<<"typed float">>, {n, 1.2}}, {<<"untyped float">>, 3.456}, {<<"mixed set">>, {ns, [7.8, 9.0, 10]}}], @@ -4038,7 +4041,7 @@ put_item_input_tests(_) -> input_tests(Response, Tests). put_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"PutItem example response", " { @@ -4135,7 +4138,7 @@ put_item_output_tests(_) -> ] }}}) ], - + output_tests(?_f(erlcloud_ddb2:put_item(<<"table">>, [], [{out, record}])), Tests). %% Query test based on the API examples: @@ -4275,7 +4278,7 @@ q_input_tests(_) -> input_tests(Response, Tests). q_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"Query example 1 response", " { @@ -4371,12 +4374,12 @@ q_output_tests(_) -> } }", {ok, #ddb2_q{count = 17, - last_evaluated_key = + last_evaluated_key = [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"Exclusive key can have 3 parts">>}}, {<<"LastPostDateTime">>, {s, <<"20130102054211">>}}] }}}) ], - + output_tests(?_f(erlcloud_ddb2:q(<<"table">>, [{<<"k">>, <<"v">>, eq}], [{out, record}])), Tests). %% RestoreTableFromBackup test based on the API examples: @@ -4900,7 +4903,7 @@ scan_input_tests(_) -> }), ?_ddb_test( {"Scan example 2 request", - ?_f(erlcloud_ddb2:scan(<<"Reply">>, + ?_f(erlcloud_ddb2:scan(<<"Reply">>, [{scan_filter, [{<<"PostedBy">>, <<"joe@example.com">>, eq}]}, {return_consumed_capacity, total}])), " { @@ -4929,7 +4932,7 @@ scan_input_tests(_) -> }), ?_ddb_test( {"Scan exclusive start key", - ?_f(erlcloud_ddb2:scan(<<"Reply">>, + ?_f(erlcloud_ddb2:scan(<<"Reply">>, [{exclusive_start_key, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"LastPostDateTime">>, {n, 20130102054211}}]}])), " { @@ -5073,7 +5076,7 @@ scan_input_tests(_) -> input_tests(Response, Tests). scan_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"Scan example 1 response", " { @@ -5277,7 +5280,7 @@ scan_output_tests(_) -> {<<"LastPostDateTime">>, {n, 20130102054211}}], scanned_count = 4}}}) ], - + output_tests(?_f(erlcloud_ddb2:scan(<<"name">>, [{out, record}])), Tests). %% TagResource test based on the API: @@ -5308,7 +5311,7 @@ tag_resource_input_tests(_) -> input_tests(Response, Tests). tag_resource_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"ListTagsOfResource example response", "", ok}) @@ -5749,7 +5752,7 @@ untag_resource_input_tests(_) -> input_tests(Response, Tests). untag_resource_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"ListTagsOfResource example response", "", ok}) @@ -5816,7 +5819,7 @@ update_item_input_tests(_) -> Tests = [?_ddb_test( {"UpdateItem example request", - ?_f(erlcloud_ddb2:update_item(<<"Thread">>, + ?_f(erlcloud_ddb2:update_item(<<"Thread">>, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"How do I update multiple items?">>}}], [{<<"LastPostedBy">>, {s, <<"alice@example.com">>}, put}], @@ -5933,7 +5936,7 @@ update_item_input_tests(_) -> input_tests(Response, Tests). update_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"UpdateItem example response", " { @@ -5974,7 +5977,7 @@ update_item_output_tests(_) -> }", {ok, []}}) ], - + output_tests(?_f(erlcloud_ddb2:update_item(<<"table">>, {<<"k">>, <<"v">>}, [])), Tests). %% UpdateGlobalTable input test: @@ -5985,9 +5988,9 @@ update_global_table_input_tests(_) -> ?_f(erlcloud_ddb2:update_global_table(<<"Thread">>, [{create, {region_name, <<"us-east-1">>}}])), " { \"GlobalTableName\": \"Thread\", - \"ReplicaUpdates\": [ - { - \"Create\": { + \"ReplicaUpdates\": [ + { + \"Create\": { \"RegionName\": \"us-east-1\" } } @@ -5996,12 +5999,12 @@ update_global_table_input_tests(_) -> }), ?_ddb_test( {"UpdateGlobalTable example request (delete)", - ?_f(erlcloud_ddb2:update_global_table(<<"Thread">>, {delete, #ddb2_replica{region_name = <<"us-west-2">>}})), " + ?_f(erlcloud_ddb2:update_global_table(<<"Thread">>, [{delete, #ddb2_replica{region_name = <<"us-west-2">>}}])), " { \"GlobalTableName\": \"Thread\", - \"ReplicaUpdates\": [ - { - \"Delete\": { + \"ReplicaUpdates\": [ + { + \"Delete\": { \"RegionName\": \"us-west-2\" } } @@ -6014,13 +6017,13 @@ update_global_table_input_tests(_) -> {delete, {region_name, <<"eu-west-1">>}}])), " { \"GlobalTableName\": \"Thread\", - \"ReplicaUpdates\": [ - { - \"Create\": { + \"ReplicaUpdates\": [ + { + \"Create\": { \"RegionName\": \"us-east-1\" } - },{ - \"Delete\": { + },{ + \"Delete\": { \"RegionName\": \"eu-west-1\" } } @@ -6029,13 +6032,13 @@ update_global_table_input_tests(_) -> })], Response = " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"UPDATING\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" } ] @@ -6045,17 +6048,17 @@ update_global_table_input_tests(_) -> %% UpdateGlobalTable output test: update_global_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"UpdateGlobalTable example response with UPDATING status", " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"UPDATING\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"us-east-1\" } ] @@ -6071,13 +6074,13 @@ update_global_table_output_tests(_) -> ?_ddb_test( {"UpdateGlobalTable example response with DELETING status", " { - \"GlobalTableDescription\": { + \"GlobalTableDescription\": { \"CreationDateTime\": 1519161181.107, \"GlobalTableArn\": \"arn:aws:dynamodb::111122223333:global-table/Thread\", \"GlobalTableName\": \"Thread\", \"GlobalTableStatus\": \"DELETING\", - \"ReplicationGroup\": [ - { + \"ReplicationGroup\": [ + { \"RegionName\": \"eu-west-1\" } ] @@ -6089,7 +6092,7 @@ update_global_table_output_tests(_) -> global_table_name = <<"Thread">>, global_table_status = deleting, replication_group = [#ddb2_replica_description{region_name = <<"eu-west-1">>}]}}})], - output_tests(?_f(erlcloud_ddb2:update_global_table(<<"Thread">>, {create, {region_name, <<"us-east-1">>}})), Tests). + output_tests(?_f(erlcloud_ddb2:update_global_table(<<"Thread">>, [{create, {region_name, <<"us-east-1">>}}])), Tests). update_global_table_settings_input_tests(_) -> ReadUnits = 10, @@ -6690,7 +6693,7 @@ update_table_input_tests(_) -> ], Response = " -{ +{ \"TableDescription\": { \"AttributeDefinitions\": [ { @@ -6752,10 +6755,10 @@ update_table_input_tests(_) -> input_tests(Response, Tests). update_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"UpdateTable example response", " -{ +{ \"TableDescription\": { \"AttributeDefinitions\": [ { @@ -6801,7 +6804,7 @@ update_table_output_tests(_) -> \"WriteCapacityUnits\": 4 } } - ], + ], \"CreationDateTime\": 1.363801528686E9, \"ItemCount\": 0, \"KeySchema\": [ @@ -6859,7 +6862,7 @@ update_table_output_tests(_) -> item_count = 0, key_schema = {<<"ForumName">>, <<"LastPostDateTime">>}, projection = keys_only}], - global_secondary_indexes = + global_secondary_indexes = [#ddb2_global_secondary_index_description{ index_name = <<"SubjectIndex">>, index_size_bytes = 2048, @@ -6873,8 +6876,8 @@ update_table_output_tests(_) -> number_of_decreases_today = 2, read_capacity_units = 3, write_capacity_units = 4} - }], - provisioned_throughput = + }], + provisioned_throughput = #ddb2_provisioned_throughput_description{ last_decrease_date_time = undefined, last_increase_date_time = 1363801701.282, @@ -6885,7 +6888,7 @@ update_table_output_tests(_) -> table_size_bytes = 0, table_status = updating}}}) ], - + output_tests(?_f(erlcloud_ddb2:update_table(<<"name">>, 5, 15)), Tests). update_table_replica_auto_scaling_input_tests(_) -> @@ -7214,10 +7217,10 @@ update_time_to_live_input_tests(_) -> \"AttributeName\": \"ExpirationTime\", \"Enabled\": false } -}" +}" })], Response = " -{ +{ \"TimeToLiveSpecification\": { \"AttributeName\": \"ExpirationTime\", \"Enabled\": true @@ -7227,7 +7230,7 @@ update_time_to_live_input_tests(_) -> %% UpdateTimeToLive test: update_time_to_live_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"UpdateTimeToLive example response", " { @@ -7239,5 +7242,5 @@ update_time_to_live_output_tests(_) -> {ok, #ddb2_time_to_live_specification{ attribute_name = <<"ExpirationTime">>, enabled = true}}})], - output_tests(?_f(erlcloud_ddb2:update_time_to_live(<<"SessionData">>, + output_tests(?_f(erlcloud_ddb2:update_time_to_live(<<"SessionData">>, [{attribute_name, <<"ExpirationTime">>}, {enabled, true}])), Tests). diff --git a/test/erlcloud_ddb_tests.erl b/test/erlcloud_ddb_tests.erl index dd194d0d3..b78839518 100644 --- a/test/erlcloud_ddb_tests.erl +++ b/test/erlcloud_ddb_tests.erl @@ -180,9 +180,13 @@ error_tests(Fun, Tests) -> input_exception_test_() -> [?_assertError({erlcloud_ddb, {invalid_attr_value, {n, "string"}}}, - erlcloud_ddb:get_item(<<"Table">>, {n, "string"})), - %% This test causes an expected dialyzer error - ?_assertError({erlcloud_ddb, {invalid_item, <<"Attr">>}}, + erlcloud_ddb:get_item(<<"Table">>, {n, "string"}))] + ++ input_exception_failures_test_(). + +-dialyzer({nowarn_function, input_exception_failures_test_/0}). +input_exception_failures_test_() -> + %% This test causes an expected dialyzer error + [?_assertError({erlcloud_ddb, {invalid_item, <<"Attr">>}}, erlcloud_ddb:put_item(<<"Table">>, <<"Attr">>)), ?_assertError({erlcloud_ddb, {invalid_opt, {myopt, myval}}}, erlcloud_ddb:list_tables([{myopt, myval}])) diff --git a/test/erlcloud_ddb_util_tests.erl b/test/erlcloud_ddb_util_tests.erl index 2b4829a5c..7fb7dc875 100644 --- a/test/erlcloud_ddb_util_tests.erl +++ b/test/erlcloud_ddb_util_tests.erl @@ -652,11 +652,11 @@ q_all_tests(_) -> multi_call_tests(Tests). q_all_attributes() -> - Item1 = <<"item_1">>, - Item2 = <<"item_2">>, + Item1 = [{<<"key">>, <<"item_1">>}], + Item2 = [{<<"key">>, <<"item_2">>}], meck:new(EDDB = erlcloud_ddb2), meck:sequence(EDDB, q, 4, [ - {ok, #ddb2_q{last_evaluated_key = <<"key">>, + {ok, #ddb2_q{last_evaluated_key = {<<"key">>, <<"last1">>}, items = [Item1]}}, {ok, #ddb2_q{last_evaluated_key = undefined, items = [Item2]}} @@ -668,7 +668,7 @@ q_all_attributes() -> q_all_count() -> meck:new(EDDB = erlcloud_ddb2), meck:sequence(EDDB, q, 4, [ - {ok, #ddb2_q{last_evaluated_key = <<"key">>, + {ok, #ddb2_q{last_evaluated_key = {<<"key">>, <<"last1">>}, items = undefined, count = 2}}, {ok, #ddb2_q{last_evaluated_key = undefined, @@ -799,11 +799,11 @@ scan_all_tests(_) -> multi_call_tests(Tests). scan_all_attributes() -> - Item1 = <<"item_1">>, - Item2 = <<"item_2">>, + Item1 = [{<<"key">>, <<"item_1">>}], + Item2 = [{<<"key">>, <<"item_2">>}], meck:new(EDDB = erlcloud_ddb2), meck:sequence(EDDB, scan, 3, [ - {ok, #ddb2_scan{last_evaluated_key = <<"key">>, + {ok, #ddb2_scan{last_evaluated_key = {<<"key">>, <<"last1">>}, items = [Item1]}}, {ok, #ddb2_scan{last_evaluated_key = undefined, items = [Item2]}} @@ -815,7 +815,7 @@ scan_all_attributes() -> scan_all_count() -> meck:new(EDDB = erlcloud_ddb2), meck:sequence(EDDB, scan, 3, [ - {ok, #ddb2_scan{last_evaluated_key = <<"key">>, + {ok, #ddb2_scan{last_evaluated_key = {<<"key">>, <<"last1">>}, items = undefined, count = 2}}, {ok, #ddb2_scan{last_evaluated_key = undefined, @@ -1516,8 +1516,13 @@ set_out_opt_test_() -> erlcloud_ddb_util:set_out_opt([{typed_out, true}]))}, {"set_out_opt typed_out=false sets out=record", ?_assertEqual([{out, record}], - erlcloud_ddb_util:set_out_opt([{typed_out, false}]))}, - {"set_out_opt preserves location of out opt", + erlcloud_ddb_util:set_out_opt([{typed_out, false}]))}] + ++ set_out_opt_failures_test_(). + +% these will generate dialyzer warnings, so we isolate them +-dialyzer({nowarn_function, set_out_opt_failures_test_/0}). +set_out_opt_failures_test_() -> + [{"set_out_opt preserves location of out opt", ?_assertEqual([{foo, bar}, {out, record}], erlcloud_ddb_util:set_out_opt([{typed_out, false}, {foo, bar}, {out, record}]))}, {"set_out_opt overrides out opt with valid value", diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 35246887e..a4e57a93a 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -1223,6 +1223,7 @@ describe_spot_price_history_test_() -> {timeout, 60, fun () -> test_pagination(Tests, generate_spot_price_history_response, describe_spot_price_history, [], ["", "", [], ""]) end} . +-dialyzer({nowarn_function, describe_spot_price_history_boundaries_test_/0}). describe_spot_price_history_boundaries_test_() -> [ ?_assertException(error, function_clause, erlcloud_ec2:describe_spot_price_history(["", "", [], ""], 4, undefined)), diff --git a/test/erlcloud_ecs_tests.erl b/test/erlcloud_ecs_tests.erl index ef03b7c18..f6aed1f30 100644 --- a/test/erlcloud_ecs_tests.erl +++ b/test/erlcloud_ecs_tests.erl @@ -1319,7 +1319,7 @@ describe_container_instances_output_tests(_) -> failures = [] }}}) ], - output_tests(?_f(erlcloud_ecs:describe_container_instances("f9cc75bb-0c94-46b9-bf6d-49d320bc1551", [{out, record}])), Tests). + output_tests(?_f(erlcloud_ecs:describe_container_instances(["f9cc75bb-0c94-46b9-bf6d-49d320bc1551"], [{out, record}])), Tests). %% DescribeServices test based on the API examples: %% http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_DescribeServices.html @@ -3284,7 +3284,7 @@ input_tests(Response, Tests) -> %%%=================================================================== %% returns the mock of the erlcloud_httpc function output tests expect to be called. --spec output_expect(string()) -> fun(). +-spec output_expect(binary()) -> fun(). output_expect(Response) -> fun(_Url, post, _Headers, _Body, _Timeout, _Config) -> {ok, {{200, "OK"}, [], Response}} diff --git a/test/erlcloud_iam_tests.erl b/test/erlcloud_iam_tests.erl index aaaa4fb5d..dd37f53e3 100644 --- a/test/erlcloud_iam_tests.erl +++ b/test/erlcloud_iam_tests.erl @@ -198,7 +198,7 @@ input_tests(Response, Tests) -> %%%=================================================================== %% returns the mock of the erlcloud_httpc function output tests expect to be called. --spec output_expect(string()) -> fun(). +-spec output_expect(string()) -> meck:ret_spec(). output_expect(Response) -> meck:val({ok, {{200, "OK"}, [], list_to_binary(Response)}}). @@ -207,7 +207,7 @@ output_expect_seq(Responses) -> meck:seq([{ok, {{200, "OK"}, [], list_to_binary(Response)}} || Response <- Responses]). %% output_test converts an output_test specifier into an eunit test generator --type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string(), term()}}. +-type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string() | [string()], term()}}. -spec output_test(fun(), output_test_spec(), fun()) -> tuple(). output_test(Fun, {Line, {Description, Response, Result}}, OutputFun) -> {Description, @@ -923,7 +923,7 @@ get_access_key_last_used_output_tests() -> {access_key_last_used_service_name, "s3"}] }}) ], - output_tests(?_f(erlcloud_iam:get_access_key_last_used_output("KEYID")), Tests). + output_tests(?_f(erlcloud_iam:get_access_key_last_used("KEYID")), Tests). %% ListUsers test based on the API examples: %% http://docs.aws.amazon.com/IAM/latest/APIReference/API_ListUsers.html diff --git a/test/erlcloud_inspector_tests.erl b/test/erlcloud_inspector_tests.erl index e438d6949..9fd311665 100644 --- a/test/erlcloud_inspector_tests.erl +++ b/test/erlcloud_inspector_tests.erl @@ -499,7 +499,7 @@ update_assessment_tests(_) -> %%% Input test helpers %%%=================================================================== --type expected_body() :: string(). +-type expected_body() :: binary(). sort_json([{_, _} | _] = Json) -> %% Value is an object @@ -526,7 +526,7 @@ validate_body(Body, Expected) -> %% returns the mock of the erlcloud_httpc function input tests expect to be called. %% Validates the request body and responds with the provided response. --spec input_expect(string(), expected_body()) -> fun(). +-spec input_expect(binary(), expected_body()) -> fun(). input_expect(Response, Expected) -> fun(_Url, post, _Headers, Body, _Timeout, _Config) -> validate_body(Body, Expected), @@ -536,7 +536,7 @@ input_expect(Response, Expected) -> %% input_test converts an input_test specifier into an eunit test generator -type input_test_spec() :: {pos_integer(), {fun(), expected_body()} | {string(), fun(), expected_body()}}. --spec input_test(string(), input_test_spec()) -> tuple(). +-spec input_test(binary(), input_test_spec()) -> tuple(). input_test(Response, {Line, {Description, Fun, Expected}}) when is_list(Description) -> {Description, @@ -549,7 +549,7 @@ input_test(Response, {Line, {Description, Fun, Expected}}) %% input_tests converts a list of input_test specifiers into an eunit test generator --spec input_tests(string(), [input_test_spec()]) -> [tuple()]. +-spec input_tests(binary(), [input_test_spec()]) -> [tuple()]. input_tests(Response, Tests) -> [input_test(Response, Test) || Test <- Tests]. @@ -565,7 +565,7 @@ output_expect(Response) -> end. %% output_test converts an output_test specifier into an eunit test generator --type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string(), term()}}. +-type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), binary(), term()}}. -spec output_test(fun(), output_test_spec()) -> tuple(). output_test(Fun, {Line, {Description, Response, Result}}) -> {Description, diff --git a/test/erlcloud_kms_tests.erl b/test/erlcloud_kms_tests.erl index 76fce69f7..f085f5413 100644 --- a/test/erlcloud_kms_tests.erl +++ b/test/erlcloud_kms_tests.erl @@ -101,7 +101,7 @@ stop(_) -> %%% Input test helpers %%%=================================================================== --type expected_body() :: string(). +-type expected_body() :: binary(). sort_json([{_, _} | _] = Json) -> %% Value is an object @@ -128,7 +128,7 @@ validate_body(Body, Expected) -> %% returns the mock of the erlcloud_httpc function input tests expect to be called. %% Validates the request body and responds with the provided response. --spec input_expect(string(), expected_body()) -> fun(). +-spec input_expect(binary(), expected_body()) -> fun(). input_expect(Response, Expected) -> fun(_Url, post, _Headers, Body, _Timeout, _Config) -> validate_body(Body, Expected), @@ -138,7 +138,7 @@ input_expect(Response, Expected) -> %% input_test converts an input_test specifier into an eunit test generator -type input_test_spec() :: {pos_integer(), {fun(), expected_body()} | {string(), fun(), expected_body()}}. --spec input_test(string(), input_test_spec()) -> tuple(). +-spec input_test(binary(), input_test_spec()) -> tuple(). input_test(Response, {Line, {Description, Fun, Expected}}) when is_list(Description) -> {Description, @@ -151,7 +151,7 @@ input_test(Response, {Line, {Description, Fun, Expected}}) %% input_tests converts a list of input_test specifiers into an eunit test generator --spec input_tests(string(), [input_test_spec()]) -> [tuple()]. +-spec input_tests(binary(), [input_test_spec()]) -> [tuple()]. input_tests(Response, Tests) -> [input_test(Response, Test) || Test <- Tests]. @@ -167,7 +167,7 @@ output_expect(Response) -> end. %% output_test converts an output_test specifier into an eunit test generator --type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string(), term()}}. +-type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), binary(), term()}}. -spec output_test(fun(), output_test_spec()) -> tuple(). output_test(Fun, {Line, {Description, Response, Result}}) -> {Description, diff --git a/test/erlcloud_s3_tests.erl b/test/erlcloud_s3_tests.erl index 95b624859..5ef1a9490 100755 --- a/test/erlcloud_s3_tests.erl +++ b/test/erlcloud_s3_tests.erl @@ -73,8 +73,8 @@ httpc_expect(Method, Response) -> hackney -> Method = Method2, Insecure = false, - Proxy = "10.10.10.10", - Proxy_auth = {"AAAA", "BBBB"}; + Proxy = <<"10.10.10.10">>, + Proxy_auth = {<<"AAAA">>, <<"BBBB">>}; _else -> Method = Method2, @@ -815,8 +815,8 @@ delete_bucket_encryption_test(_) -> hackney_proxy_put_validation_test(_) -> Response = {ok, {{200, "OK"}, [{"x-amz-version-id", "version_id"}], <<>>}}, Config2 = #aws_config{hackney_client_options = #hackney_client_options{insecure = false, - proxy = "10.10.10.10", - proxy_auth = {"AAAA", "BBBB"}}, + proxy = <<"10.10.10.10">>, + proxy_auth = {<<"AAAA">>, <<"BBBB">>}}, http_client = hackney}, meck:expect(erlcloud_httpc, request, httpc_expect(put, Response)), Result = erlcloud_s3:put_object("BucketName", "Key", "Data", config(Config2)), diff --git a/test/erlcloud_sdb_tests.erl b/test/erlcloud_sdb_tests.erl index feb24d162..c8382a67c 100644 --- a/test/erlcloud_sdb_tests.erl +++ b/test/erlcloud_sdb_tests.erl @@ -1,10 +1,18 @@ -module(erlcloud_sdb_tests). --ifdef(TEST). --compile(export_all). - -include_lib("eunit/include/eunit.hrl"). +-export([setup/0]). +-export([cleanup/1]). +-export([select_single_response/0]). +-export([select_next_token/0]). +-export([select_all_single_response/0]). +-export([select_all_failure/0]). +-export([select_all_503/0]). +-export([select_all_next_token/0]). +-export([select_all_next_and_failure/0]). +-export([select_all_two_results/0]). + setup() -> erlcloud_sdb:configure("fake", "fake-secret"), meck:new(erlcloud_httpc). @@ -153,5 +161,3 @@ extract_token_test() -> ?assertEqual(next_token(), erlcloud_sdb:extract_token(parse_document(only_token_response_body()))), ?assertEqual(next_token(), erlcloud_sdb:extract_token(parse_document(single_result_and_token_response_body()))), ?assertEqual(done, erlcloud_sdb:extract_token(parse_document(single_result_response_body("item0")))). - --endif. diff --git a/test/erlcloud_sns_tests.erl b/test/erlcloud_sns_tests.erl index 3fffbd1f0..f25d4d019 100644 --- a/test/erlcloud_sns_tests.erl +++ b/test/erlcloud_sns_tests.erl @@ -285,7 +285,7 @@ set_topic_attributes_input_tests(_) -> Tests = [?_sns_test( {"Test sets topic's attribute.", - ?_f(erlcloud_sns:set_topic_attributes("DisplayName", "MyTopicName", "arn:aws:sns:us-west-2:123456789012:MyTopic")), + ?_f(erlcloud_sns:set_topic_attributes('DisplayName', "MyTopicName", "arn:aws:sns:us-west-2:123456789012:MyTopic")), [ {"Action", "SetTopicAttributes"}, {"AttributeName", "DisplayName"}, @@ -313,7 +313,7 @@ set_topic_attributes_output_tests(_) -> ", ok}) ], - output_tests(?_f(erlcloud_sns:set_topic_attributes("DisplayName", "MyTopicName", "arn:aws:sns:us-west-2:123456789012:MyTopic")), Tests). + output_tests(?_f(erlcloud_sns:set_topic_attributes('DisplayName', "MyTopicName", "arn:aws:sns:us-west-2:123456789012:MyTopic")), Tests). %% Set subscription attributes test based on the API examples: @@ -322,7 +322,7 @@ set_subscription_attributes_input_tests(_) -> Tests = [?_sns_test( {"Test sets subscriptions's attribute.", - ?_f(erlcloud_sns:set_subscription_attributes("FilterPolicy", "{\"a\": [\"b\"]}", "arn:aws:sns:us-east-1:123456789012:My-Topic:80289ba6-0fd4-4079-afb4-ce8c8260f0ca")), + ?_f(erlcloud_sns:set_subscription_attributes('FilterPolicy', "{\"a\": [\"b\"]}", "arn:aws:sns:us-east-1:123456789012:My-Topic:80289ba6-0fd4-4079-afb4-ce8c8260f0ca")), [ {"Action", "SetSubscriptionAttributes"}, {"AttributeName", "FilterPolicy"}, @@ -350,7 +350,7 @@ set_subscription_attributes_output_tests(_) -> ", ok}) ], - output_tests(?_f(erlcloud_sns:set_subscription_attributes("FilterPolicy", "{\"a\": [\"b\"]}", "arn:aws:sns:us-east-1:123456789012:My-Topic:80289ba6-0fd4-4079-afb4-ce8c8260f0ca")), Tests). + output_tests(?_f(erlcloud_sns:set_subscription_attributes('FilterPolicy', "{\"a\": [\"b\"]}", "arn:aws:sns:us-east-1:123456789012:My-Topic:80289ba6-0fd4-4079-afb4-ce8c8260f0ca")), Tests). %% List topics test based on the API example: @@ -762,6 +762,7 @@ doesnt_support_gopher(_) -> ?_assertError({sns_error, {unsupported_scheme,"gopher://"}}, erlcloud_sns:publish_to_topic("topicarn", "message", "subject", Config)). +-dialyzer({nowarn_function, doesnt_accept_non_strings/1}). doesnt_accept_non_strings(_) -> Config = (erlcloud_aws:default_config())#aws_config{sns_scheme=https}, ?_assertError({sns_error, badarg}, diff --git a/test/erlcloud_waf_tests.erl b/test/erlcloud_waf_tests.erl index 5e81ecebf..daf6455fb 100644 --- a/test/erlcloud_waf_tests.erl +++ b/test/erlcloud_waf_tests.erl @@ -41,7 +41,7 @@ byte_match_tuple = #waf_byte_match_tuple{ field_to_match = #waf_field_to_match{type = query_string}, positional_constraint = contains, - target_string = "foobar", + target_string = <<"foobar">>, text_transformation = none}}). -define(UPDATE_IP_SET, @@ -593,7 +593,7 @@ update_xss_match_set_tests(_) -> %%% Input test helpers %%%=================================================================== --type expected_body() :: string(). +-type expected_body() :: binary(). sort_json([{_, _} | _] = Json) -> %% Value is an object @@ -620,7 +620,7 @@ validate_body(Body, Expected) -> %% returns the mock of the erlcloud_httpc function input tests expect to be called. %% Validates the request body and responds with the provided response. --spec input_expect(string(), expected_body()) -> fun(). +-spec input_expect(binary(), expected_body()) -> fun(). input_expect(Response, Expected) -> fun(_Url, post, _Headers, Body, _Timeout, _Config) -> validate_body(Body, Expected), @@ -630,7 +630,7 @@ input_expect(Response, Expected) -> %% input_test converts an input_test specifier into an eunit test generator -type input_test_spec() :: {pos_integer(), {fun(), expected_body()} | {string(), fun(), expected_body()}}. --spec input_test(string(), input_test_spec()) -> tuple(). +-spec input_test(binary(), input_test_spec()) -> tuple(). input_test(Response, {Line, {Description, Fun, Expected}}) when is_list(Description) -> {Description, @@ -643,7 +643,7 @@ input_test(Response, {Line, {Description, Fun, Expected}}) %% input_tests converts a list of input_test specifiers into an eunit test generator --spec input_tests(string(), [input_test_spec()]) -> [tuple()]. +-spec input_tests(binary(), [input_test_spec()]) -> [tuple()]. input_tests(Response, Tests) -> [input_test(Response, Test) || Test <- Tests]. @@ -659,7 +659,7 @@ output_expect(Response) -> end. %% output_test converts an output_test specifier into an eunit test generator --type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string(), term()}}. +-type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), binary(), term()}}. -spec output_test(fun(), output_test_spec()) -> tuple(). output_test(Fun, {Line, {Description, Response, Result}}) -> {Description, From 4a0e23546721b8d55b923ad847ad4aa70c394070 Mon Sep 17 00:00:00 2001 From: Gennady Proskurin Date: Thu, 23 Jul 2020 23:00:47 +0100 Subject: [PATCH 142/310] Switch SES API to Signature v4 --- src/erlcloud_ses.erl | 34 +++------------------------------- 1 file changed, 3 insertions(+), 31 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index c59de0fa7..6e02119f0 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -1242,38 +1242,10 @@ decode_error(Doc) -> %%%------------------------------------------------------------------------------ ses_request(Config, Action, Params) -> - case erlcloud_aws:update_config(Config) of - {ok, Config1} -> - ses_request_no_update(Config1, Action, Params); - {error, Reason} -> - {error, Reason} - end. - -ses_request_no_update(Config, Action, Params) -> - Date = httpd_util:rfc1123_date(), - Signature = base64:encode_to_string( - erlcloud_util:sha256_mac(Config#aws_config.secret_access_key, Date)), - Auth = lists:flatten( - ["AWS3-HTTPS AWSAccessKeyId=", - Config#aws_config.access_key_id, - ",Algorithm=HmacSHA256,Signature=", - Signature]), - - Headers = [{"Date", Date}, - {"X-Amzn-Authorization", Auth}], - Headers2 = case Config#aws_config.security_token of - undefined -> - Headers; - Token -> - [{"x-amz-security-token", Token} | Headers] - end, - QParams = [{"Action", Action}, - {"Version", ?API_VERSION} | + QParams = [{"Action", Action}, + {"Version", ?API_VERSION} | Params], - Query = erlcloud_http:make_query_string(QParams), - - case erlcloud_aws:aws_request_form( - post, "https", Config#aws_config.ses_host, 443, "/", Query, Headers2, Config) of + case erlcloud_aws:aws_request4(post, "https", Config#aws_config.ses_host, 443, "/", QParams, "ses", Config) of {ok, Body} -> {ok, element(1, xmerl_scan:string(binary_to_list(Body)))}; {error, {http_error, Code, _, ErrBody}} when Code >= 400; Code =< 599 -> From b30e563a8d8a4985449bf759f92938c8f685d9a1 Mon Sep 17 00:00:00 2001 From: Josh Kennedy <32585114+kennedyjosh@users.noreply.github.com> Date: Wed, 19 Aug 2020 19:51:14 -0400 Subject: [PATCH 143/310] Add some support for AWS Secrets Manager (#654) * Implemented GetSecretValue AWS API Implemented GetSecretValue API and wrote input and output tests for it. Co-authored-by: Josh Kennedy --- include/erlcloud_aws.hrl | 3 + src/erlcloud_aws.erl | 3 + src/erlcloud_sm.erl | 134 ++++++++++++++++++++++++++ test/erlcloud_sm_tests.erl | 187 +++++++++++++++++++++++++++++++++++++ 4 files changed, 327 insertions(+) create mode 100644 src/erlcloud_sm.erl create mode 100644 test/erlcloud_sm_tests.erl diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 050b12988..920d81d80 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -118,6 +118,9 @@ mms_scheme="https://"::string(), mms_host="metering.marketplace.us-east-1.amazonaws.com"::string(), mms_port=443::non_neg_integer(), + sm_scheme="https://"::string(), + sm_host="secretsmanager.us-east-1.amazonaws.com"::string(), + sm_port=443::non_neg_integer(), guardduty_scheme="https://"::string(), guardduty_host="guardduty.us-east-1.amazonaws.com"::string(), guardduty_port=443::non_neg_integer(), diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 9577977f9..f5bcf8ba3 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -745,6 +745,9 @@ service_config( <<"sdb">> = Service, Region, Config ) -> service_config( <<"ses">>, Region, Config ) -> Host = service_host( <<"email">>, Region ), Config#aws_config{ ses_host = Host }; +service_config( <<"sm">> = Service, Region, Config) -> + Host = service_host( Service, Region ), + Config#aws_config{ sm_host = Host }; service_config( <<"sns">> = Service, Region, Config ) -> Host = service_host( Service, Region ), Config#aws_config{ sns_host = Host }; diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl new file mode 100644 index 000000000..af25e37b0 --- /dev/null +++ b/src/erlcloud_sm.erl @@ -0,0 +1,134 @@ +-module(erlcloud_sm). +-author("joshua@halloapp.com"). + +-include("erlcloud.hrl"). +-include("erlcloud_aws.hrl"). + +%%% Library initialization. +-export([new/2, new/3, new/4]). + +%%% API +-export([ + get_secret_value/2, get_secret_value/3 +]). + +%%%------------------------------------------------------------------------------ +%%% Shared types +%%%------------------------------------------------------------------------------ + +-type sm_response() :: {ok, proplists:proplist()} | {error, term()}. + +-type get_secret_value_option() :: {version_id | version_stage, binary()}. +-type get_secret_value_options() :: [get_secret_value_option()]. + +%%%------------------------------------------------------------------------------ +%%% Library initialization. +%%%------------------------------------------------------------------------------ + +-spec new(string(), string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey) -> + #aws_config{ + access_key_id = AccessKeyID, + secret_access_key = SecretAccessKey + }. + + +-spec new(string(), string(), string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey, Host) -> + #aws_config{ + access_key_id = AccessKeyID, + secret_access_key = SecretAccessKey, + sm_host = Host + }. + + +-spec new(string(), string(), string(), non_neg_integer()) -> aws_config(). +new(AccessKeyID, SecretAccessKey, Host, Port) -> + #aws_config{ + access_key_id = AccessKeyID, + secret_access_key = SecretAccessKey, + sm_host = Host, + sm_port = Port + }. + +%%------------------------------------------------------------------------------ +%% GetSecretValue +%%------------------------------------------------------------------------------ +%% @doc +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetSecretValue.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec get_secret_value(SecretId :: binary(), Opts :: get_secret_value_options()) -> sm_response(). +get_secret_value(SecretId, Opts) -> + get_secret_value(SecretId, Opts, erlcloud_aws:default_config()). + + +-spec get_secret_value(SecretId :: binary(), Opts :: get_secret_value_options(), + Config :: aws_config()) -> sm_response(). +get_secret_value(SecretId, Opts, Config) -> + Json = lists:map( + fun + ({version_id, Val}) -> {<<"VersionId">>, Val}; + ({version_stage, Val}) -> {<<"VersionStage">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId} | Opts]), + sm_request(Config, "secretsmanager.GetSecretValue", Json). + +%%%------------------------------------------------------------------------------ +%%% Internal Functions +%%%------------------------------------------------------------------------------ + +sm_request(Config, Operation, Body) -> + case erlcloud_aws:update_config(Config) of + {ok, Config1} -> + sm_request_no_update(Config1, Operation, Body); + {error, Reason} -> + {error, Reason} + end. + + +sm_request_no_update(Config, Operation, Body) -> + Payload = jsx:encode(Body), + Headers = headers(Config, Operation, Payload), + Request = #aws_request{service = sm, + uri = uri(Config), + method = post, + request_headers = Headers, + request_body = Payload}, + case erlcloud_aws:request_to_return(erlcloud_retry:request(Config, Request, fun sm_result_fun/1)) of + {ok, {_RespHeaders, <<>>}} -> {ok, []}; + {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody)}; + {error, _} = Error -> Error + end. + + +headers(Config, Operation, Body) -> + Headers = [{"host", Config#aws_config.sm_host}, + {"x-amz-target", Operation}, + {"content-type", "application/x-amz-json-1.1"}], + Region = erlcloud_aws:aws_region_from_host(Config#aws_config.sm_host), + erlcloud_aws:sign_v4_headers(Config, Headers, Body, Region, "secretsmanager"). + + +uri(#aws_config{sm_scheme = Scheme, sm_host = Host} = Config) -> + lists:flatten([Scheme, Host, port_spec(Config)]). + + +port_spec(#aws_config{sm_port = 443}) -> + ""; +port_spec(#aws_config{sm_port = Port}) -> + [":", erlang:integer_to_list(Port)]. + + +-spec sm_result_fun(Request :: aws_request()) -> aws_request(). +sm_result_fun(#aws_request{response_type = ok} = Request) -> + Request; +sm_result_fun(#aws_request{response_type = error, + error_type = aws, response_status = Status} = Request) when Status >= 500 -> + Request#aws_request{should_retry = true}; +sm_result_fun(#aws_request{response_type = error, error_type = aws} = Request) -> + Request#aws_request{should_retry = false}. + diff --git a/test/erlcloud_sm_tests.erl b/test/erlcloud_sm_tests.erl new file mode 100644 index 000000000..2d37fd2c4 --- /dev/null +++ b/test/erlcloud_sm_tests.erl @@ -0,0 +1,187 @@ +-module(erlcloud_sm_tests). +-include_lib("eunit/include/eunit.hrl"). +-include("erlcloud.hrl"). + +%% Unit tests for sm. +%% These tests work by using meck to mock erlcloud_httpc. There are two classes of test: input and output. +%% +%% Input tests verify that different function args produce the desired JSON request. +%% An input test list provides a list of funs and the JSON that is expected to result. +%% +%% Output tests verify that the http response produces the correct return from the fun. +%% An output test lists provides a list of response bodies and the expected return. + +%% The _sm_test macro provides line number annotation to a test, similar to _test, but doesn't wrap in a fun +-define(_sm_test(T), {?LINE, T}). +%% The _f macro is a terse way to wrap code in a fun. Similar to _test but doesn't annotate with a line number +-define(_f(F), fun() -> F end). + +-export([validate_body/2]). + +%%%=================================================================== +%%% Common Test Values +%%%=================================================================== + +-define(SECRET_ID, <<"MyTestDatabaseSecret">>). +-define(VERSION_ID, <<"EXAMPLE1-90ab-cdef-fedc-ba987SECRET1">>). +-define(VERSION_STAGE, <<"AWSPREVIOUS">>). + +%%%=================================================================== +%%% Test entry points +%%%=================================================================== + +operation_test_() -> + {foreach, + fun start/0, + fun stop/1, + [ + fun get_secret_value_input_tests/1, + fun get_secret_value_output_tests/1 + ]}. + +start() -> + meck:new(erlcloud_httpc), + ok. + +stop(_) -> + meck:unload(erlcloud_httpc). + +%%%=================================================================== +%%% Input test helpers +%%%=================================================================== + +sort_json([{_, _} | _] = Json) -> + %% Value is an object + SortedChildren = [{K, sort_json(V)} || {K, V} <- Json], + lists:keysort(1, SortedChildren); +sort_json([_ | _] = Json) -> + %% Value is an array + [sort_json(I) || I <- Json]; +sort_json(V) -> + V. + +%% verifies that the parameters in the body match the expected parameters +-spec validate_body(binary(), binary()) -> ok. +validate_body(Body, Expected) -> + Want = sort_json(jsx:decode(Expected)), + Actual = sort_json(jsx:decode(Body)), + case Want =:= Actual of + true -> + ok; + false -> + ?debugFmt("~nEXPECTED~n~p~nACTUAL~n~p~n", [Want, Actual]) + end, + ?assertEqual(Want, Actual). + +%% returns the mock of the erlcloud_httpc function input tests expect to be called. +%% Validates the request body and responds with the provided response. +-spec input_expect(binary(), binary()) -> fun(). +input_expect(Response, Expected) -> + fun(_Url, post, _Headers, Body, _Timeout, _Config) -> + validate_body(Body, Expected), + {ok, {{200, "OK"}, [], Response}} + end. + +%% input_test converts an input_test specifier into an eunit test generator +-type input_test_spec() :: {pos_integer(), {fun(), binary()} | {string(), fun(), binary()}}. +-spec input_test(binary(), input_test_spec()) -> tuple(). +input_test(Response, {Line, {Description, Fun, Expected}}) when + is_list(Description) -> + {Description, + {Line, + fun() -> + meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), + erlcloud_config:configure(string:copies("A", 20), + string:copies("a", 40), fun erlcloud_sm:new/2), + Fun() + end}}. + +%% input_tests converts a list of input_test specifiers into an eunit test generator +-spec input_tests(binary(), [input_test_spec()]) -> [tuple()]. +input_tests(Response, Tests) -> + [input_test(Response, Test) || Test <- Tests]. + +%%%=================================================================== +%%% Output test helpers +%%%=================================================================== + +%% returns the mock of the erlcloud_httpc function output tests expect to be called. +-spec output_expect(string()) -> fun(). +output_expect(Response) -> + fun(_Url, post, _Headers, _Body, _Timeout, _Config) -> + {ok, {{200, "OK"}, [], Response}} + end. + +%% output_test converts an output_test specifier into an eunit test generator +-type output_test_spec() :: {pos_integer(), {binary(), term()} | {string(), binary(), term()}}. +-spec output_test(fun(), output_test_spec()) -> tuple(). +output_test(Fun, {Line, {Description, Response, Result}}) -> + {Description, + {Line, + fun() -> + meck:expect(erlcloud_httpc, request, output_expect(Response)), + erlcloud_config:configure(string:copies("A", 20), + string:copies("a", 40), fun erlcloud_sm:new/2), + Actual = Fun(), + case Result =:= Actual of + true -> ok; + false -> + ?debugFmt("~nEXPECTED~n~p~nACTUAL~n~p~n", [Result, Actual]) + end, + ?assertEqual(Result, Actual) + end}}. + +%% output_tests converts a list of output_test specifiers into an eunit test generator +-spec output_tests(fun(), [output_test_spec()]) -> [term()]. +output_tests(Fun, Tests) -> + [output_test(Fun, Test) || Test <- Tests]. + +%%%=================================================================== +%%% Tests +%%%=================================================================== + +get_secret_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"get_secret_value2id input test", + ?_f(erlcloud_sm:get_secret_value(?SECRET_ID, [{version_id, ?VERSION_ID}])), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"VersionId">>, ?VERSION_ID} + ]) + }), + ?_sm_test( + {"get_secret_value2stage input test", + ?_f(erlcloud_sm:get_secret_value(?SECRET_ID, [{version_stage, ?VERSION_STAGE}])), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"VersionStage">>, ?VERSION_STAGE} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + +-define(GET_SECRET_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"CreatedDate">>, 1.523477145713E9}, + {<<"Name">>, ?SECRET_ID}, + {<<"SecretString">>, + <<"{\n \"username\":\"david\",\n \"password\":\"BnQw&XDWgaEeT9XGTT29\"\n}\n">>}, + {<<"VersionId">>, ?VERSION_ID}, + {<<"VersionStages">>, [?VERSION_STAGE]} +]). + + +get_secret_value_output_tests(_) -> + Tests = [?_sm_test( + {"get_secret_value output test", + jsx:encode(?GET_SECRET_VALUE_RESP), + {ok, ?GET_SECRET_VALUE_RESP}} + )], + + output_tests(?_f(erlcloud_sm:get_secret_value(?SECRET_ID, [{version_id, ?VERSION_ID}])), Tests), + output_tests(?_f(erlcloud_sm:get_secret_value(?SECRET_ID, [{version_stage, ?VERSION_STAGE}])), Tests). + From be597d93a7a6a4c281ed3ce65f4ab26de3fa1ab3 Mon Sep 17 00:00:00 2001 From: carlisom Date: Fri, 11 Sep 2020 13:25:36 -0500 Subject: [PATCH 144/310] Add support for sending raw emails via SES (#657) * Add support for sending raw emails via SES --- src/erlcloud_ses.erl | 84 ++++++++++++++++++++++++++++++++++++- test/erlcloud_ses_tests.erl | 39 +++++++++++++++++ 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index 6e02119f0..0aacc7d19 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -12,7 +12,6 @@ %% * ListIdentityPolicies %% * ListVerifiedEmailAddresses (deprecated; use ListIdentities) %% * PutIdentityPolicy -%% * SendRawEmail %% * VerifyEmailAddress (deprecated; use VerifyEmailIdentity) %% %% @end @@ -44,6 +43,8 @@ -export([send_email/4, send_email/5, send_email/6]). +-export([send_raw_email/1, send_raw_email/2, send_raw_email/3]). + -export([set_identity_dkim_enabled/2, set_identity_dkim_enabled/3]). -export([set_identity_feedback_forwarding_enabled/2, set_identity_feedback_forwarding_enabled/3]). -export([set_identity_notification_topic/3, set_identity_notification_topic/4]). @@ -732,6 +733,66 @@ send_email(Destination, Body, Subject, Source, Opts, Config) -> {error, Reason} -> {error, Reason} end. +%%%------------------------------------------------------------------------------ +%%% SendRawEmail +%%%------------------------------------------------------------------------------ + +-type send_raw_email_message() :: binary() | string(). + +-type send_raw_email_opt() :: {source, email()} | + {destinations, emails()}. + +-type send_raw_email_opts() :: [send_raw_email_opt()]. + +-type send_raw_email_result() :: {ok, [{message_id, string()}]} | {error, term()}. + + +send_raw_email(RawMessage) -> + send_raw_email(RawMessage, [], default_config()). + +send_raw_email(RawMessage, #aws_config{} = Config) -> + send_raw_email(RawMessage, [], Config); +send_raw_email(RawMessage, Opts) -> + send_raw_email(RawMessage, Opts, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% SES API: +%% [https://docs.aws.amazon.com/ses/latest/APIReference/API_SendRawEmail.html] +%% +%% ===Example=== +%% +%% Send a raw email. +%% +%% ` +%% {ok, [{message_id, "00000131d51d2292-159ad6eb-077c-46e6-ad09-ae7c05925ed4-000000"}]} = +%% erlcloud_ses:send_raw_email(<<"From: b@from.com\nTo: a@to.com\nSubject: Subject\nMIME-Version: 1.0\nContent-type: Multipart/Mixed; boundary=\"NextPart\"\n\n--NextPart\nContent-Type: text/plain\n\nEmail Body\n\n--NextPart--">>, []). +%% ' +%% +%% All supported inputs. +%% +%% ` +%% {ok, [{message_id, "00000131d51d2292-159ad6eb-077c-46e6-ad09-ae7c05925ed4-000000"}]} = +%% erlcloud_ses:send_email(<<"To: d@to.com\nCC: c@cc.com\nBCC: a@bcc.com, b@bcc.com\nSubject: Subject\nMIME-Version: 1.0\nContent-type: Multipart/Mixed; boundary=\"NextPart\"\n\n--NextPart\nContent-Type: text/plain\n\nEmail Body\n\n--NextPart--">>, +%% [{destinations, [<<"a@bcc.com">>, "b@bcc.com", <<"c@cc.com">>, "d@to.com"]}, +%% {source, "e@from.com"}]. +%% ' +%% @end +%%------------------------------------------------------------------------------ + +-spec send_raw_email(send_raw_email_message(), + send_raw_email_opts(), + aws_config()) -> + send_raw_email_result(). +send_raw_email(RawMessage, Opts, Config) -> + Params = encode_params([{raw_message, RawMessage}, + {send_raw_email_opts, Opts}]), + case ses_request(Config, "SendRawEmail", Params) of + {ok, Doc} -> + {ok, erlcloud_xml:decode([{message_id, "SendRawEmailResult/MessageId", text}], Doc)}; + {error, Reason} -> {error, Reason} + end. + %%%------------------------------------------------------------------------------ %%% SetIdentityDkimEnabled @@ -1035,8 +1096,12 @@ encode_params([{next_token, NextToken} | T], Acc) when is_list(NextToken); is_bi encode_params(T, [{"NextToken", NextToken} | Acc]); encode_params([{notification_type, NotificationType} | T], Acc) -> encode_params(T, encode_notification_type(NotificationType, Acc)); +encode_params([{raw_message, RawMessage} | T], Acc) -> + encode_params(T, [{"RawMessage.Data", base64:encode(RawMessage)} | Acc]); encode_params([{send_email_opts, Opts} | T], Acc) -> encode_params(T, encode_opts(Opts, Acc)); +encode_params([{send_raw_email_opts, Opts} | T], Acc) -> + encode_params(T, encode_raw_opts(Opts, Acc)); encode_params([{sns_topic, SnsTopic} | T], Acc) when is_list(SnsTopic); is_binary(SnsTopic) -> encode_params(T, [{"SnsTopic", SnsTopic} | Acc]); encode_params([{source, Source} | T], Acc) when is_list(Source); is_binary(Source) -> @@ -1093,6 +1158,12 @@ encode_destination(ToAddress, Acc) when is_list(ToAddress); is_binary(ToAddress) %% Single entry encode_destination_pairs([{to_addresses, [ToAddress]}], Acc). +encode_destinations([Dest | _T] = Destinations, Acc) when is_list(Dest); is_binary(Dest) -> + encode_list("Destinations", Destinations, Acc); +encode_destinations(ToAddress, Acc) when is_list(ToAddress); is_binary(ToAddress) -> + %% Single entry + encode_destinations([ToAddress], Acc). + encode_content_pairs(_, [], Acc) -> Acc; encode_content_pairs(Prefix, [{charset, Charset} | T], Acc) -> @@ -1128,6 +1199,12 @@ encode_opts([{reply_to_addresses, List} | T], Acc) -> encode_opts([{return_path, ReturnPath} | T], Acc) -> encode_opts(T, [{"ReturnPath", ReturnPath} | Acc]). +encode_raw_opts([], Acc) -> + Acc; +encode_raw_opts([{source, Source} | T], Acc) when is_list(Source); is_binary(Source) -> + encode_raw_opts(T, [{"Source", Source} | Acc]); +encode_raw_opts([{destinations, Destination} | T], Acc) -> + encode_raw_opts(T, encode_destinations(Destination, Acc)). encode_identity_type(email_address, Acc) -> [{"IdentityType", "EmailAddress"} | Acc]; @@ -1229,7 +1306,10 @@ decode_error_code("CustomVerificationEmailTemplateDoesNotExist") -> custom_verif decode_error_code("ProductionAccessNotGranted") -> production_access_not_granted; decode_error_code("CustomVerificationEmailInvalidContent") -> custom_verification_email_invalid_content; decode_error_code("FromEmailAddressNotVerified") -> from_email_address_not_verified; -decode_error_code("CustomVerificationEmailTemplateAlreadyExists") -> custom_verification_email_template_already_exists. +decode_error_code("CustomVerificationEmailTemplateAlreadyExists") -> custom_verification_email_template_already_exists; +decode_error_code("AccountSendingPaused") -> account_sending_paused; +decode_error_code("ConfigurationSetSendingPaused") -> configuration_set_sending_paused; +decode_error_code("MailFromDomainNotVerified") -> mail_from_domain_not_verified. decode_error(Doc) -> diff --git a/test/erlcloud_ses_tests.erl b/test/erlcloud_ses_tests.erl index b46516a7d..9c5a97a7c 100644 --- a/test/erlcloud_ses_tests.erl +++ b/test/erlcloud_ses_tests.erl @@ -22,6 +22,7 @@ operation_test_() -> fun get_send_statistics_tests/1, fun list_identities_tests/1, fun send_email_tests/1, + fun send_raw_email_tests/1, fun set_identity_dkim_enabled_tests/1, fun set_identity_feedback_forwarding_enabled_tests/1, fun set_identity_notification_topic_tests/1, @@ -349,6 +350,44 @@ send_email_tests(_) -> end ]. +send_raw_email_tests(_) -> + [ + fun() -> + configure(), + Expected = "Action=SendRawEmail&Version=2010-12-01&RawMessage.Data=RnJvbTogYkBmcm9tLmNvbQpUbzogYUB0by5jb20KU3ViamVjdDogU3ViamVjdApNSU1FLVZlcnNpb246IDEuMApDb250ZW50LXR5cGU6IE11bHRpcGFydC9NaXhlZDsgYm91bmRhcnk9Ik5leHRQYXJ0IgoKLS1OZXh0UGFydApDb250ZW50LVR5cGU6IHRleHQvcGxhaW4KCkVtYWlsIEJvZHkKCi0tTmV4dFBhcnQtLQ%3D%3D", + Response = +" + + 00000131d51d2292-159ad6eb-077c-46e6-ad09-ae7c05925ed4-000000 + + + d5964849-c866-11e0-9beb-01a62d68c57f + +", + meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), + ?assertEqual({ok, [{message_id, "00000131d51d2292-159ad6eb-077c-46e6-ad09-ae7c05925ed4-000000"}]}, + erlcloud_ses:send_raw_email("From: b@from.com\nTo: a@to.com\nSubject: Subject\nMIME-Version: 1.0\nContent-type: Multipart/Mixed; boundary=\"NextPart\"\n\n--NextPart\nContent-Type: text/plain\n\nEmail Body\n\n--NextPart--", [])) + end, + fun() -> + configure(), + Expected = "Action=SendRawEmail&Version=2010-12-01&RawMessage.Data=VG86IGRAdG8uY29tCkNDOiBjQGNjLmNvbQpCQ0M6IGFAYmNjLmNvbSwgYkBiY2MuY29tClN1YmplY3Q6IFN1YmplY3QKTUlNRS1WZXJzaW9uOiAxLjAKQ29udGVudC10eXBlOiBNdWx0aXBhcnQvTWl4ZWQ7IGJvdW5kYXJ5PSJOZXh0UGFydCIKCi0tTmV4dFBhcnQKQ29udGVudC1UeXBlOiB0ZXh0L3BsYWluCgpFbWFpbCBCb2R5CgotLU5leHRQYXJ0LS0%3D&Source=e%40from.com&Destinations.member.1=d%40to.com&Destinations.member.2=c%40cc.com&Destinations.member.3=a%40bcc.com&Destinations.member.4=b%40bcc.com", + Response = +" + + 00000131d51d2292-159ad6eb-077c-46e6-ad09-ae7c05925ed4-000000 + + + d5964849-c866-11e0-9beb-01a62d68c57f + +", + meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), + ?assertEqual({ok, [{message_id, "00000131d51d2292-159ad6eb-077c-46e6-ad09-ae7c05925ed4-000000"}]}, + erlcloud_ses:send_raw_email(<<"To: d@to.com\nCC: c@cc.com\nBCC: a@bcc.com, b@bcc.com\nSubject: Subject\nMIME-Version: 1.0\nContent-type: Multipart/Mixed; boundary=\"NextPart\"\n\n--NextPart\nContent-Type: text/plain\n\nEmail Body\n\n--NextPart--">>, + [{source, "e@from.com"}, + {destinations, ["d@to.com", <<"c@cc.com">>, <<"a@bcc.com">>, "b@bcc.com"]}])) + end + ]. + set_identity_dkim_enabled_tests(_) -> [fun() -> configure(), From 0b6a5929c7d6e24c4d2f8bc86f0bfead3d26cd05 Mon Sep 17 00:00:00 2001 From: Creighton Medley Date: Thu, 17 Sep 2020 11:56:21 -0400 Subject: [PATCH 145/310] add platform to instance --- src/erlcloud_ec2.erl | 1 + test/erlcloud_ec2_tests.erl | 1 + 2 files changed, 2 insertions(+) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index d8f30fc60..b7d03754d 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -1495,6 +1495,7 @@ extract_instance(Node) -> {product_codes, get_list("productCodes/item/productCode", Node)}, {instance_type, get_text("instanceType", Node)}, {launch_time, erlcloud_xml:get_time("launchTime", Node)}, + {platform, get_text("platform", Node)}, {placement, [{availability_zone, get_text("placement/availabilityZone", Node)}]}, {kernel_id, get_text("kernelId", Node)}, {ramdisk_id, get_text("ramdiskId", Node)}, diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index a4e57a93a..16e58cb1d 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -1273,6 +1273,7 @@ generate_one_instance(N) -> t2.small 2016-09-26T10:35:00.000Z + us-east-1a From f8db3dc85ffeb50f347ba77fdd36fc38b67dc336 Mon Sep 17 00:00:00 2001 From: wellknownmelodies Date: Mon, 21 Sep 2020 12:25:41 +0900 Subject: [PATCH 146/310] add support for ses config sets and tags --- src/erlcloud_ses.erl | 23 ++++++++++++++++++----- test/erlcloud_ses_tests.erl | 8 +++++--- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index 0aacc7d19..cbd7ae222 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -663,8 +663,10 @@ send_custom_verification_email(EmailAddress, TemplateName, Config) -> -type send_email_source() :: email(). --type send_email_opt() :: {reply_to_addresses, emails()} | - {return_path, email()}. +-type send_email_opt() :: {configuration_set_name, string() | binary()} | + {reply_to_addresses, emails()} | + {return_path, email()} | + {tags, [{string() | binary(), string() | binary()}]}. -type send_email_opts() :: [send_email_opt()]. @@ -1134,11 +1136,18 @@ encode_list(Prefix, List, Acc) -> encode_list(_, [], _, Acc) -> Acc; +encode_list(Prefix, [{Name, Value} | T], N, Acc) when is_list(Name) orelse is_binary(Name), + is_list(Value) orelse is_binary(Value) -> + encode_list(Prefix, T, N + 1, + [{encode_param_index(Prefix, N) ++ ".Name", Name}, + {encode_param_index(Prefix, N) ++ ".Value", Value} | Acc]); encode_list(Prefix, [H | T], N, Acc) when is_list(H); is_binary(H) -> - encode_list(Prefix, T, N + 1, [{Prefix ++ ".member." ++ integer_to_list(N), H} | Acc]); + encode_list(Prefix, T, N + 1, [{encode_param_index(Prefix, N), H} | Acc]); encode_list(Prefix, V, N, Acc) when is_list(V); is_binary(V) -> encode_list(Prefix, [V], N, Acc). - + +encode_param_index(Prefix, N) -> Prefix ++ ".member." ++ integer_to_list(N). + encode_destination_pairs([], Acc) -> Acc; encode_destination_pairs([{bcc_addresses, List} | T], Acc) -> @@ -1194,10 +1203,14 @@ encode_body(Body, Acc) when is_list(Body); is_binary(Body) -> encode_opts([], Acc) -> Acc; +encode_opts([{configuration_set_name, ConfigurationSet} | T], Acc) -> + encode_opts(T, [{"ConfigurationSetName", ConfigurationSet} | Acc]); encode_opts([{reply_to_addresses, List} | T], Acc) -> encode_opts(T, encode_list("ReplyToAddresses", List, Acc)); encode_opts([{return_path, ReturnPath} | T], Acc) -> - encode_opts(T, [{"ReturnPath", ReturnPath} | Acc]). + encode_opts(T, [{"ReturnPath", ReturnPath} | Acc]); +encode_opts([{tags, Tags} | T], Acc) -> + encode_opts(T, encode_list("Tags", Tags, Acc)). encode_raw_opts([], Acc) -> Acc; diff --git a/test/erlcloud_ses_tests.erl b/test/erlcloud_ses_tests.erl index 9c5a97a7c..786dae07e 100644 --- a/test/erlcloud_ses_tests.erl +++ b/test/erlcloud_ses_tests.erl @@ -323,7 +323,7 @@ send_email_tests(_) -> end, fun() -> configure(), - Expected = "Action=SendEmail&Version=2010-12-01&Destination.BccAddresses.member.1=a%40bcc.com&Destination.BccAddresses.member.2=b%40bcc.com&Destination.CcAddresses.member.1=c%40cc.com&Destination.ToAddresses.member.1=d%40to.com&Message.Body.Html.Charset=html%20charset&Message.Body.Html.Data=html%20data&Message.Body.Text.Charset=text%20charset&Message.Body.Text.Data=text%20data&Message.Subject.Charset=subject%20charset&Message.Subject.Data=subject%20data&Source=e%40from.com&ReplyToAddresses.member.1=f%40reply.com&ReplyToAddresses.member.2=g%40reply.com&ReturnPath=return%20path", + Expected = "Action=SendEmail&Version=2010-12-01&Destination.BccAddresses.member.1=a%40bcc.com&Destination.BccAddresses.member.2=b%40bcc.com&Destination.CcAddresses.member.1=c%40cc.com&Destination.ToAddresses.member.1=d%40to.com&Message.Body.Html.Charset=html%20charset&Message.Body.Html.Data=html%20data&Message.Body.Text.Charset=text%20charset&Message.Body.Text.Data=text%20data&Message.Subject.Charset=subject%20charset&Message.Subject.Data=subject%20data&Source=e%40from.com&ConfigurationSetName=configuration%20set&ReplyToAddresses.member.1=f%40reply.com&ReplyToAddresses.member.2=g%40reply.com&ReturnPath=return%20path&Tags.member.1.Value=value-1&Tags.member.1.Name=tag-1&Tags.member.2.Value=value-2&Tags.member.2.Name=tag-2", Response = " @@ -345,8 +345,10 @@ send_email_tests(_) -> [{charset, "subject charset"}, {data, "subject data"}], "e@from.com", - [{reply_to_addresses, [<<"f@reply.com">>, "g@reply.com"]}, - {return_path, "return path"}])) + [{configuration_set_name, "configuration set"}, + {reply_to_addresses, [<<"f@reply.com">>, "g@reply.com"]}, + {return_path, "return path"}, + {tags, [{"tag-1", "value-1"}, {"tag-2", "value-2"}]}])) end ]. From c6527625ce9ec5e611bfe1a587f39adda631d5c7 Mon Sep 17 00:00:00 2001 From: kkuzmin Date: Thu, 24 Sep 2020 17:55:15 +0100 Subject: [PATCH 147/310] Use actual config param in describe_load_balancers_all() --- src/erlcloud_elb.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_elb.erl b/src/erlcloud_elb.erl index bdd532e5d..c07ba2f3e 100644 --- a/src/erlcloud_elb.erl +++ b/src/erlcloud_elb.erl @@ -245,7 +245,7 @@ describe_load_balancers_all() -> -spec describe_load_balancers_all(list(string()) | aws_config()) -> {ok, [term()]} | {error, term()}. describe_load_balancers_all(Config) when is_record(Config, aws_config) -> - describe_load_balancers_all([], default_config()); + describe_load_balancers_all([], Config); describe_load_balancers_all(Names) -> describe_load_balancers_all(Names, default_config()). From 72a31612a9fd813424668c3832e0def48d4c3007 Mon Sep 17 00:00:00 2001 From: Vitor Andriotti Date: Fri, 2 Oct 2020 12:07:55 -0300 Subject: [PATCH 148/310] erlcloud_application_autoscaler: return error tuple when #aws_request.response_type is not 'ok' Internal function `erlcloud_application_autoscaler:request_with_action/3` currently ignores any error response from AWS other than config update failures. This change updates the code to check `#aws_request.response_type` and return `{ok, _}`/`{error, _}` appropriately. --- src/erlcloud_application_autoscaler.erl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 57b8bded9..5f649cd6a 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -702,7 +702,12 @@ request_with_action(Configuration, BodyConfiguration, Action) -> Headers = [{"content-type", "application/x-amz-json-1.1"} | HeadersPrev], Request = prepare_record(Config, post, Headers, Body), Response = erlcloud_retry:request(Config, Request, fun aas_result_fun/1), - {ok, jsx:decode(Response#aws_request.response_body, [{return_maps, false}])}; + case Response#aws_request.response_type of + ok -> + {ok, jsx:decode(Response#aws_request.response_body, [{return_maps, false}])}; + _ -> + {error, jsx:decode(Response#aws_request.response_body, [{return_maps, false}])} + end; {error, Reason} -> {error, Reason} end. From 0223f476eb3c441971c51248fc83cac4f12cec32 Mon Sep 17 00:00:00 2001 From: Vitor Andriotti Date: Thu, 8 Oct 2020 15:41:36 -0300 Subject: [PATCH 149/310] Decode auto scaling application errors and return them in the format of '{error, {Type, Message}}' --- src/erlcloud_application_autoscaler.erl | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 5f649cd6a..a8ec9c065 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -706,7 +706,7 @@ request_with_action(Configuration, BodyConfiguration, Action) -> ok -> {ok, jsx:decode(Response#aws_request.response_body, [{return_maps, false}])}; _ -> - {error, jsx:decode(Response#aws_request.response_body, [{return_maps, false}])} + {error, decode_error(Response)} end; {error, Reason} -> {error, Reason} @@ -723,18 +723,26 @@ prepare_record(Config, Method, Headers, Body) -> ]), #aws_request{service = application_autoscaling, - method = Method, - request_headers = Headers, - request_body = Body, - uri = RequestURI}. + method = Method, + request_headers = Headers, + request_body = Body, + uri = RequestURI}. port_spec(#aws_config{application_autoscaling_port=80}) -> ""; port_spec(#aws_config{application_autoscaling_port=Port}) -> [":", erlang:integer_to_list(Port)]. - headers(Config, Operation, Body) -> Headers = [{"host", Config#aws_config.application_autoscaling_host}, {"x-amz-target", Operation}], erlcloud_aws:sign_v4_headers(Config, Headers, Body, erlcloud_aws:aws_region_from_host(Config#aws_config.application_autoscaling_host), "application-autoscaling"). + +%% Extracts and decodes the error from the response returning +%% in the format of `{error, {ErrorType, Message}}' (matching to how errors are returned in `ercloud_ddb2' module) +%% Example: {error, {<<"ConcurrentUpdateException">>, <<"You already have a pending update to an Auto Scaling resource.">>}} +decode_error(Response) -> + DecodedError = jsx:decode(Response#aws_request.response_body, [{return_maps, false}]), + ErrorType = proplists:get_value(<<"__type">>, DecodedError, <<>>), + ErrorMessage = proplists:get_value(<<"Message">>, DecodedError, <<>>), + {error, {ErrorType, ErrorMessage}}. From 9fc6d72066f46349571334d968247a8fc4a7c87f Mon Sep 17 00:00:00 2001 From: Vitor Andriotti Date: Wed, 14 Oct 2020 10:05:56 -0300 Subject: [PATCH 150/310] erlcloud_application_autoscaler: fix error decoder by verifying if the response contains a json body --- src/erlcloud_application_autoscaler.erl | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index a8ec9c065..5ee7cb7e7 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -61,10 +61,13 @@ -type response_attribute() :: string() | integer(). -type response_key() :: atom(). -type response() :: [{response_key(), response_attribute()}]. +-type error_type() :: binary(). +-type error_message() :: binary(). -type ok_error_response() :: {ok, jsx:json_term()} | {error, metadata_not_available | container_credentials_unavailable - | erlcloud_aws:httpc_result_error()}. + | erlcloud_aws:httpc_result_error() + | {error_type(), error_message()}}. -type aws_aas_request_body() :: proplists:proplist(). @@ -706,7 +709,7 @@ request_with_action(Configuration, BodyConfiguration, Action) -> ok -> {ok, jsx:decode(Response#aws_request.response_body, [{return_maps, false}])}; _ -> - {error, decode_error(Response)} + decode_error(Response) end; {error, Reason} -> {error, Reason} @@ -741,8 +744,13 @@ headers(Config, Operation, Body) -> %% Extracts and decodes the error from the response returning %% in the format of `{error, {ErrorType, Message}}' (matching to how errors are returned in `ercloud_ddb2' module) %% Example: {error, {<<"ConcurrentUpdateException">>, <<"You already have a pending update to an Auto Scaling resource.">>}} -decode_error(Response) -> - DecodedError = jsx:decode(Response#aws_request.response_body, [{return_maps, false}]), - ErrorType = proplists:get_value(<<"__type">>, DecodedError, <<>>), - ErrorMessage = proplists:get_value(<<"Message">>, DecodedError, <<>>), - {error, {ErrorType, ErrorMessage}}. +decode_error(#aws_request{response_body = Body} = Response) -> + case jsx:is_json(Body) of + false -> + erlcloud_aws:request_to_return(Response); + true -> + DecodedError = jsx:decode(Body, [{return_maps, false}]), + ErrorType = proplists:get_value(<<"__type">>, DecodedError, <<>>), + ErrorMessage = proplists:get_value(<<"Message">>, DecodedError, <<>>), + {error, {ErrorType, ErrorMessage}} + end. From f07f9c4f6dbea3bb28bd50f01f0950020906a674 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Fri, 16 Oct 2020 13:06:42 +0100 Subject: [PATCH 151/310] Add tags into DescribeImages response --- src/erlcloud_ec2.erl | 3 ++- test/erlcloud_ec2_tests.erl | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index b7d03754d..c5a0373bc 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -1351,7 +1351,8 @@ extract_image(Node) -> {creation_date, erlcloud_xml:get_time("creationDate", Node)}, {platform, get_text("platform", Node)}, {block_device_mapping, [extract_block_device_mapping(Item) || Item <- xmerl_xpath:string("blockDeviceMapping/item", Node)]}, - {product_codes, [extract_product_code(Item) || Item <- xmerl_xpath:string("productCodes/item", Node)]} + {product_codes, [extract_product_code(Item) || Item <- xmerl_xpath:string("productCodes/item", Node)]}, + {tag_set, [extract_tag_item(Item) || Item <- xmerl_xpath:string("tagSet/item", Node, [])]} ]. extract_block_device_mapping(Node) -> diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 16e58cb1d..7ebe2f149 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -783,7 +783,12 @@ describe_images_tests(_) -> 2016-03-26T12:00:13Z hvm - + + + Key + Value + + xen     @@ -805,7 +810,8 @@ describe_images_tests(_) -> {creation_date, {{2016,3,26},{12,0,13}}}, {platform, "windows"}, {block_device_mapping, []}, - {product_codes, []} + {product_codes, []}, + {tag_set, [[{key,"Key"},{value, "Value"}]]} ]]}})], %% Remaining AWS API examples return subsets of the same data From ac86a7caaf1a5dc9933dfef92d51eb77bca69fe7 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Fri, 13 Nov 2020 17:34:18 +0000 Subject: [PATCH 152/310] Add ECS task launch type --- include/erlcloud_ecs.hrl | 1 + src/erlcloud_ecs.erl | 3 +++ test/erlcloud_ecs_tests.erl | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/include/erlcloud_ecs.hrl b/include/erlcloud_ecs.hrl index fc89f854a..cf06489a0 100644 --- a/include/erlcloud_ecs.hrl +++ b/include/erlcloud_ecs.hrl @@ -254,6 +254,7 @@ created_at :: undefined | number(), desired_status:: undefined | binary(), last_status :: undefined | binary(), + launch_type :: undefined | binary(), overrides :: undefined | #ecs_task_override{}, started_at :: undefined | pos_integer(), started_by :: undefined | binary(), diff --git a/src/erlcloud_ecs.erl b/src/erlcloud_ecs.erl index 6e4f99fe5..40ce33650 100644 --- a/src/erlcloud_ecs.erl +++ b/src/erlcloud_ecs.erl @@ -1084,6 +1084,7 @@ task_record() -> {<<"createdAt">>, #ecs_task.created_at, fun id/2}, {<<"desiredStatus">>, #ecs_task.desired_status, fun id/2}, {<<"lastStatus">>, #ecs_task.last_status, fun id/2}, + {<<"launchType">>, #ecs_task.launch_type, fun id/2}, {<<"overrides">>, #ecs_task.overrides, fun decode_task_overrides/2}, {<<"startedAt">>, #ecs_task.started_at, fun id/2}, {<<"startedBy">>, #ecs_task.started_by, fun id/2}, @@ -2050,6 +2051,7 @@ list_task_definitions(Opts, Config) -> {sort, asc | desc} | {max_results, 1..100} | {next_token, binary()} | + {launch_type, string_param()} | out_opt(). -type list_tasks_opts() :: [list_tasks_opt()]. @@ -2065,6 +2067,7 @@ list_tasks_opts() -> {service_name, <<"serviceName">>, fun to_binary/1}, {started_by, <<"startedBy">>, fun to_binary/1}, {max_results, <<"maxResults">>, fun id/1}, + {launch_type, <<"launchType">>, fun to_binary/1}, {next_token, <<"nextToken">>, fun to_binary/1} ]. diff --git a/test/erlcloud_ecs_tests.erl b/test/erlcloud_ecs_tests.erl index f6aed1f30..0f6da4621 100644 --- a/test/erlcloud_ecs_tests.erl +++ b/test/erlcloud_ecs_tests.erl @@ -1706,6 +1706,7 @@ describe_tasks_output_tests(_) -> ], \"desiredStatus\": \"RUNNING\", \"lastStatus\": \"RUNNING\", + \"launchType\": \"FARGATE\", \"overrides\": { \"containerOverrides\": [ { @@ -1753,6 +1754,7 @@ describe_tasks_output_tests(_) -> ], desired_status = <<"RUNNING">>, last_status = <<"RUNNING">>, + launch_type = <<"FARGATE">>, overrides = #ecs_task_override{ container_overrides = [ #ecs_container_override{ @@ -2416,6 +2418,7 @@ run_task_output_tests(_) -> ], \"desiredStatus\": \"RUNNING\", \"lastStatus\": \"PENDING\", + \"launchType\": \"EC2\", \"overrides\": { \"containerOverrides\": [ { @@ -2454,6 +2457,7 @@ run_task_output_tests(_) -> ], desired_status = <<"RUNNING">>, last_status = <<"PENDING">>, + launch_type = <<"EC2">>, overrides = #ecs_task_override{ container_overrides = [ #ecs_container_override{ @@ -2556,6 +2560,7 @@ start_task_output_tests(_) -> ], \"desiredStatus\": \"RUNNING\", \"lastStatus\": \"PENDING\", + \"launchType\": \"FARGATE\", \"overrides\": { \"containerOverrides\": [ { @@ -2594,6 +2599,7 @@ start_task_output_tests(_) -> ], desired_status = <<"RUNNING">>, last_status = <<"PENDING">>, + launch_type = <<"FARGATE">>, overrides = #ecs_task_override{ container_overrides = [ #ecs_container_override{ From 6dacbec2bea45ebad7c19ea831696c127823db6c Mon Sep 17 00:00:00 2001 From: Pavel Puchkin Date: Fri, 13 Nov 2020 20:22:40 +0200 Subject: [PATCH 153/310] Implement CreatePlatformApplication request for SNS --- src/erlcloud_sns.erl | 38 +++++++++++++++++++++++++++++++++- test/erlcloud_sns_tests.erl | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_sns.erl b/src/erlcloud_sns.erl index d073e2830..76debede8 100644 --- a/src/erlcloud_sns.erl +++ b/src/erlcloud_sns.erl @@ -5,6 +5,10 @@ -author('elbrujohalcon@inaka.net'). -export([add_permission/3, add_permission/4, + create_platform_application/2, + create_platform_application/3, + create_platform_application/4, + create_platform_application/5, create_platform_endpoint/2, create_platform_endpoint/3, create_platform_endpoint/4, create_platform_endpoint/5, create_platform_endpoint/6, @@ -161,6 +165,30 @@ create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attribut create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attributes, AccessKeyID, SecretAccessKey) -> create_platform_endpoint(PlatformApplicationArn, Token, CustomUserData, Attributes, new_config(AccessKeyID, SecretAccessKey)). +-spec create_platform_application(string(), string()) -> string(). +create_platform_application(Name, Platform) -> + create_platform_application(Name, Platform, []). + +-spec create_platform_application(string(), string(), [{sns_endpoint_attribute(), string()}]) -> string(). +create_platform_application(Name, Platform, Attributes) -> + create_platform_application(Name, Platform, Attributes, default_config()). + +-spec create_platform_application(string(), string(), [{sns_endpoint_attribute(), string()}], aws_config()) -> string(). +create_platform_application(Name, Platform, Attributes, Config) -> + Doc = + sns_xml_request( + Config, "CreatePlatformApplication", + [{"Name", Name}, + {"Platform", Platform} + | encode_attributes(Attributes) + ]), + erlcloud_xml:get_text( + "CreatePlatformApplicationResult/PlatformApplicationArn", Doc). + +-spec create_platform_application(string(), string(), [{sns_endpoint_attribute(), string()}], string(), string()) -> string(). +create_platform_application(Name, Platform, Attributes, AccessKeyID, SecretAccessKey) -> + create_platform_application(Name, Platform, Attributes, new_config(AccessKeyID, SecretAccessKey)). + -spec create_topic(string()) -> string(). create_topic(TopicName) -> @@ -682,7 +710,15 @@ encode_attributes(Attributes) -> {Prefix ++ ".value", Value} | Acc] end, [], lists:zip(lists:seq(1, length(Attributes)), Attributes)). -encode_attribute_name(custom_user_data) -> "CustomUserData"; +encode_attribute_name(platform_credential) -> "PlatformCredential"; +encode_attribute_name(platform_principal) -> "PlatformPrincipal"; +encode_attribute_name(event_endpoint_created) -> "EventEndpointCreated"; +encode_attribute_name(event_endpoint_deleted) -> "EventEndpointDeleted"; +encode_attribute_name(event_endpoint_updated) -> "EventEndpointUpdated"; +encode_attribute_name(event_delivery_failure) -> "EventDeliveryFailure"; +encode_attribute_name(success_feedback_role_arn) -> "SuccessFeedbackRoleArn"; +encode_attribute_name(failure_feedback_role_arn) -> "FailureFeedbackRoleArn"; +encode_attribute_name(success_feedback_sample_rate) -> "SuccessFeedbackSampleRate"; encode_attribute_name(enabled) -> "Enabled"; encode_attribute_name(token) -> "Token". diff --git a/test/erlcloud_sns_tests.erl b/test/erlcloud_sns_tests.erl index f25d4d019..be5634de5 100644 --- a/test/erlcloud_sns_tests.erl +++ b/test/erlcloud_sns_tests.erl @@ -37,6 +37,8 @@ sns_api_test_() -> fun delete_topic_input_tests/1, fun subscribe_input_tests/1, fun subscribe_output_tests/1, + fun create_platform_application_input_tests/1, + fun create_platform_application_output_tests/1, fun set_topic_attributes_input_tests/1, fun set_topic_attributes_output_tests/1, fun set_subscription_attributes_input_tests/1, @@ -279,6 +281,45 @@ subscribe_output_tests(_) -> "arn:aws:sns:us-west-2:123456789012:MyTopic")), Tests). +create_platform_application_input_tests(_) -> + Tests = + [?_sns_test( + {"Test to create platform application.", + ?_f(erlcloud_sns:create_platform_application("TestApp", "ADM")), + [ + {"Action", "CreatePlatformApplication"}, + {"Name", "TestApp"}, + {"Platform", "ADM"} + ]}) + ], + + Response = " + + + arn:aws:sns:us-west-2:123456789012:app/ADM/TestApp + + + b6f0e78b-e9d4-5a0e-b973-adc04e8a4ff9 + + ", + + input_tests(Response, Tests). + +create_platform_application_output_tests(_) -> + Tests = [?_sns_test( + {"This is a create platform application test.", + " + + arn:aws:sns:us-west-2:123456789012:app/ADM/TestApp + + + b6f0e78b-e9d4-5a0e-b973-adc04e8a4ff9 + + ", + "arn:aws:sns:us-west-2:123456789012:app/ADM/TestApp"}) + ], + output_tests(?_f(erlcloud_sns:create_platform_application("ADM", "TestApp")), Tests). + %% Set topic attributes test based on the API examples: %% http://docs.aws.amazon.com/sns/latest/APIReference/API_SetTopicAttributes.html set_topic_attributes_input_tests(_) -> From 7190d7a8007a0f765295d1d8ddaadacfeffca8c3 Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Fri, 13 Nov 2020 16:15:18 -0600 Subject: [PATCH 154/310] erlcloud_sm: fix use of jsx Follow up to https://github.com/erlcloud/erlcloud/pull/645. I missed that I didn't catch this regression when `erlcloud_sm` was introduced in https://github.com/erlcloud/erlcloud/pull/654. This change just adds the eplicit `[{return_maps, false}]` option to the `jsx:decode(...)` call to ensure that it comes back in the expected format, regardless of which jsx version the client uses. --- src/erlcloud_sm.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl index af25e37b0..3c4f81ae9 100644 --- a/src/erlcloud_sm.erl +++ b/src/erlcloud_sm.erl @@ -100,7 +100,7 @@ sm_request_no_update(Config, Operation, Body) -> request_body = Payload}, case erlcloud_aws:request_to_return(erlcloud_retry:request(Config, Request, fun sm_result_fun/1)) of {ok, {_RespHeaders, <<>>}} -> {ok, []}; - {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody)}; + {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody, [{return_maps, false}])}; {error, _} = Error -> Error end. From a59aa2d65a2029e4c91bb1714a5172badba0ee2a Mon Sep 17 00:00:00 2001 From: Evgeny Bob Date: Tue, 17 Nov 2020 14:18:47 +0000 Subject: [PATCH 155/310] Update erlcloud_aws.erl --- src/erlcloud_aws.erl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index f5bcf8ba3..a3dc484d1 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -822,9 +822,16 @@ get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> %% resolve through ENV if any Endpoints = case ConfiguredEndpoints of {env, EnvVarName} when is_list(EnvVarName) -> - Es = string_split(os:getenv(EnvVarName, ""), ","), % ignore "" env var or ",," cases - [list_to_binary(E) || E <- Es, E /= ""]; + % also handle "zoneID:zoneName" form when it comes from CFN + Es = string_split(string_split(os:getenv(EnvVarName, ""), ","), ":"), + lists:filtermap( + fun ([""]) -> false; + ([Name]) -> {true, list_to_binary(Name)}; + ([_Id, Name]) -> {true, list_to_binary(Name)} + end, + Es + ); EndpointsList when is_list(EndpointsList) -> EndpointsList end, From b183b01776012e83107e64f0b3f676f5fd547d1c Mon Sep 17 00:00:00 2001 From: Evgeny Bob Date: Tue, 17 Nov 2020 14:42:29 +0000 Subject: [PATCH 156/310] Update erlcloud_aws.erl --- src/erlcloud_aws.erl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index a3dc484d1..502bce00b 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -824,11 +824,14 @@ get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> {env, EnvVarName} when is_list(EnvVarName) -> % ignore "" env var or ",," cases % also handle "zoneID:zoneName" form when it comes from CFN - Es = string_split(string_split(os:getenv(EnvVarName, ""), ","), ":"), + Es = string_split(os:getenv(EnvVarName, ""), ","), lists:filtermap( - fun ([""]) -> false; - ([Name]) -> {true, list_to_binary(Name)}; - ([_Id, Name]) -> {true, list_to_binary(Name)} + fun ("") -> false; + (Value) -> + case string_split(Value, ":") of + [_Id, Name] -> {true, list_to_binary(Name)}; + [Name] ->{true, list_to_binary(Name)} + end end, Es ); From 1faba1748363b9b9bdcd85dc2581b226fda32494 Mon Sep 17 00:00:00 2001 From: Evgeny Bob Date: Tue, 17 Nov 2020 14:43:34 +0000 Subject: [PATCH 157/310] Update erlcloud_aws.erl --- src/erlcloud_aws.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 502bce00b..7cbffd594 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -830,7 +830,7 @@ get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> (Value) -> case string_split(Value, ":") of [_Id, Name] -> {true, list_to_binary(Name)}; - [Name] ->{true, list_to_binary(Name)} + [Name] -> {true, list_to_binary(Name)} end end, Es From c727ee494c6bb2a50f2a405470136f8eb0879335 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Tue, 1 Dec 2020 15:45:29 +0000 Subject: [PATCH 158/310] Enrich ecs:DescribeTasks responses. --- include/erlcloud_ecs.hrl | 52 ++++++++++++++++++++++--- src/erlcloud_ecs.erl | 71 +++++++++++++++++++++++++++++++++- test/erlcloud_ecs_tests.erl | 76 ++++++++++++++++++++++++++++++++++++- 3 files changed, 191 insertions(+), 8 deletions(-) diff --git a/include/erlcloud_ecs.hrl b/include/erlcloud_ecs.hrl index cf06489a0..41b8b4c68 100644 --- a/include/erlcloud_ecs.hrl +++ b/include/erlcloud_ecs.hrl @@ -226,14 +226,26 @@ protocol :: undefined | ecs_protocol() }). +-record(ecs_network_interface, { + attachment_id :: undefined | binary(), + private_ipv4_address :: undefined | binary() +}). + -record(ecs_container, { container_arn :: undefined | binary(), - exit_code :: undefined | pos_integer(), + cpu :: undefined | binary(), + exit_code :: undefined | binary(), + health_status :: undefined | binary(), + image :: undefined | binary(), last_status :: undefined | binary(), + memory_reservation :: undefined | binary(), name :: undefined | binary(), network_bindings :: undefined | [#ecs_network_binding{}], + network_interfaces :: undefined | [#ecs_network_interface{}], reason :: undefined | binary(), + runtime_id :: undefined | binary(), task_arn :: undefined | binary() + }). -record(ecs_container_override, { @@ -247,21 +259,51 @@ task_role_arn :: undefined | binary() }). +-record(ecs_attachment_detail, { + name :: undefined | binary(), + value :: undefined | binary() +}). + +-record(ecs_attachment, { + id :: undefined | binary(), + type :: undefined | binary(), + status :: undefined | binary(), + details :: undefined | [#ecs_attachment_detail{}] +}). + +-record(ecs_tag, { + key :: undefined | binary(), + value :: undefined | binary() +}). + -record(ecs_task, { + attachments :: undefined | [#ecs_attachment{}], + availability_zone :: undefined | binary(), cluster_arn :: undefined | binary(), + connectivity :: undefined | binary(), + connectivity_at :: undefined | number(), container_instance_arn :: undefined | binary(), containers :: undefined | [#ecs_container{}], + cpu :: undefined | binary(), created_at :: undefined | number(), - desired_status:: undefined | binary(), + desired_status :: undefined | binary(), + group :: undefined | binary(), + health_status :: undefined | binary(), last_status :: undefined | binary(), launch_type :: undefined | binary(), + memory :: undefined | binary(), overrides :: undefined | #ecs_task_override{}, - started_at :: undefined | pos_integer(), + platform_version :: undefined | binary(), + pull_started_at :: undefined | number(), + pull_stopped_at :: undefined | number(), + started_at :: undefined | number(), started_by :: undefined | binary(), - stopped_at :: undefined | pos_integer(), + stopped_at :: undefined | number(), stopped_reason :: undefined | binary(), + tags :: undefined | [#ecs_tag{}], task_arn :: undefined | binary(), - task_definition_arn :: undefined | binary() + task_definition_arn :: undefined | binary(), + version :: undefined | pos_integer() }). -record(ecs_describe_tasks, { diff --git a/src/erlcloud_ecs.erl b/src/erlcloud_ecs.erl index 40ce33650..cf5d8c411 100644 --- a/src/erlcloud_ecs.erl +++ b/src/erlcloud_ecs.erl @@ -1041,16 +1041,51 @@ network_binding_record() -> ] }. +-spec network_interface_record() -> record_desc(). +network_interface_record() -> + {#ecs_network_interface{}, + [ + {<<"attachmentId">>, #ecs_network_interface.attachment_id, fun id/2}, + {<<"privateIpv4Address">>, #ecs_network_interface.private_ipv4_address, fun id/2} + ] + }. + +-spec attachment_record() -> record_desc(). +attachment_record() -> + {#ecs_attachment{}, + [ + {<<"id">>, #ecs_attachment.id, fun id/2}, + {<<"type">>, #ecs_attachment.type, fun id/2}, + {<<"status">>, #ecs_attachment.status, fun id/2}, + {<<"details">>, #ecs_attachment.details, fun decode_attachment_details_list/2} + ] + }. + +-spec attachment_detail_record() -> record_desc(). +attachment_detail_record() -> + {#ecs_attachment_detail{}, + [ + {<<"name">>, #ecs_attachment_detail.name, fun id/2}, + {<<"value">>, #ecs_attachment_detail.value, fun id/2} + ] + }. + -spec container_record() -> record_desc(). container_record() -> {#ecs_container{}, [ {<<"containerArn">>, #ecs_container.container_arn, fun id/2}, + {<<"cpu">>, #ecs_container.cpu, fun id/2}, {<<"exitCode">>, #ecs_container.exit_code, fun id/2}, + {<<"healthStatus">>, #ecs_container.health_status, fun id/2}, + {<<"image">>, #ecs_container.image, fun id/2}, {<<"lastStatus">>, #ecs_container.last_status, fun id/2}, + {<<"memoryReservation">>, #ecs_container.memory_reservation, fun id/2}, {<<"name">>, #ecs_container.name, fun id/2}, {<<"networkBindings">>, #ecs_container.network_bindings, fun decode_network_bindings_list/2}, + {<<"networkInterfaces">>, #ecs_container.network_interfaces, fun decode_network_interfaces_list/2}, {<<"reason">>, #ecs_container.reason, fun id/2}, + {<<"runtimeId">>, #ecs_container.runtime_id, fun id/2}, {<<"taskArn">>, #ecs_container.task_arn, fun id/2} ] }. @@ -1074,24 +1109,46 @@ task_override_record() -> ] }. +-spec tag_record() -> record_desc(). +tag_record() -> + {#ecs_tag{}, + [ + {<<"key">>, #ecs_tag.key, fun id/2}, + {<<"value">>, #ecs_tag.value, fun id/2} + ] + }. + -spec task_record() -> record_desc(). task_record() -> {#ecs_task{}, [ + {<<"attachments">>, #ecs_task.attachments, fun decode_attachments_list/2}, + {<<"availabilityZone">>, #ecs_task.availability_zone, fun id/2}, {<<"clusterArn">>, #ecs_task.cluster_arn, fun id/2}, + {<<"connectivity">>, #ecs_task.connectivity, fun id/2}, + {<<"connectivityAt">>, #ecs_task.connectivity_at, fun id/2}, {<<"containerInstanceArn">>, #ecs_task.container_instance_arn, fun id/2}, {<<"containers">>, #ecs_task.containers, fun decode_containers_list/2}, + {<<"cpu">>, #ecs_task.cpu, fun id/2}, {<<"createdAt">>, #ecs_task.created_at, fun id/2}, {<<"desiredStatus">>, #ecs_task.desired_status, fun id/2}, + {<<"group">>, #ecs_task.group, fun id/2}, + {<<"healthStatus">>, #ecs_task.health_status, fun id/2}, {<<"lastStatus">>, #ecs_task.last_status, fun id/2}, {<<"launchType">>, #ecs_task.launch_type, fun id/2}, + {<<"memory">>, #ecs_task.memory, fun id/2}, {<<"overrides">>, #ecs_task.overrides, fun decode_task_overrides/2}, + {<<"platformVersion">>, #ecs_task.platform_version, fun id/2}, + {<<"pullStartedAt">>, #ecs_task.pull_started_at, fun id/2}, + {<<"pullStoppedAt">>, #ecs_task.pull_stopped_at, fun id/2}, {<<"startedAt">>, #ecs_task.started_at, fun id/2}, {<<"startedBy">>, #ecs_task.started_by, fun id/2}, {<<"stoppedAt">>, #ecs_task.stopped_at, fun id/2}, {<<"stoppedReason">>, #ecs_task.stopped_reason, fun id/2}, + {<<"tags">>, #ecs_task.tags, fun decode_tags_list/2}, {<<"taskArn">>, #ecs_task.task_arn, fun id/2}, - {<<"taskDefinitionArn">>, #ecs_task.task_definition_arn, fun id/2} + {<<"taskDefinitionArn">>, #ecs_task.task_definition_arn, fun id/2}, + {<<"version">>, #ecs_task.version, fun id/2} ] }. @@ -1161,15 +1218,27 @@ decode_volumes_from_list(V, Opts) -> decode_tasks_list(V, Opts) -> [decode_record(task_record(), I, Opts) || I <- V]. +decode_attachments_list(V, Opts) -> + [decode_record(attachment_record(), I, Opts) || I <- V]. + +decode_attachment_details_list(V, Opts) -> + [decode_record(attachment_detail_record(), I, Opts) || I <- V]. + decode_containers_list(V, Opts) -> [decode_record(container_record(), I, Opts) || I <- V]. decode_network_bindings_list(V, Opts) -> [decode_record(network_binding_record(), I, Opts) || I <- V]. +decode_network_interfaces_list(V, Opts) -> + [decode_record(network_interface_record(), I, Opts) || I <- V]. + decode_task_overrides(V, Opts) -> decode_record(task_override_record(), V, Opts). +decode_tags_list(V, Opts) -> + [decode_record(tag_record(), I, Opts) || I <- V]. + decode_container_overrides_list(V, Opts) -> [decode_record(container_override_record(), I, Opts) || I <- V]. %%%------------------------------------------------------------------------------ diff --git a/test/erlcloud_ecs_tests.erl b/test/erlcloud_ecs_tests.erl index 0f6da4621..74ed2a935 100644 --- a/test/erlcloud_ecs_tests.erl +++ b/test/erlcloud_ecs_tests.erl @@ -1680,7 +1680,35 @@ describe_tasks_output_tests(_) -> \"failures\": [], \"tasks\": [ { + \"attachments\": [ + { + \"id\": \"a1fda7d8-9592-4644-81ad-14578bc379ed\", + \"type\": \"ElasticNetworkInterface\", + \"status\": \"ATTACHED\", + \"details\": [ + { + \"name\": \"subnetId\", + \"value\": \"subnet-0c28a8c6b37e69447\" + }, + { + \"name\": \"networkInterfaceId\", + \"value\": \"eni-07f4ac7c00742fbfe\" + }, + { + \"name\": \"macAddress\", + \"value\": \"0e:2b:61:08:f9:21\" + }, + { + \"name\": \"privateIPv4Address\", + \"value\": \"10.0.0.92\" + } + ] + } + ], + \"availabilityZone\": \"us-east-1a\", \"clusterArn\": \"arn:aws:ecs:us-east-1:012345678910:cluster/default\", + \"connectivity\": \"CONNECTED\", + \"connectivityAt\": \"2020-11-17T13:54:45.465000+00:00\", \"containerInstanceArn\": \"arn:aws:ecs:us-east-1:012345678910:container-instance/84818520-995f-4d94-9d70-7714bacc2953\", \"containers\": [ { @@ -1704,9 +1732,14 @@ describe_tasks_output_tests(_) -> \"taskArn\": \"arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe\" } ], + \"cpu\": \"256\", + \"createdAt\": \"2020-11-17T13:54:33.275000+00:00\", \"desiredStatus\": \"RUNNING\", + \"group\": \"service:kktest-fargate-service\", + \"healthStatus\": \"UNKNOWN\", \"lastStatus\": \"RUNNING\", \"launchType\": \"FARGATE\", + \"memory\": \"512\", \"overrides\": { \"containerOverrides\": [ { @@ -1717,9 +1750,17 @@ describe_tasks_output_tests(_) -> } ] }, + \"platformVersion\": \"1.3.0\", + \"pullStartedAt\": \"2020-11-17T13:55:30.792000+00:00\", + \"pullStoppedAt\": \"2020-11-17T13:55:35.792000+00:00\", + \"startedAt\": \"2020-11-17T13:55:37.792000+00:00\", \"startedBy\": \"ecs-svc/9223370606521064774\", + \"tags\": [ + {\"key\": \"test-key\", \"value\": \"test-value\"} + ], \"taskArn\": \"arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe\", - \"taskDefinitionArn\": \"arn:aws:ecs:us-east-1:012345678910:task-definition/hello_world:10\" + \"taskDefinitionArn\": \"arn:aws:ecs:us-east-1:012345678910:task-definition/hello_world:10\", + \"version\": 3 } ] } @@ -1728,7 +1769,23 @@ describe_tasks_output_tests(_) -> failures = [], tasks = [ #ecs_task{ + attachments = [ + #ecs_attachment{ + id = <<"a1fda7d8-9592-4644-81ad-14578bc379ed">>, + type = <<"ElasticNetworkInterface">>, + status = <<"ATTACHED">>, + details = [ + #ecs_attachment_detail{name = <<"subnetId">>, value = <<"subnet-0c28a8c6b37e69447">>}, + #ecs_attachment_detail{name = <<"networkInterfaceId">>, value = <<"eni-07f4ac7c00742fbfe">>}, + #ecs_attachment_detail{name = <<"macAddress">>, value = <<"0e:2b:61:08:f9:21">>}, + #ecs_attachment_detail{name = <<"privateIPv4Address">>, value = <<"10.0.0.92">>} + ] + } + ], + availability_zone = <<"us-east-1a">>, cluster_arn = <<"arn:aws:ecs:us-east-1:012345678910:cluster/default">>, + connectivity = <<"CONNECTED">>, + connectivity_at = <<"2020-11-17T13:54:45.465000+00:00">>, container_instance_arn = <<"arn:aws:ecs:us-east-1:012345678910:container-instance/84818520-995f-4d94-9d70-7714bacc2953">>, containers = [ #ecs_container{ @@ -1752,9 +1809,14 @@ describe_tasks_output_tests(_) -> task_arn = <<"arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe">> } ], + cpu = <<"256">>, + created_at = <<"2020-11-17T13:54:33.275000+00:00">>, + group = <<"service:kktest-fargate-service">>, desired_status = <<"RUNNING">>, last_status = <<"RUNNING">>, launch_type = <<"FARGATE">>, + health_status = <<"UNKNOWN">>, + memory = <<"512">>, overrides = #ecs_task_override{ container_overrides = [ #ecs_container_override{ @@ -1765,9 +1827,19 @@ describe_tasks_output_tests(_) -> } ] }, + platform_version = <<"1.3.0">>, + pull_started_at = <<"2020-11-17T13:55:30.792000+00:00">>, + pull_stopped_at = <<"2020-11-17T13:55:35.792000+00:00">>, + started_at = <<"2020-11-17T13:55:37.792000+00:00">>, started_by = <<"ecs-svc/9223370606521064774">>, + stopped_at = undefined, + stopped_reason = undefined, + tags = [ + #ecs_tag{key = <<"test-key">>, value = <<"test-value">>} + ], task_arn = <<"arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe">>, - task_definition_arn = <<"arn:aws:ecs:us-east-1:012345678910:task-definition/hello_world:10">> + task_definition_arn = <<"arn:aws:ecs:us-east-1:012345678910:task-definition/hello_world:10">>, + version = 3 } ] }}}) From f0dc1e800da1396b0fbb93e6e1380956d25d4696 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Wed, 2 Dec 2020 10:27:33 +0000 Subject: [PATCH 159/310] Fix record spec --- include/erlcloud_ecs.hrl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/erlcloud_ecs.hrl b/include/erlcloud_ecs.hrl index 41b8b4c68..c13909ee0 100644 --- a/include/erlcloud_ecs.hrl +++ b/include/erlcloud_ecs.hrl @@ -281,11 +281,11 @@ availability_zone :: undefined | binary(), cluster_arn :: undefined | binary(), connectivity :: undefined | binary(), - connectivity_at :: undefined | number(), + connectivity_at :: undefined | binary(), container_instance_arn :: undefined | binary(), containers :: undefined | [#ecs_container{}], cpu :: undefined | binary(), - created_at :: undefined | number(), + created_at :: undefined | binary(), desired_status :: undefined | binary(), group :: undefined | binary(), health_status :: undefined | binary(), @@ -294,11 +294,11 @@ memory :: undefined | binary(), overrides :: undefined | #ecs_task_override{}, platform_version :: undefined | binary(), - pull_started_at :: undefined | number(), - pull_stopped_at :: undefined | number(), - started_at :: undefined | number(), + pull_started_at :: undefined | binary(), + pull_stopped_at :: undefined | binary(), + started_at :: undefined | binary(), started_by :: undefined | binary(), - stopped_at :: undefined | number(), + stopped_at :: undefined | binary(), stopped_reason :: undefined | binary(), tags :: undefined | [#ecs_tag{}], task_arn :: undefined | binary(), From 86675fb1c528a940cd4d09fb2d04f6793ccaa142 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Wed, 2 Dec 2020 12:49:32 +0000 Subject: [PATCH 160/310] Address comments --- include/erlcloud_ecs.hrl | 14 +++++------ test/erlcloud_ecs_tests.erl | 50 ++++++++++++++++++++++++++++--------- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/include/erlcloud_ecs.hrl b/include/erlcloud_ecs.hrl index c13909ee0..e6d6dfb3b 100644 --- a/include/erlcloud_ecs.hrl +++ b/include/erlcloud_ecs.hrl @@ -234,7 +234,7 @@ -record(ecs_container, { container_arn :: undefined | binary(), cpu :: undefined | binary(), - exit_code :: undefined | binary(), + exit_code :: undefined | integer(), health_status :: undefined | binary(), image :: undefined | binary(), last_status :: undefined | binary(), @@ -281,11 +281,11 @@ availability_zone :: undefined | binary(), cluster_arn :: undefined | binary(), connectivity :: undefined | binary(), - connectivity_at :: undefined | binary(), + connectivity_at :: undefined | number(), container_instance_arn :: undefined | binary(), containers :: undefined | [#ecs_container{}], cpu :: undefined | binary(), - created_at :: undefined | binary(), + created_at :: undefined | number(), desired_status :: undefined | binary(), group :: undefined | binary(), health_status :: undefined | binary(), @@ -294,11 +294,11 @@ memory :: undefined | binary(), overrides :: undefined | #ecs_task_override{}, platform_version :: undefined | binary(), - pull_started_at :: undefined | binary(), - pull_stopped_at :: undefined | binary(), - started_at :: undefined | binary(), + pull_started_at :: undefined | number(), + pull_stopped_at :: undefined | number(), + started_at :: undefined | number(), started_by :: undefined | binary(), - stopped_at :: undefined | binary(), + stopped_at :: undefined | number(), stopped_reason :: undefined | binary(), tags :: undefined | [#ecs_tag{}], task_arn :: undefined | binary(), diff --git a/test/erlcloud_ecs_tests.erl b/test/erlcloud_ecs_tests.erl index 74ed2a935..e29a2cbc5 100644 --- a/test/erlcloud_ecs_tests.erl +++ b/test/erlcloud_ecs_tests.erl @@ -1708,14 +1708,26 @@ describe_tasks_output_tests(_) -> \"availabilityZone\": \"us-east-1a\", \"clusterArn\": \"arn:aws:ecs:us-east-1:012345678910:cluster/default\", \"connectivity\": \"CONNECTED\", - \"connectivityAt\": \"2020-11-17T13:54:45.465000+00:00\", + \"connectivityAt\": 1605621285.465, \"containerInstanceArn\": \"arn:aws:ecs:us-east-1:012345678910:container-instance/84818520-995f-4d94-9d70-7714bacc2953\", \"containers\": [ { \"containerArn\": \"arn:aws:ecs:us-east-1:012345678910:container/76c980a8-2454-4a9c-acc4-9eb103117273\", + \"cpu\": \"256\", + \"exitCode\": 0, + \"healthStatus\": \"UNKNOWN\", + \"image\": \"nginx:latest\", \"lastStatus\": \"RUNNING\", + \"memoryReservation\": \"512\", \"name\": \"mysql\", \"networkBindings\": [], + \"networkInterfaces\": [ + { + \"attachmentId\": \"a1fda7d8-9592-4644-81ad-14578bc379ed\", + \"privateIpv4Address\": \"10.0.0.92\" + } + ], + \"runtimeId\": \"19c89c93db7d248eb14044ba24925d9e50ef581e5030b23490e71d1beff3fc4e\", \"taskArn\": \"arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe\" }, { @@ -1733,7 +1745,7 @@ describe_tasks_output_tests(_) -> } ], \"cpu\": \"256\", - \"createdAt\": \"2020-11-17T13:54:33.275000+00:00\", + \"createdAt\": 1605621273.275, \"desiredStatus\": \"RUNNING\", \"group\": \"service:kktest-fargate-service\", \"healthStatus\": \"UNKNOWN\", @@ -1751,10 +1763,12 @@ describe_tasks_output_tests(_) -> ] }, \"platformVersion\": \"1.3.0\", - \"pullStartedAt\": \"2020-11-17T13:55:30.792000+00:00\", - \"pullStoppedAt\": \"2020-11-17T13:55:35.792000+00:00\", - \"startedAt\": \"2020-11-17T13:55:37.792000+00:00\", + \"pullStartedAt\": 1605621330.792, + \"pullStoppedAt\": 1605621335.792, + \"startedAt\": 1605621337.792, \"startedBy\": \"ecs-svc/9223370606521064774\", + \"stoppedAt\": 1606906997.776, + \"stoppedReason\": \"Task stopped by user\", \"tags\": [ {\"key\": \"test-key\", \"value\": \"test-value\"} ], @@ -1785,14 +1799,26 @@ describe_tasks_output_tests(_) -> availability_zone = <<"us-east-1a">>, cluster_arn = <<"arn:aws:ecs:us-east-1:012345678910:cluster/default">>, connectivity = <<"CONNECTED">>, - connectivity_at = <<"2020-11-17T13:54:45.465000+00:00">>, + connectivity_at = 1605621285.465, container_instance_arn = <<"arn:aws:ecs:us-east-1:012345678910:container-instance/84818520-995f-4d94-9d70-7714bacc2953">>, containers = [ #ecs_container{ container_arn = <<"arn:aws:ecs:us-east-1:012345678910:container/76c980a8-2454-4a9c-acc4-9eb103117273">>, + cpu = <<"256">>, + exit_code = 0, + health_status = <<"UNKNOWN">>, + image = <<"nginx:latest">>, last_status = <<"RUNNING">>, + memory_reservation = <<"512">>, name = <<"mysql">>, network_bindings = [], + network_interfaces = [ + #ecs_network_interface{ + attachment_id = <<"a1fda7d8-9592-4644-81ad-14578bc379ed">>, + private_ipv4_address = <<"10.0.0.92">> + } + ], + runtime_id = <<"19c89c93db7d248eb14044ba24925d9e50ef581e5030b23490e71d1beff3fc4e">>, task_arn = <<"arn:aws:ecs:us-east-1:012345678910:task/c09f0188-7f87-4b0f-bfc3-16296622b6fe">> }, #ecs_container{ @@ -1810,7 +1836,7 @@ describe_tasks_output_tests(_) -> } ], cpu = <<"256">>, - created_at = <<"2020-11-17T13:54:33.275000+00:00">>, + created_at = 1605621273.275, group = <<"service:kktest-fargate-service">>, desired_status = <<"RUNNING">>, last_status = <<"RUNNING">>, @@ -1828,12 +1854,12 @@ describe_tasks_output_tests(_) -> ] }, platform_version = <<"1.3.0">>, - pull_started_at = <<"2020-11-17T13:55:30.792000+00:00">>, - pull_stopped_at = <<"2020-11-17T13:55:35.792000+00:00">>, - started_at = <<"2020-11-17T13:55:37.792000+00:00">>, + pull_started_at = 1605621330.792, + pull_stopped_at = 1605621335.792, + started_at = 1605621337.792, started_by = <<"ecs-svc/9223370606521064774">>, - stopped_at = undefined, - stopped_reason = undefined, + stopped_at = 1606906997.776, + stopped_reason = <<"Task stopped by user">>, tags = [ #ecs_tag{key = <<"test-key">>, value = <<"test-value">>} ], From fa3d02d87d22198e52bd5aae4d8e63f02b3f697e Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Wed, 2 Dec 2020 18:15:36 +0000 Subject: [PATCH 161/310] Fix task version spec --- include/erlcloud_ecs.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/erlcloud_ecs.hrl b/include/erlcloud_ecs.hrl index e6d6dfb3b..2e428d7be 100644 --- a/include/erlcloud_ecs.hrl +++ b/include/erlcloud_ecs.hrl @@ -303,7 +303,7 @@ tags :: undefined | [#ecs_tag{}], task_arn :: undefined | binary(), task_definition_arn :: undefined | binary(), - version :: undefined | pos_integer() + version :: undefined | non_neg_integer() }). -record(ecs_describe_tasks, { From 450ed460441de21b7db6771e600bdb5fa0c46f45 Mon Sep 17 00:00:00 2001 From: aaronlelevier Date: Sat, 5 Dec 2020 10:15:04 -0800 Subject: [PATCH 162/310] Add RoleLastUsed field to erlcloud_iam:get_role response This PR adds the RoleLastUsed field from the GetRole AWS API endpoint to the erlcloud_iam:get_role response fixes #670 --- src/erlcloud_iam.erl | 15 +++++++++-- test/erlcloud_iam_tests.erl | 51 +++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index db09def4a..b313899c5 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -416,10 +416,10 @@ get_role(RoleName) when is_list(RoleName) -> -spec get_role(string(), aws_config()) -> {ok, proplist()} | {error, any()}. get_role(RoleName, Config) -> get_role_impl([{"RoleName", RoleName}], Config). - + get_role_impl(RoleNameParam, #aws_config{} = Config) -> ItemPath = "/GetRoleResponse/GetRoleResult/Role", - case iam_query(Config, "GetRole", RoleNameParam, ItemPath, data_type("Role")) of + case iam_query(Config, "GetRole", RoleNameParam, ItemPath, data_type("RoleGetDetail")) of {ok, [Role]} -> {ok, Role}; {error, _} = Error -> Error end. @@ -981,6 +981,17 @@ data_type("Role") -> {"RoleId", role_id, "String"}, {"RoleName", role_name, "String"}, {"Path", path, "String"}]; +data_type("RoleGetDetail") -> + [{"Arn", arn, "String"}, + {"CreateDate", create_date, "DateTime"}, + {"AssumeRolePolicyDocument", assume_role_policy_doc, "Uri"}, + {"RoleId", role_id, "String"}, + {"RoleName", role_name, "String"}, + {"Path", path, "String"}, + {"RoleLastUsed", role_last_used, data_type("RoleLastUsed")}]; +data_type("RoleLastUsed") -> + [{"LastUsedDate", last_used_date, "DateTime"}, + {"Region", region, "String"}]; data_type("RoleDetail") -> [{"RolePolicyList/member", role_policy_list, data_type("PolicyDetail")}, {"RoleName", role_name, "String"}, diff --git a/test/erlcloud_iam_tests.erl b/test/erlcloud_iam_tests.erl index dd37f53e3..e6b25e383 100644 --- a/test/erlcloud_iam_tests.erl +++ b/test/erlcloud_iam_tests.erl @@ -38,6 +38,8 @@ iam_api_test_() -> fun get_group_policy_output_tests/1, fun get_login_profile_input_tests/1, fun get_login_profile_output_tests/1, + fun get_role_input_tests/1, + fun get_role_output_tests/1, fun get_role_policy_input_tests/1, fun get_role_policy_output_tests/1, fun get_user_policy_input_tests/1, @@ -682,6 +684,55 @@ get_login_profile_output_tests(_) -> ], output_tests(?_f(erlcloud_iam:get_login_profile("Bob")), Tests). +-define(GET_ROLE_RESP, + " + + + /application_abc/component_xyz/ + arn:aws:iam::123456789012:role/application_abc/component_xyz/S3Access + S3Access + {\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"ec2.amazonaws.com\"]},\"Action\":[\"sts:AssumeRole\"]}]} + 2012-05-08T23:34:01Z + AROADBQP57FF2AEXAMPLE + + 2013-05-08T23:34:01Z + us-east-1 + + + + "). + +get_role_input_tests(_) -> + Tests = + [?_iam_test( + {"Test returning role.", + ?_f(erlcloud_iam:get_role("test")), + [ + {"Action", "GetRole"}, + {"RoleName", "test"} + ]}) + ], + + input_tests(?GET_ROLE_RESP, Tests). + +get_role_output_tests(_) -> + Tests = [?_iam_test( + {"This returns the role", + ?GET_ROLE_RESP, + {ok, [{role_last_used, + [[{region,"us-east-1"}, + {last_used_date,{{2013,5,8},{23,34,1}}}]]}, + {path,"/application_abc/component_xyz/"}, + {role_name,"S3Access"}, + {role_id,"AROADBQP57FF2AEXAMPLE"}, + {assume_role_policy_doc, + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"ec2.amazonaws.com\"]},\"Action\":[\"sts:AssumeRole\"]}]}"}, + {create_date,{{2012,5,8},{23,34,1}}}, + {arn, "arn:aws:iam::123456789012:role/application_abc/component_xyz/S3Access"}]} + }) + ], + output_tests(?_f(erlcloud_iam:get_role("test")), Tests). + -define(GET_ROLE_POLICY_RESP, " From 75fb8789fbf2cc0dfd3aa79477f8f498c07fe76b Mon Sep 17 00:00:00 2001 From: aaronlelevier Date: Sat, 5 Dec 2020 21:00:41 -0800 Subject: [PATCH 163/310] fixup indentation --- test/erlcloud_iam_tests.erl | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/erlcloud_iam_tests.erl b/test/erlcloud_iam_tests.erl index e6b25e383..d565f0d1f 100644 --- a/test/erlcloud_iam_tests.erl +++ b/test/erlcloud_iam_tests.erl @@ -708,28 +708,28 @@ get_role_input_tests(_) -> {"Test returning role.", ?_f(erlcloud_iam:get_role("test")), [ - {"Action", "GetRole"}, - {"RoleName", "test"} - ]}) + {"Action", "GetRole"}, + {"RoleName", "test"} + ]}) ], input_tests(?GET_ROLE_RESP, Tests). get_role_output_tests(_) -> Tests = [?_iam_test( - {"This returns the role", - ?GET_ROLE_RESP, - {ok, [{role_last_used, - [[{region,"us-east-1"}, - {last_used_date,{{2013,5,8},{23,34,1}}}]]}, - {path,"/application_abc/component_xyz/"}, - {role_name,"S3Access"}, - {role_id,"AROADBQP57FF2AEXAMPLE"}, - {assume_role_policy_doc, - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"ec2.amazonaws.com\"]},\"Action\":[\"sts:AssumeRole\"]}]}"}, - {create_date,{{2012,5,8},{23,34,1}}}, - {arn, "arn:aws:iam::123456789012:role/application_abc/component_xyz/S3Access"}]} - }) + {"This returns the role", + ?GET_ROLE_RESP, + {ok, [{role_last_used, + [[{region,"us-east-1"}, + {last_used_date,{{2013,5,8},{23,34,1}}}]]}, + {path,"/application_abc/component_xyz/"}, + {role_name,"S3Access"}, + {role_id,"AROADBQP57FF2AEXAMPLE"}, + {assume_role_policy_doc, + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":[\"ec2.amazonaws.com\"]},\"Action\":[\"sts:AssumeRole\"]}]}"}, + {create_date,{{2012,5,8},{23,34,1}}}, + {arn, "arn:aws:iam::123456789012:role/application_abc/component_xyz/S3Access"}]} + }) ], output_tests(?_f(erlcloud_iam:get_role("test")), Tests). From db41e5bd197844c4420b05e437a2d411c7889a78 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Mon, 7 Dec 2020 16:22:47 +0000 Subject: [PATCH 164/310] Fix ECS to_binary conersion --- src/erlcloud_ecs.erl | 1 + test/erlcloud_ecs_tests.erl | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/erlcloud_ecs.erl b/src/erlcloud_ecs.erl index cf5d8c411..9630a9304 100644 --- a/src/erlcloud_ecs.erl +++ b/src/erlcloud_ecs.erl @@ -2591,6 +2591,7 @@ to_binary(undefined) -> undefined; to_binary(true) -> true; to_binary(false) -> false; to_binary(L) when is_list(L), is_list(hd(L)) -> [to_binary(V) || V <- L]; +to_binary(L) when is_list(L), is_binary(hd(L)) -> [to_binary(V) || V <- L]; to_binary(L) when is_list(L) -> list_to_binary(L); to_binary(B) when is_binary(B) -> B; to_binary(A) when is_atom(A) -> atom_to_binary(A, latin1). diff --git a/test/erlcloud_ecs_tests.erl b/test/erlcloud_ecs_tests.erl index e29a2cbc5..83e8dd0ec 100644 --- a/test/erlcloud_ecs_tests.erl +++ b/test/erlcloud_ecs_tests.erl @@ -1619,6 +1619,16 @@ describe_tasks_input_tests(_) -> \"c09f0188-7f87-4b0f-bfc3-16296622b6fe\" ] } +" + }), + ?_ecs_test( + {"DescribeTasks binary input example request", + ?_f(erlcloud_ecs:describe_tasks([<<"c09f0188-7f87-4b0f-bfc3-16296622b6fe">>])), " +{ + \"tasks\": [ + \"c09f0188-7f87-4b0f-bfc3-16296622b6fe\" + ] +} " }) ], From f65a9e09dc009560c101e47d77d811da9f7e70d1 Mon Sep 17 00:00:00 2001 From: aaronlelevier Date: Mon, 7 Dec 2020 18:08:20 -0800 Subject: [PATCH 165/310] change data_type 'Role' to 'RoleList' --- src/erlcloud_iam.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index b313899c5..ff0148226 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -436,7 +436,7 @@ list_roles(PathPrefix) -> list_roles(PathPrefix, #aws_config{} = Config) when is_list(PathPrefix) -> ItemPath = "/ListRolesResponse/ListRolesResult/Roles/member", - iam_query(Config, "ListRoles", [{"PathPrefix", PathPrefix}], ItemPath, data_type("Role")). + iam_query(Config, "ListRoles", [{"PathPrefix", PathPrefix}], ItemPath, data_type("RoleList")). -spec list_roles_all() -> {ok, proplist()} | {error, any()}. list_roles_all() -> list_roles([]). @@ -450,7 +450,7 @@ list_roles_all(PathPrefix) -> list_roles_all(PathPrefix, #aws_config{} = Config) when is_list(PathPrefix) -> ItemPath = "/ListRolesResponse/ListRolesResult/Roles/member", - iam_query_all(Config, "ListRoles", [{"PathPrefix", PathPrefix}], ItemPath, data_type("Role")). + iam_query_all(Config, "ListRoles", [{"PathPrefix", PathPrefix}], ItemPath, data_type("RoleList")). -spec list_role_policies(string()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. list_role_policies(RoleName) -> @@ -938,7 +938,7 @@ data_type("InstanceProfile") -> {"Arn", arn, "String"}, {"Path", path, "String"}, {"InstanceProfileName", instance_profile_name, "String"}, - {"Roles/member", roles, data_type("Role")}, + {"Roles/member", roles, data_type("RoleList")}, {"InstanceProfileId", instance_profile_id, "String"}]; data_type("Group") -> [{"Path", path, "String"}, @@ -974,7 +974,7 @@ data_type("PasswordPolicy") -> data_type("PolicyDetail") -> [{"PolicyName", policy_name, "String"}, {"PolicyDocument", policy_document, "Uri"}]; -data_type("Role") -> +data_type("RoleList") -> [{"Arn", arn, "String"}, {"CreateDate", create_date, "DateTime"}, {"AssumeRolePolicyDocument", assume_role_policy_doc, "Uri"}, From effc230cfac88e745efed194c27672253c2c24e0 Mon Sep 17 00:00:00 2001 From: aaronlelevier Date: Mon, 7 Dec 2020 18:14:35 -0800 Subject: [PATCH 166/310] change get_role to use data_type 'Role' --- src/erlcloud_iam.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index ff0148226..5a80a6899 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -419,7 +419,7 @@ get_role(RoleName, Config) -> get_role_impl(RoleNameParam, #aws_config{} = Config) -> ItemPath = "/GetRoleResponse/GetRoleResult/Role", - case iam_query(Config, "GetRole", RoleNameParam, ItemPath, data_type("RoleGetDetail")) of + case iam_query(Config, "GetRole", RoleNameParam, ItemPath, data_type("Role")) of {ok, [Role]} -> {ok, Role}; {error, _} = Error -> Error end. @@ -981,7 +981,7 @@ data_type("RoleList") -> {"RoleId", role_id, "String"}, {"RoleName", role_name, "String"}, {"Path", path, "String"}]; -data_type("RoleGetDetail") -> +data_type("Role") -> [{"Arn", arn, "String"}, {"CreateDate", create_date, "DateTime"}, {"AssumeRolePolicyDocument", assume_role_policy_doc, "Uri"}, From 0736703dde55f2d50625be205cb8efd78803df17 Mon Sep 17 00:00:00 2001 From: aaronlelevier Date: Mon, 7 Dec 2020 18:39:41 -0800 Subject: [PATCH 167/310] move data_type 'RoleLastUsed' after 'RoleDetail' --- src/erlcloud_iam.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index 5a80a6899..aa143d357 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -989,9 +989,6 @@ data_type("Role") -> {"RoleName", role_name, "String"}, {"Path", path, "String"}, {"RoleLastUsed", role_last_used, data_type("RoleLastUsed")}]; -data_type("RoleLastUsed") -> - [{"LastUsedDate", last_used_date, "DateTime"}, - {"Region", region, "String"}]; data_type("RoleDetail") -> [{"RolePolicyList/member", role_policy_list, data_type("PolicyDetail")}, {"RoleName", role_name, "String"}, @@ -1001,6 +998,9 @@ data_type("RoleDetail") -> {"CreateDate", create_date, "DateTime"}, {"AssumeRolePolicyDocument", assume_role_policy_document, "Uri"}, {"Arn", arn, "String"}]; +data_type("RoleLastUsed") -> + [{"LastUsedDate", last_used_date, "DateTime"}, + {"Region", region, "String"}]; data_type("RolePolicyList") -> [{"PolicyDocument", policy_document, "Uri"}, {"RoleName", role_name, "String"}, From ffc48c2f97bf3ffb97e1d1e74e7cbf285fb213ba Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Mon, 7 Dec 2020 18:16:55 -0600 Subject: [PATCH 168/310] erlcloud_iam: fix get_account_authorization_details When reviewing https://github.com/erlcloud/erlcloud/pull/671, I noticed that IAM [GetAccountAuthorizationDetails](https://docs.aws.amazon.com/IAM/latest/APIReference/API_GetAccountAuthorizationDetails.html) is a paginated API, and when you use it on an account with a paginated response, it crashes rather inelegantly. This change makes it work correctly. `get_account_authorization_details_all` is left as an exercise for some other time. --- src/erlcloud_iam.erl | 16 +- test/erlcloud_iam_tests.erl | 337 ++++++++++++++++++++++++++++++++++++ 2 files changed, 349 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index db09def4a..0b28e5934 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -52,7 +52,7 @@ list_instance_profiles/0, list_instance_profiles/1, list_instance_profiles/2, list_instance_profiles_all/0, list_instance_profiles_all/1, list_instance_profiles_all/2, get_instance_profile/1, get_instance_profile/2, - get_account_authorization_details/0, get_account_authorization_details/1, + get_account_authorization_details/0, get_account_authorization_details/1, get_account_authorization_details/2, get_account_summary/0, get_account_summary/1, get_account_password_policy/0, get_account_password_policy/1, generate_credential_report/0, generate_credential_report/1, @@ -664,17 +664,25 @@ get_instance_profile(ProfileName, #aws_config{} = Config) -> % % Account APIs % --spec get_account_authorization_details() -> {ok, proplist()} | {error, any()}. +-spec get_account_authorization_details() -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. get_account_authorization_details() -> get_account_authorization_details(default_config()). --spec get_account_authorization_details(aws_config()) -> {ok, proplist()} | {error, any()}. +-spec get_account_authorization_details(list() | aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. get_account_authorization_details(#aws_config{} = Config) -> + get_account_authorization_details([], #aws_config{} = Config); +get_account_authorization_details(Params) -> + get_account_authorization_details(Params, default_config()). + +-spec get_account_authorization_details(list(), aws_config()) -> {ok, proplist()} | {ok, proplist(), string()} | {error, any()}. +get_account_authorization_details(Params, #aws_config{} = Config) when is_list(Params) -> ItemPath = "/GetAccountAuthorizationDetailsResponse/GetAccountAuthorizationDetailsResult", DataTypeDef = data_type("AccountAuthorizationDetails"), - case iam_query(Config, "GetAccountAuthorizationDetails", [], ItemPath, DataTypeDef) of + case iam_query(Config, "GetAccountAuthorizationDetails", Params, ItemPath, DataTypeDef) of {ok, [Summary]} -> {ok, Summary}; + {ok, [Summary], Marker} -> + {ok, Summary, Marker}; {error, _} = Error -> Error end. diff --git a/test/erlcloud_iam_tests.erl b/test/erlcloud_iam_tests.erl index dd37f53e3..dc435c5e6 100644 --- a/test/erlcloud_iam_tests.erl +++ b/test/erlcloud_iam_tests.erl @@ -1992,6 +1992,247 @@ get_instance_profile_output_tests(_) -> "). +-define(GET_ACCOUNT_AUTHORIZATION_DETAILS_RESP_TRUNC, + " + + true + + + + Admins + + + AIDACKCEVSQ6C2EXAMPLE + / + Alice + arn:aws:iam::123456789012:user/Alice + 2013-10-14T18:32:24Z + + + + Admins + + + + + DenyBillingAndIAMPolicy + {\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Deny\",\"Action\":[\"aws-portal:*\",\"iam:*\"],\"Resource\":\"*\"}} + + + AIDACKCEVSQ6C3EXAMPLE + / + Bob + arn:aws:iam::123456789012:user/Bob + 2013-10-14T18:32:25Z + + + + Dev + + + AIDACKCEVSQ6C4EXAMPLE + / + Charlie + arn:aws:iam::123456789012:user/Charlie + 2013-10-14T18:33:56Z + + + + Dev + + + AIDACKCEVSQ6C5EXAMPLE + / + Danielle + arn:aws:iam::123456789012:user/Danielle + 2013-10-14T18:33:56Z + + + + Finance + + + AIDACKCEVSQ6C6EXAMPLE + / + Elaine + arn:aws:iam::123456789012:user/Elaine + 2013-10-14T18:57:48Z + + + EXAMPLEkakv9BCuUNFDtxWSyfzetYwEx2ADc8dnzfvERF5S6YMvXKx41t6gCl/eeaCX3Jo94/bKqezEAg8TEVS99EKFLxm3jtbpl25FDWEXAMPLE + + + AIDACKCEVSQ6C7EXAMPLE + + + AdministratorAccess + arn:aws:iam::aws:policy/AdministratorAccess + + + Admins + / + arn:aws:iam::123456789012:group/Admins + 2013-10-14T18:32:24Z + + + + AIDACKCEVSQ6C8EXAMPLE + + + PowerUserAccess + arn:aws:iam::aws:policy/PowerUserAccess + + + Dev + / + arn:aws:iam::123456789012:group/Dev + 2013-10-14T18:33:55Z + + + + AIDACKCEVSQ6C9EXAMPLE + + Finance + / + arn:aws:iam::123456789012:group/Finance + 2013-10-14T18:57:48Z + + + policygen-201310141157 + {\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"aws-portal:*\"],\"Sid\":\"Stmt1381777017000\",\"Resource\":[\"*\"],\"Effect\":\"Allow\"}]} + + + + + + + + + + AmazonS3FullAccess + arn:aws:iam::aws:policy/AmazonS3FullAccess + + + AmazonDynamoDBFullAccess + arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + + + + + EC2role + + + / + arn:aws:iam::123456789012:role/EC2role + EC2role + {\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]} + 2014-07-30T17:09:20Z + AROAFP4BKI7Y7TEXAMPLE + + 2019-11-20T17:09:20Z + us-east-1 + + + + / + arn:aws:iam::123456789012:instance-profile/EC2role + AIPAFFYRBHWXW2EXAMPLE + 2014-07-30T17:09:20Z + + + / + arn:aws:iam::123456789012:role/EC2role + EC2role + {\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]} + 2014-07-30T17:09:20Z + AROAFP4BKI7Y7TEXAMPLE + + + + create-update-delete-set-managed-policies + v1 + ANPAJ2UCCR6DPCEXAMPLE + / + + + {\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":[\"iam:CreatePolicy\",\"iam:CreatePolicyVersion\",\"iam:DeletePolicy\",\"iam:DeletePolicyVersion\",\"iam:GetPolicy\",\"iam:GetPolicyVersion\",\"iam:ListPolicies\",\"iam:ListPolicyVersions\",\"iam:SetDefaultPolicyVersion\"],\"Resource\":\"*\"}} + true + v1 + 2015-02-06T19:58:34Z + + + + arn:aws:iam::123456789012:policy/create-update-delete-set-managed-policies + + 1 + 2015-02-06T19:58:34Z + true + 2015-02-06T19:58:34Z + + + S3-read-only-specific-bucket + v1 + ANPAJ4AE5446DAEXAMPLE + / + + + {\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:Get*\",\"s3:List*\"],\"Resource\":[\"arn:aws:s3:::example-bucket\",\"arn:aws:s3:::example-bucket/*\"]}]} + true + v1 + 2015-01-21T21:39:41Z + + + arn:aws:iam::123456789012:policy/S3-read-only-specific-bucket + 1 + 2015-01-21T21:39:41Z + true + 2015-01-21T23:39:41Z + + + AWSOpsWorksRole + v1 + ANPAE376NQ77WV6KGJEBE + /service-role/ + + + {\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"cloudwatch:GetMetricStatistics\",\"ec2:DescribeAccountAttributes\",\"ec2:DescribeAvailabilityZones\",\"ec2:DescribeInstances\",\"ec2:DescribeKeyPairs\",\"ec2:DescribeSecurityGroups\",\"ec2:DescribeSubnets\",\"ec2:DescribeVpcs\",\"elasticloadbalancing:DescribeInstanceHealth\",\"elasticloadbalancing:DescribeLoadBalancers\",\"iam:GetRolePolicy\",\"iam:ListInstanceProfiles\",\"iam:ListRoles\",\"iam:ListUsers\",\"iam:PassRole\",\"opsworks:*\",\"rds:*\"],\"Resource\":[\"*\"]}]} + true + v1 + 2014-12-10T22:57:47Z + + + arn:aws:iam::aws:policy/service-role/AWSOpsWorksRole + 1 + 2015-02-06T18:41:27Z + true + 2015-02-06T18:41:27Z + + + AmazonEC2FullAccess + v1 + ANPAE3QWE5YT46TQ34WLG + / + + + {\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":\"ec2:*\",\"Effect\":\"Allow\",\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":\"elasticloadbalancing:*\",\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":\"cloudwatch:*\",\"Resource\":\"*\"},{\"Effect\":\"Allow\",\"Action\":\"autoscaling:*\",\"Resource\":\"*\"}]} + true + v1 + 2014-10-30T20:59:46Z + + + arn:aws:iam::aws:policy/AmazonEC2FullAccess + 1 + 2015-02-06T18:40:15Z + true + 2015-02-06T18:40:15Z + + + + + 92e79ae7-7399-11e4-8c85-4b53eEXAMPLE + + "). + get_account_authorization_details_input_tests(_) -> Tests = [?_iam_test( @@ -1999,6 +2240,13 @@ get_account_authorization_details_input_tests(_) -> ?_f(erlcloud_iam:get_account_authorization_details()), [ {"Action", "GetAccountAuthorizationDetails"} + ]}), + ?_iam_test( + {"Test returning the authorization details.", + ?_f(erlcloud_iam:get_account_authorization_details([{"Marker", "XXX"}])), + [ + {"Action", "GetAccountAuthorizationDetails"}, + {"Marker", "XXX"} ]}) ], @@ -2101,6 +2349,95 @@ get_account_authorization_details_output_tests(_) -> {user_id,"AIDACKCEVSQ6C6EXAMPLE"}, {user_name,"Elaine"}, {user_policy_list,[]}]]}]} + }), + ?_iam_test( + {"This returns the authorization details (truncated)", + ?GET_ACCOUNT_AUTHORIZATION_DETAILS_RESP_TRUNC, + {ok,[{roles, + [[{arn,"arn:aws:iam::123456789012:role/EC2role"}, + {assume_role_policy_document, + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}"}, + {create_date,{{2014,7,30},{17,9,20}}}, + {instance_profiles, + [[{instance_profile_id,"AIPAFFYRBHWXW2EXAMPLE"}, + {roles, + [[{path,"/"}, + {role_name,"EC2role"}, + {role_id,"AROAFP4BKI7Y7TEXAMPLE"}, + {assume_role_policy_doc, + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"ec2.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}"}, + {create_date,{{2014,7,30},{17,9,20}}}, + {arn, "arn:aws:iam::123456789012:role/EC2role"}]]}, + {instance_profile_name,"EC2role"}, + {path,"/"}, + {arn,"arn:aws:iam::123456789012:instance-profile/EC2role"}, + {create_date,{{2014,7,30},{17,9,20}}}]]}, + {path,"/"}, + {role_id,"AROAFP4BKI7Y7TEXAMPLE"}, + {role_name,"EC2role"}, + {role_policy_list,[]}]]}, + {groups, + [[{arn,"arn:aws:iam::123456789012:group/Admins"}, + {create_date,{{2013,10,14},{18,32,24}}}, + {group_id,"AIDACKCEVSQ6C7EXAMPLE"}, + {group_name,"Admins"}, + {group_policy_list,[]}, + {path,"/"}], + [{arn,"arn:aws:iam::123456789012:group/Dev"}, + {create_date,{{2013,10,14},{18,33,55}}}, + {group_id,"AIDACKCEVSQ6C8EXAMPLE"}, + {group_name,"Dev"}, + {group_policy_list,[]}, + {path,"/"}], + [{arn,"arn:aws:iam::123456789012:group/Finance"}, + {create_date,{{2013,10,14},{18,57,48}}}, + {group_id,"AIDACKCEVSQ6C9EXAMPLE"}, + {group_name,"Finance"}, + {group_policy_list, + [[{policy_document, + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Action\":[\"aws-portal:*\"],\"Sid\":\"Stmt1381777017000\",\"Resource\":[\"*\"],\"Effect\":\"Allow\"}]}"}, + {policy_name,"policygen-201310141157"}]]}, + {path,"/"}]]}, + {users, + [[{arn,"arn:aws:iam::123456789012:user/Alice"}, + {create_date,{{2013,10,14},{18,32,24}}}, + {group_list,[[{group_name,"Admins"}]]}, + {path,"/"}, + {user_id,"AIDACKCEVSQ6C2EXAMPLE"}, + {user_name,"Alice"}, + {user_policy_list,[]}], + [{arn,"arn:aws:iam::123456789012:user/Bob"}, + {create_date,{{2013,10,14},{18,32,25}}}, + {group_list,[[{group_name,"Admins"}]]}, + {path,"/"}, + {user_id,"AIDACKCEVSQ6C3EXAMPLE"}, + {user_name,"Bob"}, + {user_policy_list, + [[{policy_document, + "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Deny\",\"Action\":[\"aws-portal:*\",\"iam:*\"],\"Resource\":\"*\"}}"}, + {policy_name,"DenyBillingAndIAMPolicy"}]]}], + [{arn,"arn:aws:iam::123456789012:user/Charlie"}, + {create_date,{{2013,10,14},{18,33,56}}}, + {group_list,[[{group_name,"Dev"}]]}, + {path,"/"}, + {user_id,"AIDACKCEVSQ6C4EXAMPLE"}, + {user_name,"Charlie"}, + {user_policy_list,[]}], + [{arn,"arn:aws:iam::123456789012:user/Danielle"}, + {create_date,{{2013,10,14},{18,33,56}}}, + {group_list,[[{group_name,"Dev"}]]}, + {path,"/"}, + {user_id,"AIDACKCEVSQ6C5EXAMPLE"}, + {user_name,"Danielle"}, + {user_policy_list,[]}], + [{arn,"arn:aws:iam::123456789012:user/Elaine"}, + {create_date,{{2013,10,14},{18,57,48}}}, + {group_list,[[{group_name,"Finance"}]]}, + {path,"/"}, + {user_id,"AIDACKCEVSQ6C6EXAMPLE"}, + {user_name,"Elaine"}, + {user_policy_list,[]}]]}], + "EXAMPLEkakv9BCuUNFDtxWSyfzetYwEx2ADc8dnzfvERF5S6YMvXKx41t6gCl/eeaCX3Jo94/bKqezEAg8TEVS99EKFLxm3jtbpl25FDWEXAMPLE"} }) ], output_tests(?_f(erlcloud_iam:get_account_authorization_details()), Tests). From ed0d82f2302d76d6c77b496c856c5ee6c0d248ba Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 8 Dec 2020 21:24:04 +0000 Subject: [PATCH 169/310] Implement GitHub Actions --- .github/workflows/ci.yml | 25 +++++++++++++++++++++++++ .github/workflows/publish.yml | 13 +++++++++++++ README.md | 2 +- 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..9bd2a0581 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +--- +name: build +on: + push: + branches: + - master + pull_request: + branches: + - master +jobs: + ci: + name: Run checks and tests over ${{matrix.otp_vsn}} and ${{matrix.os}} (rebar2? ${{matrix.force_rebar2}}) + runs-on: ${{matrix.os}} + strategy: + matrix: + otp_vsn: [19.3, 20.3, 21.3, 22.3, 23.0] + os: [ubuntu-latest] + force_rebar2: [true, false] + steps: + - uses: actions/checkout@v2 + - run: export FORCE_REBAR2=${{matrix.force_rebar2}} + - run: make travis-install + - run: make warnings + - run: make check + - run: make eunit diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..80afac6cb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,13 @@ +--- +name: publish +on: + push: + tags: + - '*' +jobs: + pub: + name: Publish to Hex.pm + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: make travis-publish diff --git a/README.md b/README.md index 6683bcd91..afe370d85 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # erlcloud: AWS APIs library for Erlang # -[![Build Status](https://secure.travis-ci.org/erlcloud/erlcloud.png?branch=master)](http://travis-ci.org/erlcloud/erlcloud) +[![Build Status](https://github.com/erlcloud/erlcloud/workflows/build/badge.svg)](https://github.com/erlcloud/erlcloud) This library is not developed or maintained by AWS thus lots of functionality is still missing comparing to [aws-cli](https://aws.amazon.com/cli/) or [boto](https://github.com/boto/boto). Required functionality is being added upon request. From 148d40c345e80cf6acac04392005264dae8961aa Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Wed, 9 Dec 2020 11:08:49 +0000 Subject: [PATCH 170/310] Adde constraint when requesting from ddb --- src/erlcloud_ddb_impl.erl | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 1e3f9b02f..09006edd9 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -68,15 +68,16 @@ ]). %% Internal impl api --export([request/3]). +-export([request/4]). -export_type([json_return/0, attempt/0, retry_fun/0]). -type json_return() :: ok | {ok, jsx:json_term()} | {error, term()}. -type operation() :: string(). --spec request(aws_config(), operation(), jsx:json_term()) -> json_return(). -request(Config0, Operation, Json) -> + +-spec request(aws_config(), boolean(), operation(), jsx:json_term()) -> json_return(). +request(Config0, MakeRequest, Operation, Json) -> Body = case Json of [] -> <<"{}">>; _ -> jsx:encode(Json) @@ -84,7 +85,7 @@ request(Config0, Operation, Json) -> case erlcloud_aws:update_config(Config0) of {ok, Config} -> Headers = headers(Config, Operation, Body), - request_and_retry(Config, Headers, Body, {attempt, 1}); + maybe_request_and_retry(Config, Headers, Body, {attempt, 1}, MakeRequest); {error, Reason} -> {error, Reason} end. @@ -99,6 +100,11 @@ request(Config0, Operation, Json) -> %% This algorithm is similar, except that it waits a random interval up to 2^(Attempt-2)*100ms. The average %% wait time should be the same as boto. +maybe_request_and_retry(Config, Headers, Body, Attempt, false) -> + request_and_retry(Config, Headers, Body, Attempt); +maybe_request_and_retry(_Config, Headers, Body, _Attempt, true) -> + [Headers, Body]. + %% TODO refactor retry logic so that it can be used by all requests and move to erlcloud_aws -define(NUM_ATTEMPTS, 10). From 1383e90885f72e8e0e16cb96f214133986967fde Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Wed, 9 Dec 2020 11:17:25 +0000 Subject: [PATCH 171/310] Updated requests to be compliant with new request --- src/erlcloud_ddb2.erl | 88 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index f2130ce2a..5e44b02f1 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -959,11 +959,12 @@ id(X) -> X. -type out_type() :: json | record | typed_record | simple. -type out_opt() :: {out, out_type()}. +-type no_request_opt() :: {no_request, boolean()}. -type boolean_opt(Name) :: Name | {Name, boolean()}. -type property() :: proplists:property(). -type aws_opts() :: [json_pair()]. --type ddb_opts() :: [out_opt()]. +-type ddb_opts() :: [out_opt() | no_request_opt()]. -type opts() :: {aws_opts(), ddb_opts()}. -spec verify_ddb_opt(atom(), term()) -> ok. @@ -974,6 +975,13 @@ verify_ddb_opt(out, Value) -> false -> error({erlcloud_ddb, {invalid_opt, {out, Value}}}) end; +verify_ddb_opt(no_request, Value) -> + case is_boolean(Value) of + true -> + ok; + false -> + error({erlcloud_ddb, {invalid_opt, {no_request, Value}}}) + end; verify_ddb_opt(Name, Value) -> error({erlcloud_ddb, {invalid_opt, {Name, Value}}}). @@ -1150,7 +1158,9 @@ out({ok, Json}, Undynamize, Opts) -> {ok, Undynamize(Json, [{typed, true}])}; simple -> {simple, Undynamize(Json, [])} - end. + end; +out(Request, _Undynamize, _Opts) when is_list(Request)-> + Request. %% Returns specified field of tuple for simple return -spec out(erlcloud_ddb_impl:json_return(), undynamize_fun(), ddb_opts(), pos_integer()) @@ -1574,8 +1584,10 @@ batch_get_item(RequestItems, Opts) -> batch_get_item_return(). batch_get_item(RequestItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(batch_get_item_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.BatchGetItem", [{<<"RequestItems">>, dynamize_batch_get_item_request_items(RequestItems)}] ++ AwsOpts), @@ -1699,8 +1711,10 @@ batch_write_item(RequestItems, Opts) -> batch_write_item_return(). batch_write_item(RequestItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(batch_write_item_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.BatchWriteItem", [{<<"RequestItems">>, dynamize_batch_write_item_request_items(RequestItems)}] ++ AwsOpts), @@ -1770,8 +1784,10 @@ create_backup(BackupName, TableName, Opts) create_backup(BackupName, TableName, Opts, Config) when is_binary(BackupName), is_binary(TableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.CreateBackup", [{<<"TableName">>, TableName}, {<<"BackupName">>, BackupName}]), @@ -1833,8 +1849,10 @@ create_global_table(GlobalTableName, ReplicationGroup, Config) when is_record(Co -> create_global_table_return(). create_global_table(GlobalTableName, ReplicationGroup, Opts, Config) -> {[], DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.CreateGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}, {<<"ReplicationGroup">>, dynamize_maybe_list(fun dynamize_replica/1, ReplicationGroup)}]), @@ -1943,8 +1961,10 @@ create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits) create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, [], default_config()); create_table(Table, AttrDefs, KeySchema, Opts, Config) -> {AwsOpts, DdbOpts} = opts(create_table_opts(KeySchema), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.CreateTable", [{<<"TableName">>, Table}, {<<"AttributeDefinitions">>, dynamize_attr_defs(AttrDefs)}, @@ -2073,8 +2093,10 @@ delete_backup(BackupArn, Opts) delete_backup(BackupArn, Opts, Config) when is_binary(BackupArn) -> {_AwsOpts, DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DeleteBackup", [{<<"BackupArn">>, BackupArn}]), out(Return, fun(Json, UOpts) -> undynamize_record(delete_backup_record(), Json, UOpts) end, @@ -2164,8 +2186,10 @@ delete_item(Table, Key, Opts) -> -spec delete_item(table_name(), key(), delete_item_opts(), aws_config()) -> delete_item_return(). delete_item(Table, Key, Opts, Config) -> {AwsOpts, DdbOpts} = opts(delete_item_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DeleteItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] @@ -2212,8 +2236,10 @@ delete_table(Table, Opts) -> -spec delete_table(table_name(), ddb_opts(), aws_config()) -> delete_table_return(). delete_table(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DeleteTable", [{<<"TableName">>, Table}]), out(Return, fun(Json, UOpts) -> undynamize_record(delete_table_record(), Json, UOpts) end, @@ -2261,8 +2287,10 @@ describe_backup(BackupArn, Opts) describe_backup(BackupArn, Opts, Config) when is_binary(BackupArn) -> {_AwsOpts, DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DescribeBackup", [{<<"BackupArn">>, BackupArn}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_backup_record(), Json, UOpts) end, @@ -2326,8 +2354,10 @@ describe_continuous_backups(TableName, Opts) describe_continuous_backups(TableName, Opts, Config) when is_binary(TableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DescribeContinuousBackups", [{<<"TableName">>, TableName}]), out(Return, fun(Json, UOpts) -> undynamize_record(continuous_backups_record(), Json, UOpts) end, @@ -2376,8 +2406,10 @@ describe_global_table(GlobalTableName, Config) when is_record(Config, aws_config -> describe_global_table_return(). describe_global_table(GlobalTableName, Opts, Config) -> {[], DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DescribeGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_global_table_record(), Json, UOpts) end, @@ -2424,8 +2456,10 @@ describe_global_table_settings(GlobalTableName, Opts) -> -spec describe_global_table_settings(table_name(), ddb_opts(), aws_config()) -> describe_global_table_settings_return(). describe_global_table_settings(GlobalTableName, Opts, Config) -> {[], DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DescribeGlobalTableSettings", [{<<"GlobalTableName">>, GlobalTableName}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_global_table_settings_record(), Json, UOpts) end, @@ -2472,8 +2506,10 @@ describe_limits(Opts) -> -spec describe_limits(ddb_opts(), aws_config()) -> describe_limits_return(). describe_limits(Opts, Config) -> {[], DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DescribeLimits", []), case out(Return, fun(Json, UOpts) -> undynamize_record(describe_limits_record(), Json, UOpts) end, @@ -2522,8 +2558,10 @@ describe_table(Table, Opts) -> -spec describe_table(table_name(), ddb_opts(), aws_config()) -> describe_table_return(). describe_table(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DescribeTable", [{<<"TableName">>, Table}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_table_record(), Json, UOpts) end, @@ -2569,8 +2607,10 @@ describe_table_replica_auto_scaling(Table, Opts) -> -spec describe_table_replica_auto_scaling(table_name(), ddb_opts(), aws_config()) -> describe_table_replica_auto_scaling_return(). describe_table_replica_auto_scaling(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DescribeTableReplicaAutoScaling", [{<<"TableName">>, Table}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_table_replica_auto_scaling_record(), Json, UOpts) end, @@ -2628,8 +2668,10 @@ describe_time_to_live(Table, DbOpts) -> %%------------------------------------------------------------------------------ -spec describe_time_to_live(table_name(), ddb_opts(), aws_config()) -> describe_time_to_live_return(). describe_time_to_live(Table, DbOpts, Config) -> + MakeRequest = proplists:get_value(DbOpts, no_request, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.DescribeTimeToLive", [{<<"TableName">>, Table}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_time_to_live_record(), Json, UOpts) end, @@ -2696,8 +2738,10 @@ get_item(Table, Key, Opts) -> -spec get_item(table_name(), key(), get_item_opts(), aws_config()) -> get_item_return(). get_item(Table, Key, Opts, Config) -> {AwsOpts, DdbOpts} = opts(get_item_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.GetItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] @@ -2784,8 +2828,10 @@ list_backups(Opts) -> -spec list_backups([list_backups_opt()], aws_config()) -> list_backups_return(). list_backups(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_backups_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.ListBackups", AwsOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_backups_record(), Json, UOpts) end, @@ -2850,8 +2896,10 @@ list_global_tables(Config) when is_record(Config, aws_config) -> -> list_global_tables_return(). list_global_tables(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_global_tables_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.ListGlobalTables", AwsOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_global_tables_record(), Json, UOpts) end, @@ -2908,8 +2956,10 @@ list_tables(Opts) -> -spec list_tables(list_tables_opts(), aws_config()) -> list_tables_return(). list_tables(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_tables_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.ListTables", AwsOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_tables_record(), Json, UOpts) end, @@ -2974,8 +3024,10 @@ list_tags_of_resource(ResourceArn, Opts) -> -> list_tags_of_resource_return(). list_tags_of_resource(ResourceArn, Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_tags_of_resource_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.ListTagsOfResource", [{<<"ResourceArn">>, ResourceArn} | AwsOpts]), out(Return, fun(Json, UOpts) -> undynamize_record(list_tags_of_resource_record(), Json, UOpts) end, @@ -3075,8 +3127,10 @@ put_item(Table, Item, Opts) -> -spec put_item(table_name(), in_item(), put_item_opts(), aws_config()) -> put_item_return(). put_item(Table, Item, Opts, Config) -> {AwsOpts, DdbOpts} = opts(put_item_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.PutItem", [{<<"TableName">>, Table}, {<<"Item">>, dynamize_item(Item)}] @@ -3186,8 +3240,10 @@ q(Table, KeyConditionsOrExpression, Opts) -> -spec q(table_name(), conditions() | expression(), q_opts(), aws_config()) -> q_return(). q(Table, KeyConditionsOrExpression, Opts, Config) -> {AwsOpts, DdbOpts} = opts(q_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.Query", [{<<"TableName">>, Table}, dynamize_q_key_conditions_or_expression(KeyConditionsOrExpression)] @@ -3237,8 +3293,10 @@ restore_table_from_backup(BackupArn, TargetTableName, Opts) restore_table_from_backup(BackupArn, TargetTableName, Opts, Config) when is_binary(BackupArn), is_binary(TargetTableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.RestoreTableFromBackup", [{<<"BackupArn">>, BackupArn}, {<<"TargetTableName">>, TargetTableName}]), @@ -3296,8 +3354,10 @@ restore_table_to_point_in_time(SourceTableName, TargetTableName, Opts) restore_table_to_point_in_time(SourceTableName, TargetTableName, Opts, Config) when is_binary(SourceTableName), is_binary(TargetTableName) -> {AwsOpts, DdbOpts} = opts(restore_table_to_point_in_time_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.RestoreTableToPointInTime", [{<<"SourceTableName">>, SourceTableName}, {<<"TargetTableName">>, TargetTableName}] ++ AwsOpts), @@ -3386,8 +3446,10 @@ scan(Table, Opts) -> -spec scan(table_name(), scan_opts(), aws_config()) -> scan_return(). scan(Table, Opts, Config) -> {AwsOpts, DdbOpts} = opts(scan_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.Scan", [{<<"TableName">>, Table}] ++ AwsOpts), @@ -3434,7 +3496,7 @@ tag_resource(ResourceArn, Tags) -> -> tag_resource_return(). tag_resource(ResourceArn, Tags, Config) -> erlcloud_ddb_impl:request( - Config, + Config, false, "DynamoDB_20120810.TagResource", [{<<"ResourceArn">>, ResourceArn}, {<<"Tags">>, dynamize_tags(Tags)}]). @@ -3532,8 +3594,10 @@ transact_get_items(RequestItems, Opts) -> transact_get_items_return(). transact_get_items(TransactItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(transact_get_items_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.TransactGetItems", [{<<"TransactItems">>, dynamize_transact_get_items_transact_items(TransactItems)}] ++ AwsOpts), @@ -3676,8 +3740,10 @@ transact_write_items(RequestItems, Opts) -> transact_write_items_return(). transact_write_items(TransactItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(transact_write_items_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.TransactWriteItems", [{<<"TransactItems">>, dynamize_transact_write_items_transact_items(TransactItems)}] ++ AwsOpts), @@ -3718,7 +3784,7 @@ untag_resource(ResourceArn, TagKeys) -> -> untag_resource_return(). untag_resource(ResourceArn, TagKeys, Config) -> erlcloud_ddb_impl:request( - Config, + Config, false, "DynamoDB_20120810.UntagResource", [{<<"ResourceArn">>, ResourceArn}, {<<"TagKeys">>, TagKeys}]). @@ -3762,8 +3828,10 @@ update_continuous_backups(TableName, PointInTimeRecoveryEnabled, Opts) update_continuous_backups(TableName, PointInTimeRecoveryEnabled, Opts, Config) when is_binary(TableName), is_boolean(PointInTimeRecoveryEnabled) -> {_AwsOpts, DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.UpdateContinuousBackups", [{<<"TableName">>, TableName}, {<<"PointInTimeRecoverySpecification">>, dynamize_point_in_time_recovery_enabled(PointInTimeRecoveryEnabled)}]), @@ -3883,8 +3951,10 @@ update_item(Table, Key, UpdatesOrExpression, Opts) -> -> update_item_return(). update_item(Table, Key, UpdatesOrExpression, Opts, Config) -> {AwsOpts, DdbOpts} = opts(update_item_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.UpdateItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] @@ -3948,8 +4018,10 @@ update_global_table(GlobalTableName, ReplicaUpdates, Config) when is_record(Conf -> update_global_table_return(). update_global_table(GlobalTableName, ReplicaUpdates, Opts, Config) -> {[], DdbOpts} = opts([], Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.UpdateGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}, {<<"ReplicaUpdates">>, dynamize_maybe_list(fun dynamize_replica_update/1, ReplicaUpdates)}]), @@ -4096,8 +4168,10 @@ update_global_table_settings(GlobalTableName, Opts) -> -spec update_global_table_settings(table_name(), update_global_table_settings_opts(), aws_config()) -> update_global_table_settings_return(). update_global_table_settings(GlobalTableName, Opts, Config) -> {AwsOpts, DdbOpts} = opts(update_global_table_settings_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.UpdateGlobalTableSettings", [{<<"GlobalTableName">>, GlobalTableName} | AwsOpts]), out(Return, fun(Json, UOpts) -> undynamize_record(update_global_table_settings_record(), Json, UOpts) end, @@ -4265,8 +4339,10 @@ update_table(Table, Opts) -> (table_name(), read_units(), write_units()) -> update_table_return(). update_table(Table, Opts, Config) when is_list(Opts) -> {AwsOpts, DdbOpts} = opts(update_table_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.UpdateTable", [{<<"TableName">>, Table} | AwsOpts]), out(Return, fun(Json, UOpts) -> undynamize_record(update_table_record(), Json, UOpts) end, @@ -4411,8 +4487,10 @@ update_table_replica_auto_scaling(Table, Opts, Config) when is_list(Opts) -> % Any non-default timeout already written to the config will not be overridden. UpdatedConfig = maybe_update_config_timeout(Config, 25000), {AwsOpts, DdbOpts} = opts(update_table_replica_auto_scaling_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( UpdatedConfig, + MakeRequest, "DynamoDB_20120810.UpdateTableReplicaAutoScaling", [{<<"TableName">>, Table} | AwsOpts]), out(Return, fun(Json, UOpts) -> undynamize_record(update_table_replica_auto_scaling_record(), Json, UOpts) end, @@ -4482,9 +4560,11 @@ update_time_to_live(Table, AttributeName, Enabled, Config) -> (table_name(), attr_name(), boolean()) -> update_time_to_live_return(). update_time_to_live(Table, Opts, Config) when is_list(Opts) -> {AwsOpts, DdbOpts} = opts(update_time_to_live_opts(), Opts), + MakeRequest = proplists:get_value(no_request, DdbOpts, false), Body = [{<<"TableName">>, Table}, {<<"TimeToLiveSpecification">>, AwsOpts}], Return = erlcloud_ddb_impl:request( Config, + MakeRequest, "DynamoDB_20120810.UpdateTimeToLive", Body), out(Return, fun(Json, UOpts) -> undynamize_record(update_time_to_live_record(), Json, UOpts) end, From 80f55c2d203351e4d5247d83de1b43818303ac6e Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Wed, 9 Dec 2020 11:19:31 +0000 Subject: [PATCH 172/310] Updated no_request variable for readibility purposes --- src/erlcloud_ddb2.erl | 140 +++++++++++++++++++++--------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 5e44b02f1..a2a88e05c 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1584,10 +1584,10 @@ batch_get_item(RequestItems, Opts) -> batch_get_item_return(). batch_get_item(RequestItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(batch_get_item_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.BatchGetItem", [{<<"RequestItems">>, dynamize_batch_get_item_request_items(RequestItems)}] ++ AwsOpts), @@ -1711,10 +1711,10 @@ batch_write_item(RequestItems, Opts) -> batch_write_item_return(). batch_write_item(RequestItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(batch_write_item_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.BatchWriteItem", [{<<"RequestItems">>, dynamize_batch_write_item_request_items(RequestItems)}] ++ AwsOpts), @@ -1784,10 +1784,10 @@ create_backup(BackupName, TableName, Opts) create_backup(BackupName, TableName, Opts, Config) when is_binary(BackupName), is_binary(TableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.CreateBackup", [{<<"TableName">>, TableName}, {<<"BackupName">>, BackupName}]), @@ -1849,10 +1849,10 @@ create_global_table(GlobalTableName, ReplicationGroup, Config) when is_record(Co -> create_global_table_return(). create_global_table(GlobalTableName, ReplicationGroup, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.CreateGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}, {<<"ReplicationGroup">>, dynamize_maybe_list(fun dynamize_replica/1, ReplicationGroup)}]), @@ -1961,10 +1961,10 @@ create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits) create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, [], default_config()); create_table(Table, AttrDefs, KeySchema, Opts, Config) -> {AwsOpts, DdbOpts} = opts(create_table_opts(KeySchema), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.CreateTable", [{<<"TableName">>, Table}, {<<"AttributeDefinitions">>, dynamize_attr_defs(AttrDefs)}, @@ -2093,10 +2093,10 @@ delete_backup(BackupArn, Opts) delete_backup(BackupArn, Opts, Config) when is_binary(BackupArn) -> {_AwsOpts, DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DeleteBackup", [{<<"BackupArn">>, BackupArn}]), out(Return, fun(Json, UOpts) -> undynamize_record(delete_backup_record(), Json, UOpts) end, @@ -2186,10 +2186,10 @@ delete_item(Table, Key, Opts) -> -spec delete_item(table_name(), key(), delete_item_opts(), aws_config()) -> delete_item_return(). delete_item(Table, Key, Opts, Config) -> {AwsOpts, DdbOpts} = opts(delete_item_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DeleteItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] @@ -2236,10 +2236,10 @@ delete_table(Table, Opts) -> -spec delete_table(table_name(), ddb_opts(), aws_config()) -> delete_table_return(). delete_table(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DeleteTable", [{<<"TableName">>, Table}]), out(Return, fun(Json, UOpts) -> undynamize_record(delete_table_record(), Json, UOpts) end, @@ -2287,10 +2287,10 @@ describe_backup(BackupArn, Opts) describe_backup(BackupArn, Opts, Config) when is_binary(BackupArn) -> {_AwsOpts, DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DescribeBackup", [{<<"BackupArn">>, BackupArn}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_backup_record(), Json, UOpts) end, @@ -2354,10 +2354,10 @@ describe_continuous_backups(TableName, Opts) describe_continuous_backups(TableName, Opts, Config) when is_binary(TableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DescribeContinuousBackups", [{<<"TableName">>, TableName}]), out(Return, fun(Json, UOpts) -> undynamize_record(continuous_backups_record(), Json, UOpts) end, @@ -2406,10 +2406,10 @@ describe_global_table(GlobalTableName, Config) when is_record(Config, aws_config -> describe_global_table_return(). describe_global_table(GlobalTableName, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DescribeGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_global_table_record(), Json, UOpts) end, @@ -2456,10 +2456,10 @@ describe_global_table_settings(GlobalTableName, Opts) -> -spec describe_global_table_settings(table_name(), ddb_opts(), aws_config()) -> describe_global_table_settings_return(). describe_global_table_settings(GlobalTableName, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DescribeGlobalTableSettings", [{<<"GlobalTableName">>, GlobalTableName}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_global_table_settings_record(), Json, UOpts) end, @@ -2506,10 +2506,10 @@ describe_limits(Opts) -> -spec describe_limits(ddb_opts(), aws_config()) -> describe_limits_return(). describe_limits(Opts, Config) -> {[], DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DescribeLimits", []), case out(Return, fun(Json, UOpts) -> undynamize_record(describe_limits_record(), Json, UOpts) end, @@ -2558,10 +2558,10 @@ describe_table(Table, Opts) -> -spec describe_table(table_name(), ddb_opts(), aws_config()) -> describe_table_return(). describe_table(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DescribeTable", [{<<"TableName">>, Table}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_table_record(), Json, UOpts) end, @@ -2607,10 +2607,10 @@ describe_table_replica_auto_scaling(Table, Opts) -> -spec describe_table_replica_auto_scaling(table_name(), ddb_opts(), aws_config()) -> describe_table_replica_auto_scaling_return(). describe_table_replica_auto_scaling(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DescribeTableReplicaAutoScaling", [{<<"TableName">>, Table}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_table_replica_auto_scaling_record(), Json, UOpts) end, @@ -2668,10 +2668,10 @@ describe_time_to_live(Table, DbOpts) -> %%------------------------------------------------------------------------------ -spec describe_time_to_live(table_name(), ddb_opts(), aws_config()) -> describe_time_to_live_return(). describe_time_to_live(Table, DbOpts, Config) -> - MakeRequest = proplists:get_value(DbOpts, no_request, false), + NoRequest = proplists:get_value(DbOpts, no_request, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.DescribeTimeToLive", [{<<"TableName">>, Table}]), out(Return, fun(Json, UOpts) -> undynamize_record(describe_time_to_live_record(), Json, UOpts) end, @@ -2738,10 +2738,10 @@ get_item(Table, Key, Opts) -> -spec get_item(table_name(), key(), get_item_opts(), aws_config()) -> get_item_return(). get_item(Table, Key, Opts, Config) -> {AwsOpts, DdbOpts} = opts(get_item_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.GetItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] @@ -2828,10 +2828,10 @@ list_backups(Opts) -> -spec list_backups([list_backups_opt()], aws_config()) -> list_backups_return(). list_backups(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_backups_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.ListBackups", AwsOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_backups_record(), Json, UOpts) end, @@ -2896,10 +2896,10 @@ list_global_tables(Config) when is_record(Config, aws_config) -> -> list_global_tables_return(). list_global_tables(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_global_tables_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.ListGlobalTables", AwsOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_global_tables_record(), Json, UOpts) end, @@ -2956,10 +2956,10 @@ list_tables(Opts) -> -spec list_tables(list_tables_opts(), aws_config()) -> list_tables_return(). list_tables(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_tables_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.ListTables", AwsOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_tables_record(), Json, UOpts) end, @@ -3024,10 +3024,10 @@ list_tags_of_resource(ResourceArn, Opts) -> -> list_tags_of_resource_return(). list_tags_of_resource(ResourceArn, Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_tags_of_resource_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.ListTagsOfResource", [{<<"ResourceArn">>, ResourceArn} | AwsOpts]), out(Return, fun(Json, UOpts) -> undynamize_record(list_tags_of_resource_record(), Json, UOpts) end, @@ -3127,10 +3127,10 @@ put_item(Table, Item, Opts) -> -spec put_item(table_name(), in_item(), put_item_opts(), aws_config()) -> put_item_return(). put_item(Table, Item, Opts, Config) -> {AwsOpts, DdbOpts} = opts(put_item_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.PutItem", [{<<"TableName">>, Table}, {<<"Item">>, dynamize_item(Item)}] @@ -3240,10 +3240,10 @@ q(Table, KeyConditionsOrExpression, Opts) -> -spec q(table_name(), conditions() | expression(), q_opts(), aws_config()) -> q_return(). q(Table, KeyConditionsOrExpression, Opts, Config) -> {AwsOpts, DdbOpts} = opts(q_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.Query", [{<<"TableName">>, Table}, dynamize_q_key_conditions_or_expression(KeyConditionsOrExpression)] @@ -3293,10 +3293,10 @@ restore_table_from_backup(BackupArn, TargetTableName, Opts) restore_table_from_backup(BackupArn, TargetTableName, Opts, Config) when is_binary(BackupArn), is_binary(TargetTableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.RestoreTableFromBackup", [{<<"BackupArn">>, BackupArn}, {<<"TargetTableName">>, TargetTableName}]), @@ -3354,10 +3354,10 @@ restore_table_to_point_in_time(SourceTableName, TargetTableName, Opts) restore_table_to_point_in_time(SourceTableName, TargetTableName, Opts, Config) when is_binary(SourceTableName), is_binary(TargetTableName) -> {AwsOpts, DdbOpts} = opts(restore_table_to_point_in_time_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.RestoreTableToPointInTime", [{<<"SourceTableName">>, SourceTableName}, {<<"TargetTableName">>, TargetTableName}] ++ AwsOpts), @@ -3446,10 +3446,10 @@ scan(Table, Opts) -> -spec scan(table_name(), scan_opts(), aws_config()) -> scan_return(). scan(Table, Opts, Config) -> {AwsOpts, DdbOpts} = opts(scan_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.Scan", [{<<"TableName">>, Table}] ++ AwsOpts), @@ -3594,10 +3594,10 @@ transact_get_items(RequestItems, Opts) -> transact_get_items_return(). transact_get_items(TransactItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(transact_get_items_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.TransactGetItems", [{<<"TransactItems">>, dynamize_transact_get_items_transact_items(TransactItems)}] ++ AwsOpts), @@ -3740,10 +3740,10 @@ transact_write_items(RequestItems, Opts) -> transact_write_items_return(). transact_write_items(TransactItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(transact_write_items_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.TransactWriteItems", [{<<"TransactItems">>, dynamize_transact_write_items_transact_items(TransactItems)}] ++ AwsOpts), @@ -3828,10 +3828,10 @@ update_continuous_backups(TableName, PointInTimeRecoveryEnabled, Opts) update_continuous_backups(TableName, PointInTimeRecoveryEnabled, Opts, Config) when is_binary(TableName), is_boolean(PointInTimeRecoveryEnabled) -> {_AwsOpts, DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.UpdateContinuousBackups", [{<<"TableName">>, TableName}, {<<"PointInTimeRecoverySpecification">>, dynamize_point_in_time_recovery_enabled(PointInTimeRecoveryEnabled)}]), @@ -3951,10 +3951,10 @@ update_item(Table, Key, UpdatesOrExpression, Opts) -> -> update_item_return(). update_item(Table, Key, UpdatesOrExpression, Opts, Config) -> {AwsOpts, DdbOpts} = opts(update_item_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.UpdateItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] @@ -4018,10 +4018,10 @@ update_global_table(GlobalTableName, ReplicaUpdates, Config) when is_record(Conf -> update_global_table_return(). update_global_table(GlobalTableName, ReplicaUpdates, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.UpdateGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}, {<<"ReplicaUpdates">>, dynamize_maybe_list(fun dynamize_replica_update/1, ReplicaUpdates)}]), @@ -4168,10 +4168,10 @@ update_global_table_settings(GlobalTableName, Opts) -> -spec update_global_table_settings(table_name(), update_global_table_settings_opts(), aws_config()) -> update_global_table_settings_return(). update_global_table_settings(GlobalTableName, Opts, Config) -> {AwsOpts, DdbOpts} = opts(update_global_table_settings_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.UpdateGlobalTableSettings", [{<<"GlobalTableName">>, GlobalTableName} | AwsOpts]), out(Return, fun(Json, UOpts) -> undynamize_record(update_global_table_settings_record(), Json, UOpts) end, @@ -4339,10 +4339,10 @@ update_table(Table, Opts) -> (table_name(), read_units(), write_units()) -> update_table_return(). update_table(Table, Opts, Config) when is_list(Opts) -> {AwsOpts, DdbOpts} = opts(update_table_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.UpdateTable", [{<<"TableName">>, Table} | AwsOpts]), out(Return, fun(Json, UOpts) -> undynamize_record(update_table_record(), Json, UOpts) end, @@ -4487,10 +4487,10 @@ update_table_replica_auto_scaling(Table, Opts, Config) when is_list(Opts) -> % Any non-default timeout already written to the config will not be overridden. UpdatedConfig = maybe_update_config_timeout(Config, 25000), {AwsOpts, DdbOpts} = opts(update_table_replica_auto_scaling_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( UpdatedConfig, - MakeRequest, + NoRequest, "DynamoDB_20120810.UpdateTableReplicaAutoScaling", [{<<"TableName">>, Table} | AwsOpts]), out(Return, fun(Json, UOpts) -> undynamize_record(update_table_replica_auto_scaling_record(), Json, UOpts) end, @@ -4560,11 +4560,11 @@ update_time_to_live(Table, AttributeName, Enabled, Config) -> (table_name(), attr_name(), boolean()) -> update_time_to_live_return(). update_time_to_live(Table, Opts, Config) when is_list(Opts) -> {AwsOpts, DdbOpts} = opts(update_time_to_live_opts(), Opts), - MakeRequest = proplists:get_value(no_request, DdbOpts, false), + NoRequest = proplists:get_value(no_request, DdbOpts, false), Body = [{<<"TableName">>, Table}, {<<"TimeToLiveSpecification">>, AwsOpts}], Return = erlcloud_ddb_impl:request( Config, - MakeRequest, + NoRequest, "DynamoDB_20120810.UpdateTimeToLive", Body), out(Return, fun(Json, UOpts) -> undynamize_record(update_time_to_live_record(), Json, UOpts) end, From 73b448acec9decbfba1efd5d9829e2c4042920c2 Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Wed, 9 Dec 2020 11:35:32 +0000 Subject: [PATCH 173/310] Added list as type of json_return --- src/erlcloud_ddb_impl.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 09006edd9..944a6a0cc 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -72,7 +72,7 @@ -export_type([json_return/0, attempt/0, retry_fun/0]). --type json_return() :: ok | {ok, jsx:json_term()} | {error, term()}. +-type json_return() :: ok | {ok, jsx:json_term()} | {error, term()} | list(). -type operation() :: string(). From 0bbd21c7a37646eb623e674ddf286d5d8cc30f36 Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Wed, 9 Dec 2020 11:36:00 +0000 Subject: [PATCH 174/310] Updated describe_time_to_live --- src/erlcloud_ddb2.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index a2a88e05c..851d0fef6 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1159,7 +1159,7 @@ out({ok, Json}, Undynamize, Opts) -> simple -> {simple, Undynamize(Json, [])} end; -out(Request, _Undynamize, _Opts) when is_list(Request)-> +out(Request, _Undynamize, _Opts) when is_list(Request) -> Request. %% Returns specified field of tuple for simple return @@ -2668,7 +2668,7 @@ describe_time_to_live(Table, DbOpts) -> %%------------------------------------------------------------------------------ -spec describe_time_to_live(table_name(), ddb_opts(), aws_config()) -> describe_time_to_live_return(). describe_time_to_live(Table, DbOpts, Config) -> - NoRequest = proplists:get_value(DbOpts, no_request, false), + NoRequest = proplists:get_value(no_request, DbOpts, false), Return = erlcloud_ddb_impl:request( Config, NoRequest, From 7b5c167b758bde949b5ce254f1c2078fa4e5dc27 Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Wed, 9 Dec 2020 11:38:28 +0000 Subject: [PATCH 175/310] Updated request function --- src/erlcloud_ddb1.erl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/erlcloud_ddb1.erl b/src/erlcloud_ddb1.erl index 52723b02d..7c49d640b 100644 --- a/src/erlcloud_ddb1.erl +++ b/src/erlcloud_ddb1.erl @@ -93,7 +93,7 @@ batch_get_item(RequestItems) -> -spec batch_get_item([batch_get_item_request_item()], aws_config()) -> json_return(). batch_get_item(RequestItems, Config) -> Json = [{<<"RequestItems">>, [batch_get_item_request_item_json(R) || R <- RequestItems]}], - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.BatchGetItem", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.BatchGetItem", Json). -type batch_write_item_put() :: {put, item()}. -type batch_write_item_delete() :: {delete, key()}. @@ -117,7 +117,7 @@ batch_write_item(RequestItems) -> -spec batch_write_item([batch_write_item_request_item()], aws_config()) -> json_return(). batch_write_item(RequestItems, Config) -> Json = [{<<"RequestItems">>, [batch_write_item_request_item_json(R) || R <- RequestItems]}], - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.BatchWriteItem", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.BatchWriteItem", Json). -spec key_schema_value_json(key_schema_value()) -> jsx:json_term(). @@ -141,7 +141,7 @@ create_table(Table, KeySchema, ReadUnits, WriteUnits, Config) -> key_schema_json(KeySchema), {<<"ProvisionedThroughput">>, [{<<"ReadCapacityUnits">>, ReadUnits}, {<<"WriteCapacityUnits">>, WriteUnits}]}], - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.CreateTable", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.CreateTable", Json). -spec delete_item(table_name(), key()) -> json_return(). @@ -157,7 +157,7 @@ delete_item(Table, Key, Opts, Config) -> Json = [{<<"TableName">>, Table}, key_json(Key)] ++ Opts, - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.DeleteItem", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.DeleteItem", Json). -spec delete_table(table_name()) -> json_return(). @@ -167,7 +167,7 @@ delete_table(Table) -> -spec delete_table(table_name(), aws_config()) -> json_return(). delete_table(Table, Config) -> Json = [{<<"TableName">>, Table}], - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.DeleteTable", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.DeleteTable", Json). -spec describe_table(table_name()) -> json_return(). @@ -177,7 +177,7 @@ describe_table(Table) -> -spec describe_table(table_name(), aws_config()) -> json_return(). describe_table(Table, Config) -> Json = [{<<"TableName">>, Table}], - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.DescribeTable", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.DescribeTable", Json). -spec get_item(table_name(), key()) -> json_return(). @@ -193,7 +193,7 @@ get_item(Table, Key, Opts, Config) -> Json = [{<<"TableName">>, Table}, key_json(Key)] ++ Opts, - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.GetItem", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.GetItem", Json). -spec list_tables() -> json_return(). @@ -206,7 +206,7 @@ list_tables(Opts) -> -spec list_tables(opts(), aws_config()) -> json_return(). list_tables(Opts, Config) -> - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.ListTables", Opts). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.ListTables", Opts). -spec put_item(table_name(), item()) -> json_return(). @@ -222,7 +222,7 @@ put_item(Table, Item, Opts, Config) -> Json = [{<<"TableName">>, Table}, item_json(Item)] ++ Opts, - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.PutItem", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.PutItem", Json). -spec q(table_name(), hash_key()) -> json_return(). @@ -238,7 +238,7 @@ q(Table, HashKey, Opts, Config) -> Json = [{<<"TableName">>, Table}, hash_key_json(HashKey)] ++ Opts, - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.Query", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.Query", Json). -spec scan(table_name()) -> json_return(). @@ -253,7 +253,7 @@ scan(Table, Opts) -> scan(Table, Opts, Config) -> Json = [{<<"TableName">>, Table}] ++ Opts, - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.Scan", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.Scan", Json). -spec update_item(table_name(), key(), updates()) -> json_return(). @@ -270,7 +270,7 @@ update_item(Table, Key, Updates, Opts, Config) -> key_json(Key), updates_json(Updates)] ++ Opts, - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.UpdateItem", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.UpdateItem", Json). -spec update_table(table_name(), non_neg_integer(), non_neg_integer()) -> json_return(). @@ -282,7 +282,7 @@ update_table(Table, ReadUnits, WriteUnits, Config) -> Json = [{<<"TableName">>, Table}, {<<"ProvisionedThroughput">>, [{<<"ReadCapacityUnits">>, ReadUnits}, {<<"WriteCapacityUnits">>, WriteUnits}]}], - erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.UpdateTable", Json). + erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.UpdateTable", Json). %% backoff and retry are here for backwards compat. Use the ones in erlcloud_ddb_impl instead. From 50448cfabd0aa77505298b39f6b19e1fb08abcdd Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 10 Dec 2020 00:34:20 +0000 Subject: [PATCH 176/310] Actually run on different Erlang versions --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bd2a0581..ee6f66ced 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,8 @@ jobs: ci: name: Run checks and tests over ${{matrix.otp_vsn}} and ${{matrix.os}} (rebar2? ${{matrix.force_rebar2}}) runs-on: ${{matrix.os}} + container: + image: erlang:${{matrix.otp_vsn}} strategy: matrix: otp_vsn: [19.3, 20.3, 21.3, 22.3, 23.0] From 18d9a523ae7b4ca3d131d05d16a9c9c958ef2212 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 10 Dec 2020 00:34:34 +0000 Subject: [PATCH 177/310] Delete unwarranted CI element --- .github/workflows/publish.yml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 80afac6cb..000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,13 +0,0 @@ ---- -name: publish -on: - push: - tags: - - '*' -jobs: - pub: - name: Publish to Hex.pm - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - run: make travis-publish From b978dde48c5af53982aa7cfe7e65aae959587e3b Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 10 Dec 2020 00:54:33 +0000 Subject: [PATCH 178/310] Try to make it work for rebar 2 --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee6f66ced..8e3d817be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,9 +18,11 @@ jobs: otp_vsn: [19.3, 20.3, 21.3, 22.3, 23.0] os: [ubuntu-latest] force_rebar2: [true, false] + env: + FORCE_REBAR2: ${{matrix.force_rebar2}} steps: - uses: actions/checkout@v2 - - run: export FORCE_REBAR2=${{matrix.force_rebar2}} + - run: export - run: make travis-install - run: make warnings - run: make check From 9696eae5a8853d3d55b7e23d4b0742d998b42b74 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 10 Dec 2020 01:38:20 +0000 Subject: [PATCH 179/310] Bye, Travis CI --- .travis.yml | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index db346c397..000000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -sudo: false -language: erlang -otp_release: - - 19.3 - - 20.3 - - 21.3 - - 22.3 - - 23.0 -env: - global: - - MAIN_OTP=20.3 - matrix: - - FORCE_REBAR2=true - - FORCE_REBAR2=false PATH=$PATH:$PWD -branches: - only: - - master - - /^[0-9]+\.[0-9]+\.[0-9]+/ -install: - - make travis-install -script: - - make warnings - - make check - - make eunit -deploy: - skip_cleanup: true - provider: script - script: make travis-publish - on: - tags: true - condition: $FORCE_REBAR2 = false && $TRAVIS_OTP_RELEASE = $MAIN_OTP From 2695fec7f7c26f9d28e53735da457bb7ea307231 Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Thu, 10 Dec 2020 09:56:35 +0000 Subject: [PATCH 180/310] Added request/4 --- src/erlcloud_ddb_impl.erl | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 944a6a0cc..56278dba7 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -68,16 +68,23 @@ ]). %% Internal impl api --export([request/4]). +-export([request/3, + request/4]). -export_type([json_return/0, attempt/0, retry_fun/0]). --type json_return() :: ok | {ok, jsx:json_term()} | {error, term()} | list(). +-type json_return() :: ok | {ok, jsx:json_term()} | {error, term()} | {ok, {request, headers(), jsx:json_text()}}. -type operation() :: string(). --spec request(aws_config(), boolean(), operation(), jsx:json_term()) -> json_return(). -request(Config0, MakeRequest, Operation, Json) -> + +-spec request(aws_config(), operation(), jsx:json_term()) -> json_return(). +request(Config0, Operation, Json) -> + request(Config0, Operation, Json, []). + +-spec request(aws_config(), operation(), jsx:json_term(), erlcloud_ddb2:ddb_opts()) -> json_return(). +request(Config0, Operation, Json, DdbOpts) -> + NoRequest = proplists:get_value(no_request, DdbOpts, false), Body = case Json of [] -> <<"{}">>; _ -> jsx:encode(Json) @@ -85,7 +92,7 @@ request(Config0, MakeRequest, Operation, Json) -> case erlcloud_aws:update_config(Config0) of {ok, Config} -> Headers = headers(Config, Operation, Body), - maybe_request_and_retry(Config, Headers, Body, {attempt, 1}, MakeRequest); + maybe_request_and_retry(Config, Headers, Body, {attempt, 1}, NoRequest); {error, Reason} -> {error, Reason} end. @@ -100,10 +107,11 @@ request(Config0, MakeRequest, Operation, Json) -> %% This algorithm is similar, except that it waits a random interval up to 2^(Attempt-2)*100ms. The average %% wait time should be the same as boto. +-spec maybe_request_and_retry(aws_config(), headers(), jsx:json_text(), {attempt, non_neg_integer()}, boolean()) -> json_return(). maybe_request_and_retry(Config, Headers, Body, Attempt, false) -> request_and_retry(Config, Headers, Body, Attempt); maybe_request_and_retry(_Config, Headers, Body, _Attempt, true) -> - [Headers, Body]. + {ok, {request, Headers, Body}}. %% TODO refactor retry logic so that it can be used by all requests and move to erlcloud_aws From 90547725a0b0e7b3a17caf8a75f8693724195413 Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Thu, 10 Dec 2020 09:57:08 +0000 Subject: [PATCH 181/310] Removed call to request/4 --- src/erlcloud_ddb1.erl | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/erlcloud_ddb1.erl b/src/erlcloud_ddb1.erl index 7c49d640b..52723b02d 100644 --- a/src/erlcloud_ddb1.erl +++ b/src/erlcloud_ddb1.erl @@ -93,7 +93,7 @@ batch_get_item(RequestItems) -> -spec batch_get_item([batch_get_item_request_item()], aws_config()) -> json_return(). batch_get_item(RequestItems, Config) -> Json = [{<<"RequestItems">>, [batch_get_item_request_item_json(R) || R <- RequestItems]}], - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.BatchGetItem", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.BatchGetItem", Json). -type batch_write_item_put() :: {put, item()}. -type batch_write_item_delete() :: {delete, key()}. @@ -117,7 +117,7 @@ batch_write_item(RequestItems) -> -spec batch_write_item([batch_write_item_request_item()], aws_config()) -> json_return(). batch_write_item(RequestItems, Config) -> Json = [{<<"RequestItems">>, [batch_write_item_request_item_json(R) || R <- RequestItems]}], - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.BatchWriteItem", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.BatchWriteItem", Json). -spec key_schema_value_json(key_schema_value()) -> jsx:json_term(). @@ -141,7 +141,7 @@ create_table(Table, KeySchema, ReadUnits, WriteUnits, Config) -> key_schema_json(KeySchema), {<<"ProvisionedThroughput">>, [{<<"ReadCapacityUnits">>, ReadUnits}, {<<"WriteCapacityUnits">>, WriteUnits}]}], - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.CreateTable", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.CreateTable", Json). -spec delete_item(table_name(), key()) -> json_return(). @@ -157,7 +157,7 @@ delete_item(Table, Key, Opts, Config) -> Json = [{<<"TableName">>, Table}, key_json(Key)] ++ Opts, - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.DeleteItem", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.DeleteItem", Json). -spec delete_table(table_name()) -> json_return(). @@ -167,7 +167,7 @@ delete_table(Table) -> -spec delete_table(table_name(), aws_config()) -> json_return(). delete_table(Table, Config) -> Json = [{<<"TableName">>, Table}], - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.DeleteTable", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.DeleteTable", Json). -spec describe_table(table_name()) -> json_return(). @@ -177,7 +177,7 @@ describe_table(Table) -> -spec describe_table(table_name(), aws_config()) -> json_return(). describe_table(Table, Config) -> Json = [{<<"TableName">>, Table}], - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.DescribeTable", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.DescribeTable", Json). -spec get_item(table_name(), key()) -> json_return(). @@ -193,7 +193,7 @@ get_item(Table, Key, Opts, Config) -> Json = [{<<"TableName">>, Table}, key_json(Key)] ++ Opts, - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.GetItem", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.GetItem", Json). -spec list_tables() -> json_return(). @@ -206,7 +206,7 @@ list_tables(Opts) -> -spec list_tables(opts(), aws_config()) -> json_return(). list_tables(Opts, Config) -> - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.ListTables", Opts). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.ListTables", Opts). -spec put_item(table_name(), item()) -> json_return(). @@ -222,7 +222,7 @@ put_item(Table, Item, Opts, Config) -> Json = [{<<"TableName">>, Table}, item_json(Item)] ++ Opts, - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.PutItem", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.PutItem", Json). -spec q(table_name(), hash_key()) -> json_return(). @@ -238,7 +238,7 @@ q(Table, HashKey, Opts, Config) -> Json = [{<<"TableName">>, Table}, hash_key_json(HashKey)] ++ Opts, - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.Query", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.Query", Json). -spec scan(table_name()) -> json_return(). @@ -253,7 +253,7 @@ scan(Table, Opts) -> scan(Table, Opts, Config) -> Json = [{<<"TableName">>, Table}] ++ Opts, - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.Scan", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.Scan", Json). -spec update_item(table_name(), key(), updates()) -> json_return(). @@ -270,7 +270,7 @@ update_item(Table, Key, Updates, Opts, Config) -> key_json(Key), updates_json(Updates)] ++ Opts, - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.UpdateItem", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.UpdateItem", Json). -spec update_table(table_name(), non_neg_integer(), non_neg_integer()) -> json_return(). @@ -282,7 +282,7 @@ update_table(Table, ReadUnits, WriteUnits, Config) -> Json = [{<<"TableName">>, Table}, {<<"ProvisionedThroughput">>, [{<<"ReadCapacityUnits">>, ReadUnits}, {<<"WriteCapacityUnits">>, WriteUnits}]}], - erlcloud_ddb_impl:request(Config, false, "DynamoDB_20111205.UpdateTable", Json). + erlcloud_ddb_impl:request(Config, "DynamoDB_20111205.UpdateTable", Json). %% backoff and retry are here for backwards compat. Use the ones in erlcloud_ddb_impl instead. From 4cfa32326453ee039f814ac6624b62c522389160 Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Thu, 10 Dec 2020 09:58:14 +0000 Subject: [PATCH 182/310] set operations to call request/4 --- src/erlcloud_ddb2.erl | 183 +++++++++++++++++------------------------- 1 file changed, 73 insertions(+), 110 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 851d0fef6..c47d16c3c 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1148,6 +1148,8 @@ out({error, Reason}, _, _) -> {error, Reason}; out(ok, _, _) -> {error, unexpected_empty_response}; +out({ok, {request, _Headers, _Body}} = Request, _Undynamize, _Opts) -> + Request; out({ok, Json}, Undynamize, Opts) -> case proplists:get_value(out, Opts, simple) of json -> @@ -1158,9 +1160,7 @@ out({ok, Json}, Undynamize, Opts) -> {ok, Undynamize(Json, [{typed, true}])}; simple -> {simple, Undynamize(Json, [])} - end; -out(Request, _Undynamize, _Opts) when is_list(Request) -> - Request. + end. %% Returns specified field of tuple for simple return -spec out(erlcloud_ddb_impl:json_return(), undynamize_fun(), ddb_opts(), pos_integer()) @@ -1584,13 +1584,12 @@ batch_get_item(RequestItems, Opts) -> batch_get_item_return(). batch_get_item(RequestItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(batch_get_item_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.BatchGetItem", [{<<"RequestItems">>, dynamize_batch_get_item_request_items(RequestItems)}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), case out(Return, fun(Json, UOpts) -> undynamize_record(batch_get_item_record(), Json, UOpts) end, DdbOpts) of @@ -1711,13 +1710,12 @@ batch_write_item(RequestItems, Opts) -> batch_write_item_return(). batch_write_item(RequestItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(batch_write_item_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.BatchWriteItem", [{<<"RequestItems">>, dynamize_batch_write_item_request_items(RequestItems)}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), case out(Return, fun(Json, UOpts) -> undynamize_record(batch_write_item_record(), Json, UOpts) end, DdbOpts) of @@ -1784,13 +1782,12 @@ create_backup(BackupName, TableName, Opts) create_backup(BackupName, TableName, Opts, Config) when is_binary(BackupName), is_binary(TableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.CreateBackup", [{<<"TableName">>, TableName}, - {<<"BackupName">>, BackupName}]), + {<<"BackupName">>, BackupName}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(create_backup_record(), Json, UOpts) end, DdbOpts, #ddb2_create_backup.backup_details). @@ -1849,13 +1846,12 @@ create_global_table(GlobalTableName, ReplicationGroup, Config) when is_record(Co -> create_global_table_return(). create_global_table(GlobalTableName, ReplicationGroup, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.CreateGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}, - {<<"ReplicationGroup">>, dynamize_maybe_list(fun dynamize_replica/1, ReplicationGroup)}]), + {<<"ReplicationGroup">>, dynamize_maybe_list(fun dynamize_replica/1, ReplicationGroup)}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(create_global_table_record(), Json, UOpts) end, DdbOpts, #ddb2_create_global_table.global_table_description). @@ -1961,15 +1957,14 @@ create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits) create_table(Table, AttrDefs, KeySchema, ReadUnits, WriteUnits, [], default_config()); create_table(Table, AttrDefs, KeySchema, Opts, Config) -> {AwsOpts, DdbOpts} = opts(create_table_opts(KeySchema), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.CreateTable", [{<<"TableName">>, Table}, {<<"AttributeDefinitions">>, dynamize_attr_defs(AttrDefs)}, {<<"KeySchema">>, dynamize_key_schema(KeySchema)}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(create_table_record(), Json, UOpts) end, DdbOpts, #ddb2_create_table.table_description). @@ -2093,12 +2088,11 @@ delete_backup(BackupArn, Opts) delete_backup(BackupArn, Opts, Config) when is_binary(BackupArn) -> {_AwsOpts, DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DeleteBackup", - [{<<"BackupArn">>, BackupArn}]), + [{<<"BackupArn">>, BackupArn}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(delete_backup_record(), Json, UOpts) end, DdbOpts, #ddb2_delete_backup.backup_description). @@ -2186,14 +2180,13 @@ delete_item(Table, Key, Opts) -> -spec delete_item(table_name(), key(), delete_item_opts(), aws_config()) -> delete_item_return(). delete_item(Table, Key, Opts, Config) -> {AwsOpts, DdbOpts} = opts(delete_item_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DeleteItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(delete_item_record(), Json, UOpts) end, DdbOpts, #ddb2_delete_item.attributes, {ok, []}). @@ -2236,12 +2229,11 @@ delete_table(Table, Opts) -> -spec delete_table(table_name(), ddb_opts(), aws_config()) -> delete_table_return(). delete_table(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DeleteTable", - [{<<"TableName">>, Table}]), + [{<<"TableName">>, Table}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(delete_table_record(), Json, UOpts) end, DdbOpts, #ddb2_delete_table.table_description). @@ -2287,12 +2279,11 @@ describe_backup(BackupArn, Opts) describe_backup(BackupArn, Opts, Config) when is_binary(BackupArn) -> {_AwsOpts, DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DescribeBackup", - [{<<"BackupArn">>, BackupArn}]), + [{<<"BackupArn">>, BackupArn}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(describe_backup_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_backup.backup_description). @@ -2354,12 +2345,11 @@ describe_continuous_backups(TableName, Opts) describe_continuous_backups(TableName, Opts, Config) when is_binary(TableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DescribeContinuousBackups", - [{<<"TableName">>, TableName}]), + [{<<"TableName">>, TableName}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(continuous_backups_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_continuous_backups.continuous_backups_description). @@ -2406,12 +2396,11 @@ describe_global_table(GlobalTableName, Config) when is_record(Config, aws_config -> describe_global_table_return(). describe_global_table(GlobalTableName, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DescribeGlobalTable", - [{<<"GlobalTableName">>, GlobalTableName}]), + [{<<"GlobalTableName">>, GlobalTableName}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(describe_global_table_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_table.table). @@ -2456,12 +2445,11 @@ describe_global_table_settings(GlobalTableName, Opts) -> -spec describe_global_table_settings(table_name(), ddb_opts(), aws_config()) -> describe_global_table_settings_return(). describe_global_table_settings(GlobalTableName, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DescribeGlobalTableSettings", - [{<<"GlobalTableName">>, GlobalTableName}]), + [{<<"GlobalTableName">>, GlobalTableName}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(describe_global_table_settings_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_global_table_settings.replica_settings). @@ -2506,12 +2494,11 @@ describe_limits(Opts) -> -spec describe_limits(ddb_opts(), aws_config()) -> describe_limits_return(). describe_limits(Opts, Config) -> {[], DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DescribeLimits", - []), + [], + DdbOpts), case out(Return, fun(Json, UOpts) -> undynamize_record(describe_limits_record(), Json, UOpts) end, DdbOpts) of {simple, Record} -> {ok, Record}; @@ -2558,12 +2545,11 @@ describe_table(Table, Opts) -> -spec describe_table(table_name(), ddb_opts(), aws_config()) -> describe_table_return(). describe_table(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DescribeTable", - [{<<"TableName">>, Table}]), + [{<<"TableName">>, Table}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(describe_table_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_table.table). @@ -2607,12 +2593,11 @@ describe_table_replica_auto_scaling(Table, Opts) -> -spec describe_table_replica_auto_scaling(table_name(), ddb_opts(), aws_config()) -> describe_table_replica_auto_scaling_return(). describe_table_replica_auto_scaling(Table, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DescribeTableReplicaAutoScaling", - [{<<"TableName">>, Table}]), + [{<<"TableName">>, Table}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(describe_table_replica_auto_scaling_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_table_replica_auto_scaling.table_auto_scaling_description). @@ -2668,12 +2653,11 @@ describe_time_to_live(Table, DbOpts) -> %%------------------------------------------------------------------------------ -spec describe_time_to_live(table_name(), ddb_opts(), aws_config()) -> describe_time_to_live_return(). describe_time_to_live(Table, DbOpts, Config) -> - NoRequest = proplists:get_value(no_request, DbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.DescribeTimeToLive", - [{<<"TableName">>, Table}]), + [{<<"TableName">>, Table}], + DbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(describe_time_to_live_record(), Json, UOpts) end, DbOpts, #ddb2_describe_time_to_live.time_to_live_description). @@ -2738,14 +2722,13 @@ get_item(Table, Key, Opts) -> -spec get_item(table_name(), key(), get_item_opts(), aws_config()) -> get_item_return(). get_item(Table, Key, Opts, Config) -> {AwsOpts, DdbOpts} = opts(get_item_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.GetItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(get_item_record(), Json, UOpts) end, DdbOpts, #ddb2_get_item.item, {ok, []}). @@ -2828,12 +2811,11 @@ list_backups(Opts) -> -spec list_backups([list_backups_opt()], aws_config()) -> list_backups_return(). list_backups(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_backups_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.ListBackups", - AwsOpts), + AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_backups_record(), Json, UOpts) end, DdbOpts, #ddb2_list_backups.backup_summaries). @@ -2896,12 +2878,11 @@ list_global_tables(Config) when is_record(Config, aws_config) -> -> list_global_tables_return(). list_global_tables(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_global_tables_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.ListGlobalTables", - AwsOpts), + AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_global_tables_record(), Json, UOpts) end, DdbOpts, #ddb2_list_global_tables.global_tables, {ok, []}). @@ -2956,12 +2937,11 @@ list_tables(Opts) -> -spec list_tables(list_tables_opts(), aws_config()) -> list_tables_return(). list_tables(Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_tables_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.ListTables", - AwsOpts), + AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_tables_record(), Json, UOpts) end, DdbOpts, #ddb2_list_tables.table_names, {ok, []}). @@ -3024,12 +3004,10 @@ list_tags_of_resource(ResourceArn, Opts) -> -> list_tags_of_resource_return(). list_tags_of_resource(ResourceArn, Opts, Config) -> {AwsOpts, DdbOpts} = opts(list_tags_of_resource_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.ListTagsOfResource", - [{<<"ResourceArn">>, ResourceArn} | AwsOpts]), + [{<<"ResourceArn">>, ResourceArn} | AwsOpts], DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_tags_of_resource_record(), Json, UOpts) end, DdbOpts, #ddb2_list_tags_of_resource.tags, {ok, []}). @@ -3127,14 +3105,13 @@ put_item(Table, Item, Opts) -> -spec put_item(table_name(), in_item(), put_item_opts(), aws_config()) -> put_item_return(). put_item(Table, Item, Opts, Config) -> {AwsOpts, DdbOpts} = opts(put_item_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.PutItem", [{<<"TableName">>, Table}, {<<"Item">>, dynamize_item(Item)}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(put_item_record(), Json, UOpts) end, DdbOpts, #ddb2_put_item.attributes, {ok, []}). @@ -3240,14 +3217,14 @@ q(Table, KeyConditionsOrExpression, Opts) -> -spec q(table_name(), conditions() | expression(), q_opts(), aws_config()) -> q_return(). q(Table, KeyConditionsOrExpression, Opts, Config) -> {AwsOpts, DdbOpts} = opts(q_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), + Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.Query", [{<<"TableName">>, Table}, dynamize_q_key_conditions_or_expression(KeyConditionsOrExpression)] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(q_record(), Json, UOpts) end, DdbOpts, #ddb2_q.items, {ok, []}). @@ -3293,13 +3270,12 @@ restore_table_from_backup(BackupArn, TargetTableName, Opts) restore_table_from_backup(BackupArn, TargetTableName, Opts, Config) when is_binary(BackupArn), is_binary(TargetTableName) -> {_AwsOpts, DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.RestoreTableFromBackup", [{<<"BackupArn">>, BackupArn}, - {<<"TargetTableName">>, TargetTableName}]), + {<<"TargetTableName">>, TargetTableName}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(restore_table_from_backup_record(), Json, UOpts) end, DdbOpts, #ddb2_restore_table_from_backup.table_description). @@ -3354,13 +3330,12 @@ restore_table_to_point_in_time(SourceTableName, TargetTableName, Opts) restore_table_to_point_in_time(SourceTableName, TargetTableName, Opts, Config) when is_binary(SourceTableName), is_binary(TargetTableName) -> {AwsOpts, DdbOpts} = opts(restore_table_to_point_in_time_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.RestoreTableToPointInTime", [{<<"SourceTableName">>, SourceTableName}, - {<<"TargetTableName">>, TargetTableName}] ++ AwsOpts), + {<<"TargetTableName">>, TargetTableName}] ++ AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(restore_table_to_point_in_time(), Json, UOpts) end, DdbOpts, #ddb2_restore_table_to_point_in_time.table_description). @@ -3446,13 +3421,12 @@ scan(Table, Opts) -> -spec scan(table_name(), scan_opts(), aws_config()) -> scan_return(). scan(Table, Opts, Config) -> {AwsOpts, DdbOpts} = opts(scan_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.Scan", [{<<"TableName">>, Table}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(scan_record(), Json, UOpts) end, DdbOpts, #ddb2_scan.items, {ok, []}). @@ -3496,7 +3470,7 @@ tag_resource(ResourceArn, Tags) -> -> tag_resource_return(). tag_resource(ResourceArn, Tags, Config) -> erlcloud_ddb_impl:request( - Config, false, + Config, "DynamoDB_20120810.TagResource", [{<<"ResourceArn">>, ResourceArn}, {<<"Tags">>, dynamize_tags(Tags)}]). @@ -3594,13 +3568,12 @@ transact_get_items(RequestItems, Opts) -> transact_get_items_return(). transact_get_items(TransactItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(transact_get_items_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.TransactGetItems", [{<<"TransactItems">>, dynamize_transact_get_items_transact_items(TransactItems)}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), case out(Return, fun(Json, UOpts) -> undynamize_record(transact_get_items_record(), Json, UOpts) end, DdbOpts) of {simple, #ddb2_transact_get_items{responses = Responses}} -> @@ -3740,13 +3713,12 @@ transact_write_items(RequestItems, Opts) -> transact_write_items_return(). transact_write_items(TransactItems, Opts, Config) -> {AwsOpts, DdbOpts} = opts(transact_write_items_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.TransactWriteItems", [{<<"TransactItems">>, dynamize_transact_write_items_transact_items(TransactItems)}] - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), case out(Return, fun(Json, UOpts) -> undynamize_record(transact_write_items_record(), Json, UOpts) end, DdbOpts, #ddb2_transact_write_items.attributes, {ok, []}) of @@ -3784,7 +3756,7 @@ untag_resource(ResourceArn, TagKeys) -> -> untag_resource_return(). untag_resource(ResourceArn, TagKeys, Config) -> erlcloud_ddb_impl:request( - Config, false, + Config, "DynamoDB_20120810.UntagResource", [{<<"ResourceArn">>, ResourceArn}, {<<"TagKeys">>, TagKeys}]). @@ -3828,13 +3800,12 @@ update_continuous_backups(TableName, PointInTimeRecoveryEnabled, Opts) update_continuous_backups(TableName, PointInTimeRecoveryEnabled, Opts, Config) when is_binary(TableName), is_boolean(PointInTimeRecoveryEnabled) -> {_AwsOpts, DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.UpdateContinuousBackups", [{<<"TableName">>, TableName}, - {<<"PointInTimeRecoverySpecification">>, dynamize_point_in_time_recovery_enabled(PointInTimeRecoveryEnabled)}]), + {<<"PointInTimeRecoverySpecification">>, dynamize_point_in_time_recovery_enabled(PointInTimeRecoveryEnabled)}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(continuous_backups_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_continuous_backups.continuous_backups_description). @@ -3951,15 +3922,14 @@ update_item(Table, Key, UpdatesOrExpression, Opts) -> -> update_item_return(). update_item(Table, Key, UpdatesOrExpression, Opts, Config) -> {AwsOpts, DdbOpts} = opts(update_item_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.UpdateItem", [{<<"TableName">>, Table}, {<<"Key">>, dynamize_key(Key)}] ++ dynamize_update_item_updates_or_expression(UpdatesOrExpression) - ++ AwsOpts), + ++ AwsOpts, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_item_record(), Json, UOpts) end, DdbOpts, #ddb2_update_item.attributes, {ok, []}). @@ -4018,13 +3988,12 @@ update_global_table(GlobalTableName, ReplicaUpdates, Config) when is_record(Conf -> update_global_table_return(). update_global_table(GlobalTableName, ReplicaUpdates, Opts, Config) -> {[], DdbOpts} = opts([], Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.UpdateGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}, - {<<"ReplicaUpdates">>, dynamize_maybe_list(fun dynamize_replica_update/1, ReplicaUpdates)}]), + {<<"ReplicaUpdates">>, dynamize_maybe_list(fun dynamize_replica_update/1, ReplicaUpdates)}], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_global_table_record(), Json, UOpts) end, DdbOpts, #ddb2_update_global_table.global_table_description). @@ -4168,12 +4137,10 @@ update_global_table_settings(GlobalTableName, Opts) -> -spec update_global_table_settings(table_name(), update_global_table_settings_opts(), aws_config()) -> update_global_table_settings_return(). update_global_table_settings(GlobalTableName, Opts, Config) -> {AwsOpts, DdbOpts} = opts(update_global_table_settings_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.UpdateGlobalTableSettings", - [{<<"GlobalTableName">>, GlobalTableName} | AwsOpts]), + [{<<"GlobalTableName">>, GlobalTableName} | AwsOpts], DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_global_table_settings_record(), Json, UOpts) end, DdbOpts, #ddb2_update_global_table_settings.replica_settings). @@ -4339,12 +4306,11 @@ update_table(Table, Opts) -> (table_name(), read_units(), write_units()) -> update_table_return(). update_table(Table, Opts, Config) when is_list(Opts) -> {AwsOpts, DdbOpts} = opts(update_table_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.UpdateTable", - [{<<"TableName">>, Table} | AwsOpts]), + [{<<"TableName">>, Table} | AwsOpts], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_table_record(), Json, UOpts) end, DdbOpts, #ddb2_update_table.table_description); update_table(Table, ReadUnits, WriteUnits) -> @@ -4487,12 +4453,10 @@ update_table_replica_auto_scaling(Table, Opts, Config) when is_list(Opts) -> % Any non-default timeout already written to the config will not be overridden. UpdatedConfig = maybe_update_config_timeout(Config, 25000), {AwsOpts, DdbOpts} = opts(update_table_replica_auto_scaling_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Return = erlcloud_ddb_impl:request( UpdatedConfig, - NoRequest, "DynamoDB_20120810.UpdateTableReplicaAutoScaling", - [{<<"TableName">>, Table} | AwsOpts]), + [{<<"TableName">>, Table} | AwsOpts], DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_table_replica_auto_scaling_record(), Json, UOpts) end, DdbOpts, #ddb2_update_table_replica_auto_scaling.table_auto_scaling_description). @@ -4560,13 +4524,12 @@ update_time_to_live(Table, AttributeName, Enabled, Config) -> (table_name(), attr_name(), boolean()) -> update_time_to_live_return(). update_time_to_live(Table, Opts, Config) when is_list(Opts) -> {AwsOpts, DdbOpts} = opts(update_time_to_live_opts(), Opts), - NoRequest = proplists:get_value(no_request, DdbOpts, false), Body = [{<<"TableName">>, Table}, {<<"TimeToLiveSpecification">>, AwsOpts}], Return = erlcloud_ddb_impl:request( Config, - NoRequest, "DynamoDB_20120810.UpdateTimeToLive", - Body), + Body, + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_time_to_live_record(), Json, UOpts) end, DdbOpts, #ddb2_update_time_to_live.time_to_live_specification); From db583828a944eb3445e8e6f5efb1f628fc2440a7 Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Thu, 10 Dec 2020 16:54:52 +0000 Subject: [PATCH 183/310] Fix indentation --- src/erlcloud_ddb2.erl | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index c47d16c3c..6811576f4 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1851,7 +1851,7 @@ create_global_table(GlobalTableName, ReplicationGroup, Opts, Config) -> "DynamoDB_20120810.CreateGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}, {<<"ReplicationGroup">>, dynamize_maybe_list(fun dynamize_replica/1, ReplicationGroup)}], - DdbOpts), + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(create_global_table_record(), Json, UOpts) end, DdbOpts, #ddb2_create_global_table.global_table_description). @@ -3007,7 +3007,8 @@ list_tags_of_resource(ResourceArn, Opts, Config) -> Return = erlcloud_ddb_impl:request( Config, "DynamoDB_20120810.ListTagsOfResource", - [{<<"ResourceArn">>, ResourceArn} | AwsOpts], DdbOpts), + [{<<"ResourceArn">>, ResourceArn} | AwsOpts], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(list_tags_of_resource_record(), Json, UOpts) end, DdbOpts, #ddb2_list_tags_of_resource.tags, {ok, []}). @@ -3275,7 +3276,7 @@ restore_table_from_backup(BackupArn, TargetTableName, Opts, Config) "DynamoDB_20120810.RestoreTableFromBackup", [{<<"BackupArn">>, BackupArn}, {<<"TargetTableName">>, TargetTableName}], - DdbOpts), + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(restore_table_from_backup_record(), Json, UOpts) end, DdbOpts, #ddb2_restore_table_from_backup.table_description). @@ -3805,7 +3806,7 @@ update_continuous_backups(TableName, PointInTimeRecoveryEnabled, Opts, Config) "DynamoDB_20120810.UpdateContinuousBackups", [{<<"TableName">>, TableName}, {<<"PointInTimeRecoverySpecification">>, dynamize_point_in_time_recovery_enabled(PointInTimeRecoveryEnabled)}], - DdbOpts), + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(continuous_backups_record(), Json, UOpts) end, DdbOpts, #ddb2_describe_continuous_backups.continuous_backups_description). @@ -3993,7 +3994,7 @@ update_global_table(GlobalTableName, ReplicaUpdates, Opts, Config) -> "DynamoDB_20120810.UpdateGlobalTable", [{<<"GlobalTableName">>, GlobalTableName}, {<<"ReplicaUpdates">>, dynamize_maybe_list(fun dynamize_replica_update/1, ReplicaUpdates)}], - DdbOpts), + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_global_table_record(), Json, UOpts) end, DdbOpts, #ddb2_update_global_table.global_table_description). @@ -4140,7 +4141,8 @@ update_global_table_settings(GlobalTableName, Opts, Config) -> Return = erlcloud_ddb_impl:request( Config, "DynamoDB_20120810.UpdateGlobalTableSettings", - [{<<"GlobalTableName">>, GlobalTableName} | AwsOpts], DdbOpts), + [{<<"GlobalTableName">>, GlobalTableName} | AwsOpts], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_global_table_settings_record(), Json, UOpts) end, DdbOpts, #ddb2_update_global_table_settings.replica_settings). @@ -4456,7 +4458,8 @@ update_table_replica_auto_scaling(Table, Opts, Config) when is_list(Opts) -> Return = erlcloud_ddb_impl:request( UpdatedConfig, "DynamoDB_20120810.UpdateTableReplicaAutoScaling", - [{<<"TableName">>, Table} | AwsOpts], DdbOpts), + [{<<"TableName">>, Table} | AwsOpts], + DdbOpts), out(Return, fun(Json, UOpts) -> undynamize_record(update_table_replica_auto_scaling_record(), Json, UOpts) end, DdbOpts, #ddb2_update_table_replica_auto_scaling.table_auto_scaling_description). From 871110491aa691e08e628ef833d4b7c83fd4ed2b Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Sat, 12 Dec 2020 11:22:18 +0000 Subject: [PATCH 184/310] Add ddb_request record --- include/erlcloud_ddb2.hrl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index e1c0c986a..5b49cced2 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -447,3 +447,9 @@ {table_description :: undefined | #ddb2_table_description{} }). -endif. + +-record(ddb_request, + {headers :: erlcloud_ddb_impl:headers(), + body :: jsx:json_text(), + json :: jsx:json_term() + }). From e5ed7814c63c22228f97c676f638f0e23f403cd6 Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Sat, 12 Dec 2020 11:23:34 +0000 Subject: [PATCH 185/310] Added ddb_request record as return type --- src/erlcloud_ddb2.erl | 4 ++-- src/erlcloud_ddb_impl.erl | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 6811576f4..340012899 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1141,14 +1141,14 @@ return_item_collection_metrics_opt() -> -type undynamize_fun() :: fun((jsx:json_term(), undynamize_opts()) -> tuple()). -spec out(erlcloud_ddb_impl:json_return(), undynamize_fun(), ddb_opts()) - -> {ok, jsx:json_term() | tuple()} | + -> {ok, jsx:json_term() | tuple() | #ddb_request{}} | {simple, term()} | {error, term()}. out({error, Reason}, _, _) -> {error, Reason}; out(ok, _, _) -> {error, unexpected_empty_response}; -out({ok, {request, _Headers, _Body}} = Request, _Undynamize, _Opts) -> +out({ok, #ddb_request{}} = Request, _Undynamize, _Opts) -> Request; out({ok, Json}, Undynamize, Opts) -> case proplists:get_value(out, Opts, simple) of diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 56278dba7..629e1cf40 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -71,9 +71,9 @@ -export([request/3, request/4]). --export_type([json_return/0, attempt/0, retry_fun/0]). +-export_type([json_return/0, attempt/0, retry_fun/0, headers/0]). --type json_return() :: ok | {ok, jsx:json_term()} | {error, term()} | {ok, {request, headers(), jsx:json_text()}}. +-type json_return() :: ok | {ok, jsx:json_term()} | {error, term()} | {ok, #ddb_request{}}. -type operation() :: string(). @@ -92,7 +92,7 @@ request(Config0, Operation, Json, DdbOpts) -> case erlcloud_aws:update_config(Config0) of {ok, Config} -> Headers = headers(Config, Operation, Body), - maybe_request_and_retry(Config, Headers, Body, {attempt, 1}, NoRequest); + maybe_request_and_retry(Config, Headers, Body, Json, {attempt, 1}, NoRequest); {error, Reason} -> {error, Reason} end. @@ -107,11 +107,11 @@ request(Config0, Operation, Json, DdbOpts) -> %% This algorithm is similar, except that it waits a random interval up to 2^(Attempt-2)*100ms. The average %% wait time should be the same as boto. --spec maybe_request_and_retry(aws_config(), headers(), jsx:json_text(), {attempt, non_neg_integer()}, boolean()) -> json_return(). -maybe_request_and_retry(Config, Headers, Body, Attempt, false) -> +-spec maybe_request_and_retry(aws_config(), headers(), jsx:json_text(), jsx:json_term(), {attempt, non_neg_integer()}, boolean()) -> json_return(). +maybe_request_and_retry(Config, Headers, Body, _Json, Attempt, false) -> request_and_retry(Config, Headers, Body, Attempt); -maybe_request_and_retry(_Config, Headers, Body, _Attempt, true) -> - {ok, {request, Headers, Body}}. +maybe_request_and_retry(_Config, Headers, Body, Json, _Attempt, true) -> + {ok, #ddb_request{headers = Headers, body = Body, json = Json}}. %% TODO refactor retry logic so that it can be used by all requests and move to erlcloud_aws From f60a6bba62ad018cc705d589a4da3248425af86d Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Tue, 15 Dec 2020 23:28:05 +0000 Subject: [PATCH 186/310] Updated record position and name and added headers() type specification --- include/erlcloud_ddb2.hrl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index 5b49cced2..91dcd4fd9 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -14,6 +14,13 @@ response_body :: undefined | binary() }). +-record(ddb2_request, + {headers :: headers(), + body :: jsx:json_text(), + json :: jsx:json_term() + }). + +-type headers() :: [{string(), string()}]. -type date_time() :: number(). -type global_table_status() :: creating | active | deleting | updating. -type replica_status() :: creating | creation_failed | updating | deleting | active. @@ -447,9 +454,3 @@ {table_description :: undefined | #ddb2_table_description{} }). -endif. - --record(ddb_request, - {headers :: erlcloud_ddb_impl:headers(), - body :: jsx:json_text(), - json :: jsx:json_term() - }). From 23960fcdb23f80b4ac3912d01510d727fa1e783d Mon Sep 17 00:00:00 2001 From: Diogo Almirante Date: Tue, 15 Dec 2020 23:30:04 +0000 Subject: [PATCH 187/310] Updated record name to and removed mistakenly added whitespace from ddb2 --- src/erlcloud_ddb2.erl | 5 ++--- src/erlcloud_ddb_impl.erl | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 340012899..a6d4177c5 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1141,14 +1141,14 @@ return_item_collection_metrics_opt() -> -type undynamize_fun() :: fun((jsx:json_term(), undynamize_opts()) -> tuple()). -spec out(erlcloud_ddb_impl:json_return(), undynamize_fun(), ddb_opts()) - -> {ok, jsx:json_term() | tuple() | #ddb_request{}} | + -> {ok, jsx:json_term() | tuple() | #ddb2_request{}} | {simple, term()} | {error, term()}. out({error, Reason}, _, _) -> {error, Reason}; out(ok, _, _) -> {error, unexpected_empty_response}; -out({ok, #ddb_request{}} = Request, _Undynamize, _Opts) -> +out({ok, #ddb2_request{}} = Request, _Undynamize, _Opts) -> Request; out({ok, Json}, Undynamize, Opts) -> case proplists:get_value(out, Opts, simple) of @@ -3218,7 +3218,6 @@ q(Table, KeyConditionsOrExpression, Opts) -> -spec q(table_name(), conditions() | expression(), q_opts(), aws_config()) -> q_return(). q(Table, KeyConditionsOrExpression, Opts, Config) -> {AwsOpts, DdbOpts} = opts(q_opts(), Opts), - Return = erlcloud_ddb_impl:request( Config, "DynamoDB_20120810.Query", diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 629e1cf40..a938fdde7 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -71,9 +71,9 @@ -export([request/3, request/4]). --export_type([json_return/0, attempt/0, retry_fun/0, headers/0]). +-export_type([json_return/0, attempt/0, retry_fun/0]). --type json_return() :: ok | {ok, jsx:json_term()} | {error, term()} | {ok, #ddb_request{}}. +-type json_return() :: ok | {ok, jsx:json_term()} | {error, term()} | {ok, #ddb2_request{}}. -type operation() :: string(). @@ -111,7 +111,7 @@ request(Config0, Operation, Json, DdbOpts) -> maybe_request_and_retry(Config, Headers, Body, _Json, Attempt, false) -> request_and_retry(Config, Headers, Body, Attempt); maybe_request_and_retry(_Config, Headers, Body, Json, _Attempt, true) -> - {ok, #ddb_request{headers = Headers, body = Body, json = Json}}. + {ok, #ddb2_request{headers = Headers, body = Body, json = Json}}. %% TODO refactor retry logic so that it can be used by all requests and move to erlcloud_aws @@ -190,7 +190,6 @@ retry_v1_wrap(#ddb2_error{should_retry = false} = Error, _) -> retry_v1_wrap(Error, RetryFun) -> RetryFun(Error#ddb2_error.attempt, Error#ddb2_error.reason). --type headers() :: [{string(), string()}]. -spec request_and_retry(aws_config(), headers(), jsx:json_text(), attempt()) -> ok | {ok, jsx:json_term()} | {error, term()}. request_and_retry(_, _, _, {error, Reason}) -> From 4359fa93710b3a1e2e91148a55014bf997d852e7 Mon Sep 17 00:00:00 2001 From: kkuzmin Date: Thu, 17 Dec 2020 17:42:35 +0000 Subject: [PATCH 188/310] Add Ecs:ListTagsForResources support (#679) * Add Ecs:ListTagsForResources support --- src/erlcloud_ecs.erl | 46 ++++++++++++++++++++++++++++ test/erlcloud_ecs_tests.erl | 60 +++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/src/erlcloud_ecs.erl b/src/erlcloud_ecs.erl index 9630a9304..a8549b300 100644 --- a/src/erlcloud_ecs.erl +++ b/src/erlcloud_ecs.erl @@ -62,6 +62,7 @@ list_clusters/0, list_clusters/1, list_clusters/2, list_container_instances/0, list_container_instances/1, list_container_instances/2, list_services/0, list_services/1, list_services/2, + list_tags_for_resource/1, list_tags_for_resource/2, list_tags_for_resource/3, list_task_definition_families/0, list_task_definition_families/1, list_task_definition_families/2, list_task_definitions/0, list_task_definitions/1, list_task_definitions/2, list_tasks/0, list_tasks/1, list_tasks/2, @@ -1990,6 +1991,51 @@ list_services(Opts, Config) -> out(Return, fun(Json, UOpts) -> decode_record(list_services_record(), Json, UOpts) end, EcsOpts). +%%%------------------------------------------------------------------------------ +%% ListTagsForResource +%%%------------------------------------------------------------------------------ +-spec list_tags_for_resource( + Arn :: string_param()) -> ecs_return([#ecs_tag{}]). +list_tags_for_resource(Arn) -> + list_tags_for_resource(Arn, [], default_config()). + +-spec list_tags_for_resource( + Arn :: string_param(), + Config :: aws_config()) -> ecs_return([#ecs_tag{}]). +list_tags_for_resource(Arn, Config) when is_record(Config, aws_config) -> + list_tags_for_resource(Arn, [], default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% ECS API +%% [https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ListTagsForResource.html] +%% +%% ===Example=== +%% +%% List tags for a given resource +%% +%% ` +%% {ok, Result} = erlcloud_ecs:list_tags_for_resource("resource-arn"), +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec list_tags_for_resource( + Arn :: string_param(), + Opts :: proplist(), + Config :: aws_config()) -> ecs_return([#ecs_tag{}]). +list_tags_for_resource(Arn, Opts, #aws_config{} = Config) -> + {AwsOpts, EcsOpts} = opts([], Opts), + Return = ecs_request( + Config, + "ListTagsForResource", + [{<<"resourceArn">>, to_binary(Arn)}] ++ AwsOpts), + out(Return, fun(Json, UOpts) -> + TagsList = proplists:get_value(<<"tags">>, Json), + decode_tags_list(TagsList, UOpts) + end, + EcsOpts). + + %%%------------------------------------------------------------------------------ %% ListTaskDefinitionFamilies %%%------------------------------------------------------------------------------ diff --git a/test/erlcloud_ecs_tests.erl b/test/erlcloud_ecs_tests.erl index 83e8dd0ec..a12e592d2 100644 --- a/test/erlcloud_ecs_tests.erl +++ b/test/erlcloud_ecs_tests.erl @@ -64,6 +64,8 @@ operation_test_() -> fun list_container_instances_output_tests/1, fun list_services_input_tests/1, fun list_services_output_tests/1, + fun list_tags_for_resource_input_tests/1, + fun list_tags_for_resource_output_tests/1, fun list_task_definition_families_input_tests/1, fun list_task_definition_families_output_tests/1, fun list_task_definitions_input_tests/1, @@ -2005,6 +2007,64 @@ list_services_output_tests(_) -> ], output_tests(?_f(erlcloud_ecs:list_services([{out, record}])), Tests). +%% ListTagsForResource test based on the API examples: +%% https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_ListTagsForResource.html +list_tags_for_resource_input_tests(_) -> + Tests = + [?_ecs_test( + {"ListTagsForResource example request", + ?_f(erlcloud_ecs:list_tags_for_resource("testArn")), " +{ + \"resourceArn\": \"testArn\" +} +" + }) + ], + Response = " +{ + \"tags\": [ + { + \"key\": \"test-key1\", + \"value\": \"test-value1\" + }, + { + \"key\": \"test-key2\", + \"value\": \"test-value2\" + } + ] +} +", + input_tests(Response, Tests). + +list_tags_for_resource_output_tests(_) -> + Tests = + [?_ecs_test( + {"ListTagsForResources example response", " +{ + \"tags\": [ + { + \"key\": \"test-key1\", + \"value\": \"test-value1\" + }, + { + \"key\": \"test-key2\", + \"value\": \"test-value2\" + } + ] +} +", + {ok, [#ecs_tag{ + key = <<"test-key1">>, + value = <<"test-value1">> + }, + #ecs_tag{ + key = <<"test-key2">>, + value = <<"test-value2">> + } + ]}}) + ], + output_tests(?_f(erlcloud_ecs:list_tags_for_resource("testArn")), Tests). + %% ListTaskDefinitionFamilies test based on the API examples: %% http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ListTaskDefinitionFamilies.html list_task_definition_families_input_tests(_) -> From c5b3dda16b6bd76934d51375344634eefc575c5f Mon Sep 17 00:00:00 2001 From: Carl Isom III Date: Thu, 17 Dec 2020 13:41:04 -0600 Subject: [PATCH 189/310] Rename ddb2 headers to ddb2_req_headers --- include/erlcloud_ddb2.hrl | 4 ++-- src/erlcloud_ddb_impl.erl | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index 91dcd4fd9..390971539 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -15,12 +15,12 @@ }). -record(ddb2_request, - {headers :: headers(), + {headers :: ddb2_req_headers(), body :: jsx:json_text(), json :: jsx:json_term() }). --type headers() :: [{string(), string()}]. +-type ddb2_req_headers() :: [{string(), string()}]. -type date_time() :: number(). -type global_table_status() :: creating | active | deleting | updating. -type replica_status() :: creating | creation_failed | updating | deleting | active. diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index a938fdde7..b8d66ab4f 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -107,7 +107,7 @@ request(Config0, Operation, Json, DdbOpts) -> %% This algorithm is similar, except that it waits a random interval up to 2^(Attempt-2)*100ms. The average %% wait time should be the same as boto. --spec maybe_request_and_retry(aws_config(), headers(), jsx:json_text(), jsx:json_term(), {attempt, non_neg_integer()}, boolean()) -> json_return(). +-spec maybe_request_and_retry(aws_config(), ddb2_req_headers(), jsx:json_text(), jsx:json_term(), {attempt, non_neg_integer()}, boolean()) -> json_return(). maybe_request_and_retry(Config, Headers, Body, _Json, Attempt, false) -> request_and_retry(Config, Headers, Body, Attempt); maybe_request_and_retry(_Config, Headers, Body, Json, _Attempt, true) -> @@ -190,7 +190,7 @@ retry_v1_wrap(#ddb2_error{should_retry = false} = Error, _) -> retry_v1_wrap(Error, RetryFun) -> RetryFun(Error#ddb2_error.attempt, Error#ddb2_error.reason). --spec request_and_retry(aws_config(), headers(), jsx:json_text(), attempt()) -> +-spec request_and_retry(aws_config(), ddb2_req_headers(), jsx:json_text(), attempt()) -> ok | {ok, jsx:json_term()} | {error, term()}. request_and_retry(_, _, _, {error, Reason}) -> {error, Reason}; @@ -274,7 +274,7 @@ client_error(Body, DDBError) -> end end. --spec headers(aws_config(), string(), binary()) -> headers(). +-spec headers(aws_config(), string(), binary()) -> ddb2_req_headers(). headers(Config, Operation, Body) -> Headers = [{"host", Config#aws_config.ddb_host}, {"x-amz-target", Operation}], From 4ea0491b318553567bd12465849d29167e26e247 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Tue, 22 Dec 2020 15:15:10 +0000 Subject: [PATCH 190/310] Add include option into ECS:DescribeTasks requests --- src/erlcloud_ecs.erl | 3 ++- test/erlcloud_ecs_tests.erl | 10 ++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_ecs.erl b/src/erlcloud_ecs.erl index a8549b300..f385f7134 100644 --- a/src/erlcloud_ecs.erl +++ b/src/erlcloud_ecs.erl @@ -1781,7 +1781,8 @@ describe_task_definition(TaskDefinition, Opts, #aws_config{} = Config) -> -spec describe_tasks_opts() -> opt_table(). describe_tasks_opts() -> [ - {cluster, <<"cluster">>, fun to_binary/1} + {cluster, <<"cluster">>, fun to_binary/1}, + {include, <<"include">>, fun to_binary/1} ]. -spec describe_tasks_record() -> record_desc(). diff --git a/test/erlcloud_ecs_tests.erl b/test/erlcloud_ecs_tests.erl index a12e592d2..51adf679c 100644 --- a/test/erlcloud_ecs_tests.erl +++ b/test/erlcloud_ecs_tests.erl @@ -1615,20 +1615,26 @@ describe_tasks_input_tests(_) -> Tests = [?_ecs_test( {"DescribeTasks example request", - ?_f(erlcloud_ecs:describe_tasks(["c09f0188-7f87-4b0f-bfc3-16296622b6fe"])), " + ?_f(erlcloud_ecs:describe_tasks(["c09f0188-7f87-4b0f-bfc3-16296622b6fe"], [{include, ["TAGS"]}])), " { \"tasks\": [ \"c09f0188-7f87-4b0f-bfc3-16296622b6fe\" + ], + \"include\": [ + \"TAGS\" ] } " }), ?_ecs_test( {"DescribeTasks binary input example request", - ?_f(erlcloud_ecs:describe_tasks([<<"c09f0188-7f87-4b0f-bfc3-16296622b6fe">>])), " + ?_f(erlcloud_ecs:describe_tasks([<<"c09f0188-7f87-4b0f-bfc3-16296622b6fe">>], [{include, [<<"TAGS">>]}])), " { \"tasks\": [ \"c09f0188-7f87-4b0f-bfc3-16296622b6fe\" + ], + \"include\": [ + \"TAGS\" ] } " From 01d62bf9246e621c7c3140ace7373cebfd37fd4d Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Tue, 22 Dec 2020 15:38:10 +0000 Subject: [PATCH 191/310] Fix spec --- src/erlcloud_ecs.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_ecs.erl b/src/erlcloud_ecs.erl index f385f7134..920171ff9 100644 --- a/src/erlcloud_ecs.erl +++ b/src/erlcloud_ecs.erl @@ -1775,7 +1775,9 @@ describe_task_definition(TaskDefinition, Opts, #aws_config{} = Config) -> %%%------------------------------------------------------------------------------ %% DescribeTasks %%%------------------------------------------------------------------------------ --type describe_tasks_opt() :: {cluster, string_param()} | out_opt(). +-type describe_tasks_opt() :: {cluster, string_param()} | + {include, list(string_param())} | + out_opt(). -type describe_tasks_opts() :: [describe_tasks_opt()]. -spec describe_tasks_opts() -> opt_table(). From 1cd1f343a56923beefea0a5943198a3fc86bec22 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 12 Jan 2021 21:06:37 +0000 Subject: [PATCH 192/310] Expose erlcloud_cloudwatch_logs:start_query/stop_query/get_query_results --- src/erlcloud_cloudwatch_logs.erl | 196 +++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 9a74bdd2c..192c94b75 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -48,6 +48,22 @@ | erlcloud_aws:httpc_result_error()} | {ok, jsx:json_term()}. +-type query_status() :: cancelled + | complete + | failed + | running + | scheduled + | timeout + | unknown. +-export_type([query_status/0]). + +-type query_results() :: #{ results := [#{ field := string(), + value := string() }], + statistics := #{ bytes_scanned := float(), + records_matched := float(), + records_scanned := float() }, + status := query_status() }. +-export_type([query_results/0]). %% Library initialization -export([ @@ -95,12 +111,21 @@ describe_metric_filters/6, describe_metric_filters/7, + get_query_results/1, + get_query_results/2, + put_logs_events/4, put_logs_events/5, list_tags_log_group/1, list_tags_log_group/2, + start_query/4, + start_query/5, + start_query/6, + stop_query/1, + stop_query/2, + tag_log_group/2, tag_log_group/3 @@ -437,6 +462,78 @@ log_stream_order_by(undefined) -> <<"LogStreamName">>; log_stream_order_by(log_stream_name) -> <<"LogStreamName">>; log_stream_order_by(last_event_time) -> <<"LastEventTime">>. +%%------------------------------------------------------------------------------ +%% @doc +%% +%% GetQueryResults action +%% https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_GetQueryResults.html +%% +%% ===Example=== +%% +%% Returns the results from the specified query. +%% +%% ` +%% application:ensure_all_started(erlcloud). +%% {ok, Config} = erlcloud_aws:auto_config(). +%% {ok, Results} = erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab"). +%% ` +%% +%% @end +%%------------------------------------------------------------------------------ + +-spec get_query_results(QueryId) -> Results + when QueryId :: string(), + Results :: {ok, query_results()} | {error, erlcloud_aws:httpc_result_error()}. +get_query_results(QueryId) -> + get_query_results(QueryId, default_config()). + +-spec get_query_results(QueryId, Config) -> Results + when QueryId :: string(), + Config :: aws_config(), + Results :: {ok, query_results()} | {error, erlcloud_aws:httpc_result_error()}. +get_query_results(QueryId, Config) -> + Result0 = cw_request(Config, "GetQueryResults", [{<<"queryId">>, QueryId}]), + case Result0 of + {error, _} = E -> E; + {ok, Result} -> + Statistics = proplists:get_value(<<"statistics">>, Result), + #{ results => results_from_get_query_results(proplists:get_value(<<"results">>, Result)), + statistics => #{ bytes_scanned => proplists:get_value(<<"bytesScanned">>, Statistics), + records_matched => proplists:get_value(<<"recordsMatched">>, Statistics), + records_scanned => proplists:get_value(<<"recordsScanned">>, Statistics) + }, + status => status_from_get_query_results(proplists:get_value(<<"status">>, Result))} + end. + +-spec results_from_get_query_results(In) -> Out + when In :: [[{binary(), binary()}]], + Out :: [#{ field := string(), + value := string() }]. +results_from_get_query_results(In) -> + lists:map(fun (Result) -> + #{ field => proplists:get_value(<<"field">>, Result), + value => proplists:get_value(<<"value">>, Result) } + end, + In). + +-spec status_from_get_query_results(In) -> Out + when In :: binary(), + Out :: query_status(). +status_from_get_query_results(<<"Cancelled">>) -> + cancelled; +status_from_get_query_results(<<"Complete">>) -> + complete; +status_from_get_query_results(<<"Failed">>) -> + failed; +status_from_get_query_results(<<"Running">>) -> + running; +status_from_get_query_results(<<"Scheduled">>) -> + scheduled; +status_from_get_query_results(<<"Timeout">>) -> + timeout; +status_from_get_query_results(<<"Unknown">>) -> + unknown. + %%------------------------------------------------------------------------------ %% @doc %% @@ -628,6 +725,105 @@ list_tags_log_group(LogGroup, Config) -> Error end. +%%------------------------------------------------------------------------------ +%% @doc +%% +%% StartQuery action +%% https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_StartQuery.html +%% +%% ===Example=== +%% +%% Schedules a query of log groups using CloudWatch Logs Insights. You specify the log groups +%% and time range to query, as well as the query string to use. +%% +%% ` +%% {ok, _} = application:ensure_all_started(erlcloud). +%% {ok, Config} = erlcloud_aws:auto_config(). +%% {ok, #{ query_id := QueryId }} = erlcloud_cloudwatch_logs:start_query(["LogGroupName1", "LogGroupName2", "LogGroupName3"], "stats count(*) by eventSource, eventName, awsRegion", 1546300800, 1546309800, 100). +%% ` +%% +%% @end +%%------------------------------------------------------------------------------ + +-spec start_query(LogGroupNames, QueryString, StartTime, EndTime) -> Result + when LogGroupNames :: [log_group_name()], + QueryString :: string(), + StartTime :: non_neg_integer(), + EndTime :: non_neg_integer(), + Result :: {ok, QueryId :: string()} | {error, erlcloud_aws:httpc_result_error()}. +start_query(LogGroupNames0, QueryString, StartTime, EndTime) -> + start_query(LogGroupNames0, QueryString, StartTime, EndTime, _Limit = 1000). + +-spec start_query(LogGroupNames, QueryString, StartTime, EndTime, Limit) -> Result + when LogGroupNames :: [log_group_name()], + QueryString :: string(), + StartTime :: non_neg_integer(), + EndTime :: non_neg_integer(), + Limit :: 1..10000, + Result :: {ok, QueryId :: string()} | {error, erlcloud_aws:httpc_result_error()}. +start_query(LogGroupNames0, QueryString, StartTime, EndTime, Limit) -> + start_query(LogGroupNames0, QueryString, StartTime, EndTime, Limit, default_config()). + +-spec start_query(LogGroupNames, QueryString, StartTime, EndTime, Limit, Config) -> Result + when LogGroupNames :: [log_group_name()], + QueryString :: string(), + StartTime :: non_neg_integer(), + EndTime :: non_neg_integer(), + Limit :: 1..10000, + Config :: aws_config(), + Result :: {ok, #{ query_id => string() }} | {error, erlcloud_aws:httpc_result_error()}. +start_query(LogGroupNames0, QueryString, StartTime, EndTime, Limit, Config) -> + LogGroupNames = case LogGroupNames0 of + [LogGroupName] -> + [{<<"logGroupName">>, LogGroupName}]; + _ -> + [{<<"logGroupNames">>, LogGroupNames0}] + end, + Result = cw_request(Config, "StartQuery", LogGroupNames ++ [{<<"queryString">>, QueryString}, + {<<"startTime">>, StartTime}, + {<<"endTime">>, EndTime}, + {<<"limit">>, Limit}]), + case Result of + {error, _} = E -> E; + {ok, OK} -> {ok, #{ query_id => binary_to_list(proplists:get_value(<<"queryId">>, OK)) }} + end. + +%%------------------------------------------------------------------------------ +%% @doc +%% +%% StopQuery action +%% https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_StopQuery.html +%% +%% ===Example=== +%% +%% Stops a CloudWatch Logs Insights query that is in progress. +%% +%% ` +%% {ok, _} = application:ensure_all_started(erlcloud). +%% {ok, Config} = erlcloud_aws:auto_config(). +%% {ok, QueryId} = erlcloud_cloudwatch_logs:stop_query("ecef5848-8aa7-4c12-9665-bafe422f3247"). +%% ` +%% +%% @end +%%------------------------------------------------------------------------------ + +-spec stop_query(QueryId) -> Result + when QueryId :: string(), + Result :: ok | {error, erlcloud_aws:httpc_result_error()}. +stop_query(QueryId) -> + stop_query(QueryId, default_config()). + +-spec stop_query(QueryId, Config) -> Result + when QueryId :: string(), + Config :: aws_config(), + Result :: ok. +stop_query(QueryId, Config) -> + Result = cw_request(Config, "StopQuery", [{<<"queryId">>, QueryId}]), + case Result of + {error, _} = E -> E; + {ok, _} -> ok + end. + %%------------------------------------------------------------------------------ %% @doc %% From 9ff2af66eabdad6435be7f0553d1b72b44cf0160 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 12 Jan 2021 21:39:11 +0000 Subject: [PATCH 193/310] Add some tests and fix code according to these --- src/erlcloud_cloudwatch_logs.erl | 9 +++-- test/erlcloud_cloudwatch_logs_tests.erl | 51 ++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 192c94b75..6df0995d3 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -123,6 +123,7 @@ start_query/4, start_query/5, start_query/6, + stop_query/1, stop_query/2, @@ -511,8 +512,8 @@ get_query_results(QueryId, Config) -> value := string() }]. results_from_get_query_results(In) -> lists:map(fun (Result) -> - #{ field => proplists:get_value(<<"field">>, Result), - value => proplists:get_value(<<"value">>, Result) } + #{ field => binary_to_list(proplists:get_value(<<"field">>, Result)), + value => binary_to_list(proplists:get_value(<<"value">>, Result)) } end, In). @@ -750,7 +751,7 @@ list_tags_log_group(LogGroup, Config) -> QueryString :: string(), StartTime :: non_neg_integer(), EndTime :: non_neg_integer(), - Result :: {ok, QueryId :: string()} | {error, erlcloud_aws:httpc_result_error()}. + Result :: {ok, #{ query_id => string() }} | {error, erlcloud_aws:httpc_result_error()}. start_query(LogGroupNames0, QueryString, StartTime, EndTime) -> start_query(LogGroupNames0, QueryString, StartTime, EndTime, _Limit = 1000). @@ -760,7 +761,7 @@ start_query(LogGroupNames0, QueryString, StartTime, EndTime) -> StartTime :: non_neg_integer(), EndTime :: non_neg_integer(), Limit :: 1..10000, - Result :: {ok, QueryId :: string()} | {error, erlcloud_aws:httpc_result_error()}. + Result :: {ok, #{ query_id => string() }} | {error, erlcloud_aws:httpc_result_error()}. start_query(LogGroupNames0, QueryString, StartTime, EndTime, Limit) -> start_query(LogGroupNames0, QueryString, StartTime, EndTime, Limit, default_config()). diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index 6599842d2..92de743ea 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -106,7 +106,11 @@ erlcloud_cloudwatch_test_() -> fun describe_log_streams_input_tests/1, fun describe_log_streams_output_tests/1, - fun put_logs_events_input_tests/1 + fun put_logs_events_input_tests/1, + + fun start_query_output_tests/1, + fun stop_query_output_tests/1, + fun get_query_results_output_tests/1 ]}. @@ -576,6 +580,51 @@ put_logs_events_input_tests(_) -> ) ]). +start_query_output_tests(_) -> + output_tests(?_f(erlcloud_cloudwatch_logs:start_query(["LogGroupName1", "LogGroupName2", "LogGroupName3"], + "stats count(*) by eventSource, eventName, awsRegion", + 1546300800, + 1546309800, + 100)), [ + ?_cloudwatch_test( + {"Tests output format for start_query", + jsx:encode([{<<"queryId">>, <<"12ab3456-12ab-123a-789e-1234567890ab">>}]), + {ok, #{ query_id => "12ab3456-12ab-123a-789e-1234567890ab" }}} + ) + ]). + +stop_query_output_tests(_) -> + output_tests(?_f(erlcloud_cloudwatch_logs:stop_query("12ab3456-12ab-123a-789e-1234567890ab")), [ + ?_cloudwatch_test( + {"Tests output format for stop_query", + jsx:encode([{<<"success">>, true}]), + ok} + ) + ]). + +get_query_results_output_tests(_) -> + output_tests(?_f(erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab")), [ + ?_cloudwatch_test( + {"Tests output format for get_query_results", + jsx:encode([{<<"results">>, [[{<<"field">>, <<"LogEvent1-field1-name">>}, + {<<"value">>, <<"LogEvent1-field1-value">>}], + [{<<"field">>, <<"LogEvent1-field2-name">>}, + {<<"value">>, <<"LogEvent1-field2-value">>}]]}, + {<<"statistics">>, [{<<"bytesScanned">>, 81349723.0}, + {<<"recordsMatched">>, 360851.0}, + {<<"recordsScanned">>, 610956.0}]}, + {<<"status">>, <<"Complete">>}]), + #{ results => [#{ field => "LogEvent1-field1-name", + value => "LogEvent1-field1-value" }, + #{ field => "LogEvent1-field2-name", + value => "LogEvent1-field2-value" }], + statistics => #{ bytes_scanned => 81349723.0, + records_matched => 360851.0, + records_scanned => 610956.0 }, + status => complete }} + ) + ]). + %%============================================================================== %% Internal functions %%============================================================================== From c58e7c53d2c5382b0901c517d6f77347da25ab1b Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 13 Jan 2021 12:00:55 +0000 Subject: [PATCH 194/310] Ease consumption of type erlcloud_cloudwatch_logs:events/0 --- src/erlcloud_cloudwatch_logs.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 6df0995d3..655aa67fd 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -30,6 +30,7 @@ -type metric_namespace() :: string() | binary() | undefined. -type log_stream_order() :: log_stream_name | last_event_time | undefined. -type events() :: [#{message => binary(), timestamp => pos_integer()}]. +-export_type([events/0]). -type kms_key_id() :: string() | binary() | undefined. From 5e03c8d9ea40511d560aa7151bf37fa07fa6e904 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 13 Jan 2021 13:41:45 +0000 Subject: [PATCH 195/310] Fix as per integration testing --- src/erlcloud_cloudwatch_logs.erl | 12 ++++++------ test/erlcloud_cloudwatch_logs_tests.erl | 16 ++++++++-------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 655aa67fd..89cb2f4dc 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -499,12 +499,12 @@ get_query_results(QueryId, Config) -> {error, _} = E -> E; {ok, Result} -> Statistics = proplists:get_value(<<"statistics">>, Result), - #{ results => results_from_get_query_results(proplists:get_value(<<"results">>, Result)), - statistics => #{ bytes_scanned => proplists:get_value(<<"bytesScanned">>, Statistics), - records_matched => proplists:get_value(<<"recordsMatched">>, Statistics), - records_scanned => proplists:get_value(<<"recordsScanned">>, Statistics) - }, - status => status_from_get_query_results(proplists:get_value(<<"status">>, Result))} + {ok, #{ results => results_from_get_query_results(proplists:get_value(<<"results">>, Result)), + statistics => #{ bytes_scanned => proplists:get_value(<<"bytesScanned">>, Statistics), + records_matched => proplists:get_value(<<"recordsMatched">>, Statistics), + records_scanned => proplists:get_value(<<"recordsScanned">>, Statistics) + }, + status => status_from_get_query_results(proplists:get_value(<<"status">>, Result)) }} end. -spec results_from_get_query_results(In) -> Out diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index 92de743ea..fea176f6f 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -614,14 +614,14 @@ get_query_results_output_tests(_) -> {<<"recordsMatched">>, 360851.0}, {<<"recordsScanned">>, 610956.0}]}, {<<"status">>, <<"Complete">>}]), - #{ results => [#{ field => "LogEvent1-field1-name", - value => "LogEvent1-field1-value" }, - #{ field => "LogEvent1-field2-name", - value => "LogEvent1-field2-value" }], - statistics => #{ bytes_scanned => 81349723.0, - records_matched => 360851.0, - records_scanned => 610956.0 }, - status => complete }} + {ok, #{ results => [#{ field => "LogEvent1-field1-name", + value => "LogEvent1-field1-value" }, + #{ field => "LogEvent1-field2-name", + value => "LogEvent1-field2-value" }], + statistics => #{ bytes_scanned => 81349723.0, + records_matched => 360851.0, + records_scanned => 610956.0 }, + status => complete }}} ) ]). From 98c2d4307c58873f3866fb753e54026fa697c9f5 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 13 Jan 2021 17:48:06 +0000 Subject: [PATCH 196/310] Fix as per integration testing The AWS API is kind of misleading "results": [ [ { "field": "string", "value": "string" } ] ] but it's eventually right. --- src/erlcloud_cloudwatch_logs.erl | 6 ++++-- test/erlcloud_cloudwatch_logs_tests.erl | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 89cb2f4dc..e90428ef6 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -508,10 +508,12 @@ get_query_results(QueryId, Config) -> end. -spec results_from_get_query_results(In) -> Out - when In :: [[{binary(), binary()}]], + when In :: [[[{binary(), binary()}]]], Out :: [#{ field := string(), value := string() }]. -results_from_get_query_results(In) -> +results_from_get_query_results([]) -> + []; +results_from_get_query_results([In]) -> lists:map(fun (Result) -> #{ field => binary_to_list(proplists:get_value(<<"field">>, Result)), value => binary_to_list(proplists:get_value(<<"value">>, Result)) } diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index fea176f6f..a44e531db 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -606,10 +606,10 @@ get_query_results_output_tests(_) -> output_tests(?_f(erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab")), [ ?_cloudwatch_test( {"Tests output format for get_query_results", - jsx:encode([{<<"results">>, [[{<<"field">>, <<"LogEvent1-field1-name">>}, - {<<"value">>, <<"LogEvent1-field1-value">>}], - [{<<"field">>, <<"LogEvent1-field2-name">>}, - {<<"value">>, <<"LogEvent1-field2-value">>}]]}, + jsx:encode([{<<"results">>, [[[{<<"field">>, <<"LogEvent1-field1-name">>}, + {<<"value">>, <<"LogEvent1-field1-value">>}], + [{<<"field">>, <<"LogEvent1-field2-name">>}, + {<<"value">>, <<"LogEvent1-field2-value">>}]]]}, {<<"statistics">>, [{<<"bytesScanned">>, 81349723.0}, {<<"recordsMatched">>, 360851.0}, {<<"recordsScanned">>, 610956.0}]}, From a4c754839dfd964745f74ee49c43715e27618e64 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 13 Jan 2021 18:41:58 +0000 Subject: [PATCH 197/310] Fix as per integration testing --- src/erlcloud_cloudwatch_logs.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index e90428ef6..7990c1622 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -820,7 +820,7 @@ stop_query(QueryId) -> -spec stop_query(QueryId, Config) -> Result when QueryId :: string(), Config :: aws_config(), - Result :: ok. + Result :: ok | {error, erlcloud_aws:httpc_result_error()}. stop_query(QueryId, Config) -> Result = cw_request(Config, "StopQuery", [{<<"queryId">>, QueryId}]), case Result of From 87868d536939594619bb17f7394bfbb78ae004dd Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Mon, 25 Jan 2021 21:52:02 +0000 Subject: [PATCH 198/310] Complete example --- src/erlcloud_cloudwatch_logs.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 7990c1622..86443bce7 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -477,7 +477,7 @@ log_stream_order_by(last_event_time) -> <<"LastEventTime">>. %% ` %% application:ensure_all_started(erlcloud). %% {ok, Config} = erlcloud_aws:auto_config(). -%% {ok, Results} = erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab"). +%% {ok, Results} = erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab", Config). %% ` %% %% @end From 5e31836aaeb55ebee4b861a4c09dc9da2d8cbf93 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Mon, 25 Jan 2021 22:00:59 +0000 Subject: [PATCH 199/310] Aim for API consistency --- src/erlcloud_cloudwatch_logs.erl | 23 ++++++++++++++--------- test/erlcloud_cloudwatch_logs_tests.erl | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 86443bce7..5c932df30 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -112,8 +112,8 @@ describe_metric_filters/6, describe_metric_filters/7, - get_query_results/1, get_query_results/2, + get_query_results/3, put_logs_events/4, put_logs_events/5, @@ -477,34 +477,39 @@ log_stream_order_by(last_event_time) -> <<"LastEventTime">>. %% ` %% application:ensure_all_started(erlcloud). %% {ok, Config} = erlcloud_aws:auto_config(). -%% {ok, Results} = erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab", Config). +%% {ok, Results} = erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab", [], Config). %% ` %% %% @end %%------------------------------------------------------------------------------ --spec get_query_results(QueryId) -> Results +-spec get_query_results(QueryId, Options) -> Results when QueryId :: string(), + Options :: [{out, map}], Results :: {ok, query_results()} | {error, erlcloud_aws:httpc_result_error()}. -get_query_results(QueryId) -> - get_query_results(QueryId, default_config()). +get_query_results(QueryId, Options) -> + get_query_results(QueryId, Options, default_config()). --spec get_query_results(QueryId, Config) -> Results +-spec get_query_results(QueryId, Options, Config) -> Results when QueryId :: string(), + Options :: [{out, map}], Config :: aws_config(), Results :: {ok, query_results()} | {error, erlcloud_aws:httpc_result_error()}. -get_query_results(QueryId, Config) -> +get_query_results(QueryId, Options, Config) -> Result0 = cw_request(Config, "GetQueryResults", [{<<"queryId">>, QueryId}]), + Out = proplists:get_value(out, Options, undefined), case Result0 of {error, _} = E -> E; - {ok, Result} -> + {ok, Result} when Out =:= map -> Statistics = proplists:get_value(<<"statistics">>, Result), {ok, #{ results => results_from_get_query_results(proplists:get_value(<<"results">>, Result)), statistics => #{ bytes_scanned => proplists:get_value(<<"bytesScanned">>, Statistics), records_matched => proplists:get_value(<<"recordsMatched">>, Statistics), records_scanned => proplists:get_value(<<"recordsScanned">>, Statistics) }, - status => status_from_get_query_results(proplists:get_value(<<"status">>, Result)) }} + status => status_from_get_query_results(proplists:get_value(<<"status">>, Result)) }}; + _ -> + Result0 end. -spec results_from_get_query_results(In) -> Out diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index a44e531db..c6c24e5d3 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -603,7 +603,7 @@ stop_query_output_tests(_) -> ]). get_query_results_output_tests(_) -> - output_tests(?_f(erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab")), [ + output_tests(?_f(erlcloud_cloudwatch_logs:get_query_results("12ab3456-12ab-123a-789e-1234567890ab", [{out, map}])), [ ?_cloudwatch_test( {"Tests output format for get_query_results", jsx:encode([{<<"results">>, [[[{<<"field">>, <<"LogEvent1-field1-name">>}, From c02b1fe5dcdd5ec13702d958003881cce745edb5 Mon Sep 17 00:00:00 2001 From: kkuzmin Date: Tue, 26 Jan 2021 10:37:04 +0000 Subject: [PATCH 200/310] Add AWS Workspaces (#682) * Add AWS Workspaces * Code clean up. * Add DescribeWorkspaceDirectories * Add output in maps * Fix spec * Refactor prolists_to_map * Code cleanup * Fix spec --- include/erlcloud_aws.hrl | 3 + include/erlcloud_workspaces.hrl | 107 ++++++ src/erlcloud_aws.erl | 5 +- src/erlcloud_util.erl | 21 +- src/erlcloud_workspaces.erl | 588 +++++++++++++++++++++++++++++ test/erlcloud_util_tests.erl | 61 +++ test/erlcloud_workspaces_tests.erl | 449 ++++++++++++++++++++++ 7 files changed, 1232 insertions(+), 2 deletions(-) create mode 100644 include/erlcloud_workspaces.hrl create mode 100644 src/erlcloud_workspaces.erl create mode 100644 test/erlcloud_util_tests.erl create mode 100644 test/erlcloud_workspaces_tests.erl diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 920d81d80..ee72c9ebf 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -130,6 +130,9 @@ config_scheme="https://"::string(), config_host="config.us-east-1.amazonaws.com"::string(), config_port=443::non_neg_integer(), + workspaces_scheme="https://"::string(), + workspaces_host="workspaces.us-east-1.amazonaws.com"::string(), + workspaces_port=443::non_neg_integer(), access_key_id::string()|undefined|false, secret_access_key::string()|undefined|false, security_token=undefined::string()|undefined, diff --git a/include/erlcloud_workspaces.hrl b/include/erlcloud_workspaces.hrl new file mode 100644 index 000000000..730f59203 --- /dev/null +++ b/include/erlcloud_workspaces.hrl @@ -0,0 +1,107 @@ +-ifndef(erlcloud_workspaces_hrl). +-define(erlcloud_workspaces_hrl, 0). + +-include("erlcloud.hrl"). + +-define(WORKSPACES_LIMIT, 25). + + +%%%------------------------------------------------------------------------------ +%% +%% Common data types +%% +%%%------------------------------------------------------------------------------ + +-record(workspace_modification_state, { + resource :: undefined | binary(), + state :: undefined | binary() +}). + +-record(workspace_properties, { + computer_type_name :: undefined | binary(), + root_volume_size_gib :: undefined | non_neg_integer(), + running_mode :: undefined | binary(), + running_mode_auto_stop_timeout_in_minutes :: undefined | non_neg_integer(), + user_volume_size_gib :: undefined | non_neg_integer() +}). + +-record(workspace, { + bundle_id :: undefined | binary(), + computer_name :: undefined | binary(), + directory_id :: undefined | binary(), + error_code :: undefined | binary(), + error_message :: undefined | binary(), + ip_address :: undefined | binary(), + modification_states :: undefined | [#workspace_modification_state{}], + root_volume_encryption_enabled :: undefined | boolean(), + state :: undefined | binary(), + subnet_id :: undefined | binary(), + user_name :: undefined | binary(), + user_volume_encryption_enabled :: undefined | boolean(), + volume_encryption_key :: undefined | binary(), + workspace_id :: undefined | binary(), + workspace_properties :: undefined | #workspace_properties{} +}). + +-record(workspaces_selfservice_permissions, { + change_compute_type :: undefined | binary(), + increase_volume_size :: undefined | binary(), + rebuild_workspace :: undefined | binary(), + restart_workspace :: undefined | binary(), + switch_running_mode :: undefined | binary() +}). + +-record(workspace_access_properties, { + device_type_android :: undefined | binary(), + device_type_chrome_os :: undefined | binary(), + device_type_ios :: undefined | binary(), + device_type_osx :: undefined | binary(), + device_type_web :: undefined | binary(), + device_type_windows :: undefined | binary(), + device_type_zero_client :: undefined | binary() +}). + +-record(workspace_creation_properties, { + custom_security_group_id :: undefined | binary(), + default_ou :: undefined | binary(), + enable_internet_access :: undefined | boolean(), + enable_maintenance_mode :: undefined | boolean(), + enable_work_docs :: undefined | boolean(), + user_enabled_as_local_administrator :: undefined | boolean() +}). + +-record(workspace_directory, { + alias :: undefined | binary(), + customer_user_name :: undefined | binary(), + directory_id :: undefined | binary(), + directory_name :: undefined | binary(), + directory_type :: undefined | binary(), + dns_ip_address :: undefined | [binary()], + iam_role_id :: undefined | binary(), + ip_group_ids :: undefined | [binary()], + registration_code :: undefined | binary(), + selfservice_permissions :: undefined | #workspaces_selfservice_permissions{}, + state :: undefined | binary(), + subnet_ids :: undefined | [binary()], + tenancy :: undefined | binary(), + workspace_access_properties :: undefined | #workspace_access_properties{}, + workspace_creation_properties :: undefined | #workspace_creation_properties{}, + workspace_security_group_id :: undefined | binary() +}). + +-record(describe_workspaces, { + next_token :: undefined | binary(), + workspaces :: undefined | [#workspace{}] +}). + +-record(describe_workspace_directories, { + next_token :: undefined | binary(), + workspace_directories :: undefined | [#workspace_directory{}] +}). + +-record(workspaces_tag, { + key :: undefined | binary(), + value :: undefined | binary() +}). + +-endif. \ No newline at end of file diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 7cbffd594..4e114fea0 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -781,7 +781,10 @@ service_config( <<"cur">>, Region, Config ) -> Config#aws_config{cur_host = Host}; service_config( <<"application_autoscaling">>, Region, Config ) -> Host = service_host(<<"application_autoscaling">>, Region), - Config#aws_config{cur_host = Host}. + Config#aws_config{application_autoscaling_host = Host}; +service_config( <<"workspaces">> = Service, Region, Config ) -> + Host = service_host(Service, Region), + Config#aws_config{workspaces_host = Host}. %%%--------------------------------------------------------------------------- diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index 2aa9fe7de..7d9a45848 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -19,7 +19,8 @@ filter_undef/1, uri_parse/1, http_uri_decode/1, - http_uri_encode/1 + http_uri_encode/1, + proplists_to_map/1, proplists_to_map/2 ]). -define(MAX_ITEMS, 1000). @@ -246,3 +247,21 @@ http_uri_encode(URI) -> http_uri_encode(URI) -> http_uri:encode(URI). -endif. + +-spec proplists_to_map(proplists:proplist() | any()) -> map() | any(). +proplists_to_map([]) -> []; +proplists_to_map([{}]) -> #{}; +proplists_to_map([{_,_} | _] = Proplist) -> + proplists_to_map(Proplist, #{}); +proplists_to_map([Head | _Tail] = List) when is_list(Head) -> + [proplists_to_map(E) || E <- List]; +proplists_to_map(V) -> V. + +-spec proplists_to_map(proplists:proplist(), map()) -> map(). +proplists_to_map([], Acc) -> + Acc; +proplists_to_map([{Key, Val} | Tail], Acc) when is_list(Val) -> + proplists_to_map(Tail, Acc#{Key => proplists_to_map(Val)}); +proplists_to_map([{Key, Val} | Tail], Acc) -> + proplists_to_map(Tail, Acc#{Key => Val}). + diff --git a/src/erlcloud_workspaces.erl b/src/erlcloud_workspaces.erl new file mode 100644 index 000000000..d8d3a74b8 --- /dev/null +++ b/src/erlcloud_workspaces.erl @@ -0,0 +1,588 @@ +%% @doc +%% An Erlang interface to AWS Workspaces. +%% +%% Output is in the form of `{ok, Value}' or `{error, Reason}'. The +%% format of `Value' is controlled by the `out' option, which defaults +%% to `json'. The possible values are: +%% +%% * `json' - The output from Workspaces as processed by `jsx:decode' +%% but with no further manipulation. +%% +%% * `record' - A record containing all the information from the +%% Workspaces response except field types. +%% +%% Workspaces errors are returned in the form `{error, {ErrorCode, Message}}' +%% where `ErrorCode' and 'Message' are both binary +%% strings. + +%% See the unit tests for additional usage examples beyond what are +%% provided for each function. +%% +%% @end + +-module(erlcloud_workspaces). + +-include("erlcloud_aws.hrl"). +-include("erlcloud_workspaces.hrl"). + +%%% Library initialization. +-export([configure/2, configure/3, configure/4, new/2, new/3, new/4]). + +-define(API_VERSION, "20150408"). +-define(OUTPUT_CHOICES, [json, record, map]). + +-export([ + describe_tags/1, describe_tags/2, + describe_workspaces/0, describe_workspaces/1, describe_workspaces/2, + describe_workspace_directories/0, describe_workspace_directories/1, describe_workspace_directories/2 +]). + +-export_type([ + describe_workspaces_opt/0, + describe_workspaces_opts/0, + describe_workspace_directories_opt/0, + describe_workspace_directories_opts/0 + ]). + + +%%%------------------------------------------------------------------------------ +%%% Library initialization. +%%%------------------------------------------------------------------------------ + +-spec new(string(), string()) -> aws_config(). + +new(AccessKeyID, SecretAccessKey) -> + #aws_config{ + access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey + }. + +-spec new(string(), string(), string()) -> aws_config(). + +new(AccessKeyID, SecretAccessKey, Host) -> + #aws_config{ + access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey, + workspaces_host=Host + }. + +-spec new(string(), string(), string(), non_neg_integer()) -> aws_config(). + +new(AccessKeyID, SecretAccessKey, Host, Port) -> + #aws_config{ + access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey, + workspaces_host=Host, + workspaces_port=Port + }. + +-spec configure(string(), string()) -> ok. + +configure(AccessKeyID, SecretAccessKey) -> + put(aws_config, new(AccessKeyID, SecretAccessKey)), + ok. + +-spec configure(string(), string(), string()) -> ok. + +configure(AccessKeyID, SecretAccessKey, Host) -> + put(aws_config, new(AccessKeyID, SecretAccessKey, Host)), + ok. + +-spec configure(string(), string(), string(), non_neg_integer()) -> ok. + +configure(AccessKeyID, SecretAccessKey, Host, Port) -> + put(aws_config, new(AccessKeyID, SecretAccessKey, Host, Port)), + ok. + +default_config() -> + erlcloud_aws:default_config(). + + +%%%------------------------------------------------------------------------------ +%%% Shared types +%%%------------------------------------------------------------------------------ + +-type string_param() :: binary() | string(). + +-type json_pair() :: {binary() | atom(), jsx:json_term()}. +-type json_return() :: {ok, jsx:json_term()} | {error, term()}. +-type workspaces_return(Record) :: {ok, jsx:json_term() | Record } | {error, term()}. +-type decode_fun() :: fun((jsx:json_term(), decode_opts()) -> tuple()). + +%%%------------------------------------------------------------------------------ +%%% Shared Options +%%%------------------------------------------------------------------------------ +-type out_type() :: json | record. +-type out_opt() :: {out, out_type()}. +-type property() :: proplists:property(). + +-type aws_opts() :: [json_pair()]. +-type workspaces_opts() :: [out_opt()]. +-type opts() :: {aws_opts(), workspaces_opts()}. + +-spec verify_workspaces_opt(atom(), term()) -> ok. +verify_workspaces_opt(out, Value) -> + case lists:member(Value, ?OUTPUT_CHOICES) of + true -> + ok; + false -> + error({erlcloud_workspaces, {invalid_opt, {out, Value}}}) + end; +verify_workspaces_opt(Name, Value) -> + error({erlcloud_workspaces, {invalid_opt, {Name, Value}}}). + +-type opt_table_entry() :: {atom(), binary(), fun((_) -> jsx:json_term())}. +-type opt_table() :: [opt_table_entry()]. +-spec opt_folder(opt_table(), property(), opts()) -> opts(). +opt_folder(_, {_, undefined}, Opts) -> + %% ignore options set to undefined + Opts; +opt_folder(Table, {Name, Value}, {AwsOpts, EcsOpts}) -> + case lists:keyfind(Name, 1, Table) of + {Name, Key, ValueFun} -> + {[{Key, ValueFun(Value)} | AwsOpts], EcsOpts}; + false -> + verify_workspaces_opt(Name, Value), + {AwsOpts, [{Name, Value} | EcsOpts]} + end. + +-spec opts(opt_table(), proplist()) -> opts(). +opts(Table, Opts) when is_list(Opts) -> + %% remove duplicate options + Opts1 = lists:ukeysort(1, proplists:unfold(Opts)), + lists:foldl(fun(Opt, A) -> opt_folder(Table, Opt, A) end, {[], []}, Opts1); +opts(_, _) -> + error({erlcloud_workspaces, opts_not_list}). + +%%%------------------------------------------------------------------------------ +%%% Shared Decoders +%%%------------------------------------------------------------------------------ + +-type decode_opt() :: {typed, boolean()}. +-type decode_opts() :: [decode_opt()]. +-type record_desc() :: {tuple(), field_table()}. + +-spec id(X) -> X. +id(X) -> X. + +-spec id(X, decode_opts()) -> X. +id(X, _) -> X. + +-type field_table() :: [{binary(), pos_integer(), + fun((jsx:json_term(), decode_opts()) -> term())}]. + +-spec decode_folder(field_table(), json_pair(), decode_opts(), tuple()) -> tuple(). +decode_folder(Table, {Key, Value}, Opts, A) -> + case lists:keyfind(Key, 1, Table) of + {Key, Index, ValueFun} -> + setelement(Index, A, ValueFun(Value, Opts)); + false -> + A + end. + + +-spec decode_record(record_desc(), jsx:json_term(), decode_opts()) -> tuple(). +decode_record({Record, _}, [{}], _) -> + %% jsx returns [{}] for empty objects + Record; +decode_record({Record, Table}, Json, Opts) -> + lists:foldl(fun(Pair, A) -> decode_folder(Table, Pair, Opts, A) end, Record, Json). + +%%%------------------------------------------------------------------------------ +%%% Output +%%%------------------------------------------------------------------------------ +-spec out(json_return(), decode_fun(), workspaces_opts()) + -> {ok, jsx:json_term() | tuple()} | + {simple, term()} | + {error, term()}. +out({error, Reason}, _, _) -> + {error, Reason}; +out({ok, Json}, Decode, Opts) -> + case proplists:get_value(out, Opts, record) of + json -> + {ok, Json}; + record -> + {ok, Decode(Json, [])}; + map -> + {ok, erlcloud_util:proplists_to_map(Json)} + end. + +%%%------------------------------------------------------------------------------ +%%% Shared Records +%%%------------------------------------------------------------------------------ + +-spec workspace_record() -> record_desc(). +workspace_record() -> + {#workspace{}, + [ + {<<"BundleId">>, #workspace.bundle_id, fun id/2}, + {<<"ComputerName">>, #workspace.computer_name, fun id/2}, + {<<"DirectoryId">>, #workspace.directory_id, fun id/2}, + {<<"ErrorCode">>, #workspace.error_code, fun id/2}, + {<<"ErrorMessage">>, #workspace.error_message, fun id/2}, + {<<"IpAddress">>, #workspace.ip_address, fun id/2}, + {<<"ModificationStates">>, #workspace.modification_states, fun decode_modification_state_list/2}, + {<<"RootVolumeEncryptionEnabled">>, #workspace.root_volume_encryption_enabled, fun id/2}, + {<<"State">>, #workspace.state, fun id/2}, + {<<"SubnetId">>, #workspace.subnet_id, fun id/2}, + {<<"UserName">>, #workspace.user_name, fun id/2}, + {<<"UserVolumeEncryptionEnabled">>, #workspace.user_volume_encryption_enabled, fun id/2}, + {<<"VolumeEncryptionKey">>, #workspace.volume_encryption_key, fun id/2}, + {<<"WorkspaceId">>, #workspace.workspace_id, fun id/2}, + {<<"WorkspaceProperties">>, #workspace.workspace_properties, fun decode_workspace_properties/2} + ] + }. + +-spec modification_state_record() -> record_desc(). +modification_state_record() -> + {#workspace_modification_state{}, + [ + {<<"Resource">>, #workspace_modification_state.resource, fun id/2}, + {<<"State">>, #workspace_modification_state.state, fun id/2} + ] + }. + +-spec workspace_properties_record() -> record_desc(). +workspace_properties_record() -> + {#workspace_properties{}, + [ + {<<"ComputeTypeName">>, #workspace_properties.computer_type_name, fun id/2}, + {<<"RootVolumeSizeGib">>, #workspace_properties.root_volume_size_gib, fun id/2}, + {<<"RunningMode">>, #workspace_properties.running_mode, fun id/2}, + {<<"RunningModeAutoStopTimeoutInMinutes">>, #workspace_properties.running_mode_auto_stop_timeout_in_minutes, fun id/2}, + {<<"UserVolumeSizeGib">>, #workspace_properties.user_volume_size_gib, fun id/2} + ] + }. + +-spec tag_record() -> record_desc(). +tag_record() -> + {#workspaces_tag{}, + [ + {<<"Key">>, #workspaces_tag.key, fun id/2}, + {<<"Value">>, #workspaces_tag.value, fun id/2} + ] + }. + +-spec workspace_directory_record() -> record_desc(). +workspace_directory_record() -> + {#workspace_directory{}, + [ + {<<"Alias">>, #workspace_directory.alias, fun id/2}, + {<<"CustomerUserName">>, #workspace_directory.customer_user_name, fun id/2}, + {<<"DirectoryId">>, #workspace_directory.directory_id, fun id/2}, + {<<"DirectoryName">>, #workspace_directory.directory_name, fun id/2}, + {<<"DirectoryType">>, #workspace_directory.directory_type, fun id/2}, + {<<"DnsIpAddresses">>, #workspace_directory.dns_ip_address, fun id/2}, + {<<"IamRoleId">>, #workspace_directory.iam_role_id, fun id/2}, + {<<"ipGroupIds">>, #workspace_directory.ip_group_ids, fun id/2}, + {<<"RegistrationCode">>, #workspace_directory.registration_code, fun id/2}, + {<<"SelfservicePermissions">>, #workspace_directory.selfservice_permissions, fun decode_selfservice_permissions/2}, + {<<"State">>, #workspace_directory.state, fun id/2}, + {<<"SubnetIds">>, #workspace_directory.subnet_ids, fun id/2}, + {<<"Tenancy">>, #workspace_directory.tenancy, fun id/2}, + {<<"WorkspaceAccessProperties">>, #workspace_directory.workspace_access_properties, fun decode_workspace_access_properties/2}, + {<<"WorkspaceCreationProperties">>, #workspace_directory.workspace_creation_properties, fun decode_workspace_creation_properties/2}, + {<<"WorkspaceSecurityGroupId">>, #workspace_directory.workspace_security_group_id, fun id/2} + ] + }. + +-spec workspaces_selfservice_permissions_record() -> record_desc(). +workspaces_selfservice_permissions_record() -> + {#workspaces_selfservice_permissions{}, + [ + {<<"ChangeComputeType">>, #workspaces_selfservice_permissions.change_compute_type, fun id/2}, + {<<"IncreaseVolumeSize">>, #workspaces_selfservice_permissions.increase_volume_size, fun id/2}, + {<<"RebuildWorkspace">>, #workspaces_selfservice_permissions.rebuild_workspace, fun id/2}, + {<<"RestartWorkspace">>, #workspaces_selfservice_permissions.restart_workspace, fun id/2}, + {<<"SwitchRunningMode">>, #workspaces_selfservice_permissions.switch_running_mode, fun id/2} + ] + }. + +-spec workspace_access_properties_record() -> record_desc(). +workspace_access_properties_record() -> + {#workspace_access_properties{}, + [ + {<<"DeviceTypeAndroid">>, #workspace_access_properties.device_type_android, fun id/2}, + {<<"DeviceTypeChromeOs">>, #workspace_access_properties.device_type_chrome_os, fun id/2}, + {<<"DeviceTypeIos">>, #workspace_access_properties.device_type_ios, fun id/2}, + {<<"DeviceTypeOsx">>, #workspace_access_properties.device_type_osx, fun id/2}, + {<<"DeviceTypeWeb">>, #workspace_access_properties.device_type_web, fun id/2}, + {<<"DeviceTypeWindows">>, #workspace_access_properties.device_type_windows, fun id/2}, + {<<"DeviceTypeZeroClient">>, #workspace_access_properties.device_type_zero_client, fun id/2} + ] + }. + +-spec workspace_creation_properties_record() -> record_desc(). +workspace_creation_properties_record() -> + {#workspace_creation_properties{}, + [ + {<<"CustomSecurityGroupId">>, #workspace_creation_properties.custom_security_group_id, fun id/2}, + {<<"DefaultOu">>, #workspace_creation_properties.default_ou, fun id/2}, + {<<"EnableInternetAccess">>, #workspace_creation_properties.enable_internet_access, fun id/2}, + {<<"EnableMaintenanceMode">>, #workspace_creation_properties.enable_maintenance_mode, fun id/2}, + {<<"EnableWorkDocs">>, #workspace_creation_properties.enable_work_docs, fun id/2}, + {<<"UserEnabledAsLocalAdministrator">>, #workspace_creation_properties.user_enabled_as_local_administrator, fun id/2} + ] + }. + +decode_tags_list(V, Opts) -> + [decode_record(tag_record(), I, Opts) || I <- V]. + +decode_workspace_properties(V, Opts) -> + decode_record(workspace_properties_record(), V, Opts). + +decode_modification_state_list(V, Opts) -> + [decode_record(modification_state_record(), I, Opts) || I <- V]. + +decode_workspaces_list(V, Opts) -> + [decode_record(workspace_record(), I, Opts) || I <- V]. + +decode_workspace_directories_list(V, Opts) -> + [decode_record(workspace_directory_record(), I, Opts) || I <- V]. + +decode_selfservice_permissions(V, Opts) -> + decode_record(workspaces_selfservice_permissions_record(), V, Opts). + +decode_workspace_access_properties(V, Opts) -> + decode_record(workspace_access_properties_record(), V, Opts). + +decode_workspace_creation_properties(V, Opts) -> + decode_record(workspace_creation_properties_record(), V, Opts). + +%%%------------------------------------------------------------------------------ +%%% AWS Workspaces API Functions +%%%------------------------------------------------------------------------------ + +%%%------------------------------------------------------------------------------ +%% DescribeTags +%%%------------------------------------------------------------------------------ +-type describe_tags_opt() :: {resource_id, string_param()} | + out_opt(). +-type describe_tags_opts() :: [describe_tags_opt()]. + +-spec describe_tags_opts() -> opt_table(). +describe_tags_opts() -> + [ + {resource_id, <<"ResourceId">>, fun encode_json_value/1} + ]. + +-spec describe_tags(Opts :: describe_tags_opts()) -> workspaces_return([#workspaces_tag{}]). +describe_tags(Opts) -> + describe_tags(Opts, default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% Workspaces API +%% [https://docs.aws.amazon.com/workspaces/latest/api/API_DescribeTags.html] +%% +%% ===Example=== +%% +%% Describe tags for workspace id "ws-c8wvb67py" +%% +%% ` +%% {ok, Tags} = erlcloud_workspaces:describe_tags([{resource_id, "ws-c8wvb67py"}, {out, json}]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec describe_tags(Opts :: describe_tags_opts(), Config :: aws_config()) -> workspaces_return([#workspaces_tag{}]). +describe_tags(Opts, #aws_config{} = Config) -> + {AwsOpts, WorkspacesOpts} = opts(describe_tags_opts(), Opts), + Return = workspaces_request( + Config, + "DescribeTags", + AwsOpts), + out(Return, fun(Json, UOpts) -> + TagsList = proplists:get_value(<<"TagList">>, Json), + decode_tags_list(TagsList, UOpts) + end, + WorkspacesOpts). + +%%%------------------------------------------------------------------------------ +%% DescribeWorkspaces +%%%------------------------------------------------------------------------------ +-type describe_workspaces_opt() :: {bundle_id, string_param()} | + {directory_id, string_param()} | + {limit, pos_integer()} | + {next_token, string_param()} | + {user_name, string_param()} | + {workspace_ids, [string_param()]} | + out_opt(). +-type describe_workspaces_opts() :: [describe_workspaces_opt()]. + +-spec describe_workspaces_opts() -> opt_table(). +describe_workspaces_opts() -> + [ + {bundle_id, <<"BundleId">>, fun encode_json_value/1}, + {directory_id, <<"DirectoryId">>, fun encode_json_value/1}, + {limit, <<"Limit">>, fun id/1}, + {next_token, <<"NextToken">>, fun encode_json_value/1}, + {user_name, <<"UserName">>, fun encode_json_value/1}, + {workspace_ids, <<"WorkspaceIds">>, fun encode_json_value/1} + ]. + +-spec describe_workspaces_record() -> record_desc(). +describe_workspaces_record() -> + {#describe_workspaces{}, + [{<<"NextToken">>, #describe_workspaces.next_token, fun id/2}, + {<<"Workspaces">>, #describe_workspaces.workspaces, fun decode_workspaces_list/2} + ]}. + +-spec describe_workspaces() -> workspaces_return(#describe_workspaces{}). +describe_workspaces() -> + describe_workspaces([], default_config()). + +-spec describe_workspaces(describe_workspaces_opts() | aws_config()) -> workspaces_return(#describe_workspaces{}). +describe_workspaces(#aws_config{} = Config) -> + describe_workspaces([], Config); +describe_workspaces(Opts) -> + describe_workspaces(Opts, default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% Workspaces API +%% [https://docs.aws.amazon.com/workspaces/latest/api/API_DescribeWorkspaces.html] +%% +%% ===Example=== +%% +%% Describe workspaces in Directory "TestDirectory" +%% +%% ` +%% {ok, Clusters} = erlcloud_workspaces:describe_workspaces([{directory_id, "TestDirectory"}, {out, json}]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec describe_workspaces(Opts :: describe_workspaces_opts(), Config :: aws_config()) -> workspaces_return(#describe_workspaces{}). +describe_workspaces(Opts, #aws_config{} = Config) -> + {AwsOpts, WorkspacesOpts} = opts(describe_workspaces_opts(), Opts), + Return = workspaces_request( + Config, + "DescribeWorkspaces", + AwsOpts), + out(Return, fun(Json, UOpts) -> decode_record(describe_workspaces_record(), Json, UOpts) end, + WorkspacesOpts). + +%%%------------------------------------------------------------------------------ +%% DescribeWorkspaceDirectories +%%%------------------------------------------------------------------------------ +-type describe_workspace_directories_opt() :: {directory_ids, [string_param()]} | + {limit, pos_integer()} | + {next_token, string_param()} | + out_opt(). +-type describe_workspace_directories_opts() :: [describe_workspace_directories_opt()]. + +-spec describe_workspace_directories_opts() -> opt_table(). +describe_workspace_directories_opts() -> + [ + {directory_ids, <<"DirectoryIds">>, fun encode_json_value/1}, + {limit, <<"Limit">>, fun id/1}, + {next_token, <<"NextToken">>, fun encode_json_value/1} + ]. + +-spec describe_workspace_directories_record() -> record_desc(). +describe_workspace_directories_record() -> + {#describe_workspace_directories{}, + [{<<"NextToken">>, #describe_workspace_directories.next_token, fun id/2}, + {<<"Directories">>, #describe_workspace_directories.workspace_directories, fun decode_workspace_directories_list/2} + ]}. + +-spec describe_workspace_directories() -> workspaces_return(#describe_workspace_directories{}). +describe_workspace_directories() -> + describe_workspace_directories([], default_config()). + +-spec describe_workspace_directories(describe_workspace_directories_opts() | aws_config()) + -> workspaces_return(#describe_workspace_directories{}). +describe_workspace_directories(#aws_config{} = Config) -> + describe_workspace_directories([], Config); +describe_workspace_directories(Opts) -> + describe_workspace_directories(Opts, default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% Workspaces API +%% [https://docs.aws.amazon.com/workspaces/latest/api/API_DescribeWorkspaceDirectories.html] +%% +%% ===Example=== +%% +%% Describe workspaces directory "TestDirectory" +%% +%% ` +%% {ok, Clusters} = erlcloud_workspaces:describe_workspace_directories([{directory_ids, ["TestDirectory"]}, {out, json}]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec describe_workspace_directories( + Opts :: describe_workspace_directories_opts(), + Config :: aws_config()) + -> workspaces_return(#describe_workspace_directories{}). +describe_workspace_directories(Opts, #aws_config{} = Config) -> + {AwsOpts, WorkspacesOpts} = opts(describe_workspace_directories_opts(), Opts), + Return = workspaces_request( + Config, + "DescribeWorkspaceDirectories", + AwsOpts), + out(Return, fun(Json, UOpts) -> decode_record(describe_workspace_directories_record(), Json, UOpts) end, + WorkspacesOpts). + +%%%------------------------------------------------------------------------------ +%%% Internal Functions +%%%------------------------------------------------------------------------------ +workspaces_request(Config, Operation, Body) -> + case erlcloud_aws:update_config(Config) of + {ok, Config1} -> + workspaces_request_no_update(Config1, Operation, Body); + {error, Reason} -> + {error, Reason} + end. + +workspaces_request_no_update(Config, Operation, Body) -> + Payload = case Body of + [] -> <<"{}">>; + _ -> jsx:encode(lists:flatten(Body)) + end, + Headers = headers(Config, Operation, Payload), + Request = #aws_request{service = workspaces, + uri = uri(Config), + method = post, + request_headers = Headers, + request_body = Payload}, + case erlcloud_aws:request_to_return(erlcloud_retry:request(Config, Request, fun workspaces_result_fun/1)) of + {ok, {_RespHeaders, <<>>}} -> {ok, []}; + {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody, [{return_maps, false}])}; + {error, _} = Error-> Error + end. + +-spec workspaces_result_fun(Request :: aws_request()) -> aws_request(). +workspaces_result_fun(#aws_request{response_type = ok} = Request) -> + Request; +workspaces_result_fun(#aws_request{response_type = error, + error_type = aws, + response_status = Status} = Request) when Status >= 500 -> + Request#aws_request{should_retry = true}; +workspaces_result_fun(#aws_request{response_type = error, error_type = aws} = Request) -> + Request#aws_request{should_retry = false}. + +headers(Config, Operation, Body) -> + Headers = [{"host", Config#aws_config.workspaces_host}, + {"x-amz-target", lists:append(["WorkspacesService.", Operation])}, + {"content-type", "application/x-amz-json-1.1"}], + Region = erlcloud_aws:aws_region_from_host(Config#aws_config.workspaces_host), + erlcloud_aws:sign_v4_headers(Config, Headers, Body, Region, "workspaces"). + +uri(#aws_config{workspaces_scheme = Scheme, workspaces_host = Host} = Config) -> + lists:flatten([Scheme, Host, port_spec(Config)]). + +port_spec(#aws_config{workspaces_port=80}) -> + ""; +port_spec(#aws_config{workspaces_port=Port}) -> + [":", erlang:integer_to_list(Port)]. + + +encode_json_value(undefined) -> undefined; +encode_json_value(true) -> true; +encode_json_value(false) -> false; +encode_json_value(L) when is_list(L), is_list(hd(L)) -> [encode_json_value(V) || V <- L]; +encode_json_value(L) when is_list(L), is_binary(hd(L)) -> [encode_json_value(V) || V <- L]; +encode_json_value(L) when is_list(L) -> list_to_binary(L); +encode_json_value(B) when is_binary(B) -> B; +encode_json_value(A) when is_atom(A) -> atom_to_binary(A, latin1). + diff --git a/test/erlcloud_util_tests.erl b/test/erlcloud_util_tests.erl new file mode 100644 index 000000000..f7d4fdcac --- /dev/null +++ b/test/erlcloud_util_tests.erl @@ -0,0 +1,61 @@ +-module(erlcloud_util_tests). +-include_lib("eunit/include/eunit.hrl"). +-include("erlcloud.hrl"). + +request_test_() -> + {foreach, + fun start/0, + fun stop/1, + [fun test_proplists_to_map/1]}. + +start() -> + ok. + +stop(_) -> + ok. + +test_proplists_to_map(_) -> + Proplist = [{<<"Directories">>, + [[{<<"Alias">>,<<"d-93671bd1a1">>}, + {<<"DnsIpAddresses">>,[<<"172.16.1.31">>,<<"172.16.0.42">>]}, + {<<"SelfservicePermissions">>,[ + {<<"SwitchRunningMode">>,<<"ENABLED">>}]}, + {<<"State">>,<<"REGISTERED">>}, + {<<"SubnetIds">>, + [<<"subnet-02d62b601c121d0ca">>, + <<"subnet-08b608fe3a2ecc501">>]}, + {<<"Tenancy">>,<<"SHARED">>}, + {<<"WorkspaceAccessProperties">>, + [{<<"DeviceTypeAndroid">>,<<"ALLOW">>}, + {<<"DeviceTypeZeroClient">>,<<"ALLOW">>}]}, + {<<"WorkspaceCreationProperties">>, + [{<<"EnableInternetAccess">>,true}, + {<<"UserEnabledAsLocalAdministrator">>,true}]}, + {<<"WorkspaceSecurityGroupId">>, + <<"sg-064d3dbf40db978bd">>}]]}], + Expected = #{<<"Directories">> => + [#{<<"Alias">> => <<"d-93671bd1a1">>, + <<"DnsIpAddresses">> => + [<<"172.16.1.31">>,<<"172.16.0.42">>], + <<"SelfservicePermissions">> => + #{<<"SwitchRunningMode">> => <<"ENABLED">>}, + <<"State">> => <<"REGISTERED">>, + <<"SubnetIds">> => + [<<"subnet-02d62b601c121d0ca">>, + <<"subnet-08b608fe3a2ecc501">>], + <<"Tenancy">> => <<"SHARED">>, + <<"WorkspaceAccessProperties">> => + #{<<"DeviceTypeAndroid">> => <<"ALLOW">>, + <<"DeviceTypeZeroClient">> => <<"ALLOW">>}, + <<"WorkspaceCreationProperties">> => + #{<<"EnableInternetAccess">> => true, + <<"UserEnabledAsLocalAdministrator">> => true}, + <<"WorkspaceSecurityGroupId">> => + <<"sg-064d3dbf40db978bd">>}]}, + [?_assertEqual(Expected, erlcloud_util:proplists_to_map(Proplist)), + ?_assertEqual(#{}, erlcloud_util:proplists_to_map([{}])), + ?_assertEqual([], erlcloud_util:proplists_to_map([])), + ?_assertEqual([<<"a">>, <<"b">>], erlcloud_util:proplists_to_map([<<"a">>, <<"b">>])) + ]. + + diff --git a/test/erlcloud_workspaces_tests.erl b/test/erlcloud_workspaces_tests.erl new file mode 100644 index 000000000..bc850069c --- /dev/null +++ b/test/erlcloud_workspaces_tests.erl @@ -0,0 +1,449 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +-module(erlcloud_workspaces_tests). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/erlcloud_workspaces.hrl"). + +%% Unit tests for erlcloud_workspaces. +%% These tests work by using meck to mock erlcloud_httpc. There are two classes of test: input and output. +%% +%% Input tests verify that different function args produce the desired JSON request. +%% An input test list provides a list of funs and the JSON that is expected to result. +%% +%% Output tests verify that the http response produces the correct return from the fun. +%% An output test lists provides a list of response bodies and the expected return. + +%% The _workspaces_test macro provides line number annotation to a test, similar to _test, but doesn't wrap in a fun +-define(_workspaces_test(T), {?LINE, T}). +%% The _f macro is a terse way to wrap code in a fun. Similar to _test but doesn't annotate with a line number +-define(_f(F), fun() -> F end). + +-export([validate_body/2]). + +%%%=================================================================== +%%% Test entry points +%%%=================================================================== +operation_test_() -> + {foreach, + fun start/0, + fun stop/1, + [ + fun describe_tags_input_tests/1, + fun describe_tags_output_tests/1, + fun describe_workspaces_input_tests/1, + fun describe_workspaces_output_tests/1, + fun describe_workspace_directories_input_tests/1, + fun describe_workspace_directories_output_tests/1 + ] + }. + +start() -> + meck:new(erlcloud_httpc), + ok. + + +stop(_) -> + meck:unload(erlcloud_httpc). + +%%%=================================================================== +%%% Actual test specifiers +%%%=================================================================== + +%% DescribeTags test based on the API examples: +%% https://docs.aws.amazon.com/workspaces/latest/api/API_DescribeTags.html +describe_tags_input_tests(_) -> + Tests = + [?_workspaces_test( + {"DescribeTags example request", + ?_f(erlcloud_workspaces:describe_tags([{resource_id, "ws-c8wvb67p1"}])), " +{ + \"ResourceId\": \"ws-c8wvb67p1\" +}" + }) + ], + + Response = " +{ + \"TagList\": [ + { + \"Key\": \"testkey\", + \"Value\": \"testvalue\" + } + ] +}", + input_tests(Response, Tests). + + +describe_tags_output_tests(_) -> + Tests = + [?_workspaces_test( + {"DescribeTags example response", " +{ + \"TagList\": [ + { + \"Key\": \"testkey\", + \"Value\": \"testvalue\" + } + ] +}", + {ok,[#workspaces_tag{ + key = <<"testkey">>, + value = <<"testvalue">> + }]} + })], + output_tests(?_f(erlcloud_workspaces:describe_tags([{out, record}])), Tests). + +%% DescribeWorkspaces test based on the API examples: +%% https://docs.aws.amazon.com/workspaces/latest/api/API_DescribeWorkspaces.html +describe_workspaces_input_tests(_) -> + Tests = + [?_workspaces_test( + {"DescribeWorkspaces example request", + ?_f(erlcloud_workspaces:describe_workspaces([{bundle_id, "wsb-clj85qzj2"}, + {directory_id, "d-93671bd1a2"}, + {limit, 5}, + {next_token, "next-page"}, + {user_name, <<"root">>}, + {workspace_ids, [<<"ws-c8wvb67p1">>, <<"ws-c8wvb67p2">>]}])), " +{ + \"BundleId\": \"wsb-clj85qzj2\", + \"DirectoryId\": \"d-93671bd1a2\", + \"Limit\": 5, + \"NextToken\": \"next-page\", + \"UserName\": \"root\", + \"WorkspaceIds\": [\"ws-c8wvb67p1\", \"ws-c8wvb67p2\"] + +}" + }) + ], + + Response = " +{ + \"Workspaces\": [{ + \"BundleId\": \"wsb-clj85qzj2\", + \"ComputerName\": \"A-3AK2EEJBJC1MA\", + \"DirectoryId\": \"d-93671bd1a2\", + \"IpAddress\": \"172.16.0.121\", + \"ModificationStates\": [], + \"State\": \"STOPPED\", + \"SubnetId\": \"subnet-08b608fe3a2ecc504\", + \"UserName\": \"root1\", + \"UserRealm\": \"corp.amazonworkspaces.com\", + \"WorkspaceId\": \"ws-c8wvb67pa\", + \"WorkspaceProperties\": { + \"ComputeTypeName\": \"STANDARD\", + \"RecycleMode\": \"DISABLED\", + \"RootVolumeSizeGib\": 80, + \"RunningMode\": \"AUTO_STOP\", + \"RunningModeAutoStopTimeoutInMinutes\": 60, + \"UserVolumeSizeGib\":50 + } + }] +}", + input_tests(Response, Tests). + + +describe_workspaces_output_tests(_) -> + Tests = + [?_workspaces_test( + {"DescribeWorkspaces example response", " +{ + \"Workspaces\": [{ + \"BundleId\": \"wsb-clj85qzj2\", + \"ComputerName\": \"A-3AK2EEJBJC1MA\", + \"DirectoryId\": \"d-93671bd1a2\", + \"IpAddress\": \"172.16.0.121\", + \"ModificationStates\": [], + \"State\": \"STOPPED\", + \"SubnetId\": \"subnet-08b608fe3a2ecc504\", + \"UserName\": \"root\", + \"UserRealm\": \"corp.amazonworkspaces.com\", + \"WorkspaceId\": \"ws-c8wvb67pa\", + \"WorkspaceProperties\": { + \"ComputeTypeName\": \"STANDARD\", + \"RecycleMode\": \"DISABLED\", + \"RootVolumeSizeGib\": 80, + \"RunningMode\": \"AUTO_STOP\", + \"RunningModeAutoStopTimeoutInMinutes\": 60, + \"UserVolumeSizeGib\":50 + } + }] +}", + {ok,#describe_workspaces{next_token = undefined, + workspaces = [ + #workspace{ + bundle_id = <<"wsb-clj85qzj2">>, + computer_name = <<"A-3AK2EEJBJC1MA">>, + directory_id = <<"d-93671bd1a2">>,error_code = undefined, + error_message = undefined,ip_address = <<"172.16.0.121">>, + modification_states = [], + root_volume_encryption_enabled = undefined, + state = <<"STOPPED">>, + subnet_id = <<"subnet-08b608fe3a2ecc504">>, + user_name = <<"root">>, + user_volume_encryption_enabled = undefined, + volume_encryption_key = undefined, + workspace_id = <<"ws-c8wvb67pa">>, + workspace_properties = #workspace_properties{ + computer_type_name = <<"STANDARD">>, + root_volume_size_gib = 80,running_mode = <<"AUTO_STOP">>, + running_mode_auto_stop_timeout_in_minutes = 60, + user_volume_size_gib = 50}}]}} + })], + output_tests(?_f(erlcloud_workspaces:describe_workspaces([{out, record}])), Tests). + +%% DescribeWorkspaceDirectories test based on the API examples: +%% https://docs.aws.amazon.com/workspaces/latest/api/API_DescribeWorkspaceDirectories.html +describe_workspace_directories_input_tests(_) -> + Tests = + [?_workspaces_test( + {"DescribeWorkspaceDirectories example request", + ?_f(erlcloud_workspaces:describe_workspace_directories([ + {directory_ids, ["d-93671bd1a1", "d-93671bd1a2"]}, + {limit, 5}, + {next_token, "next-page"}])), " +{ + \"DirectoryIds\": [\"d-93671bd1a1\", \"d-93671bd1a2\"], + \"Limit\": 5, + \"NextToken\": \"next-page\" + +}" + }) + ], + + Response = " +{ + \"Directories\": [ + { + \"DirectoryId\": \"d-93671bd1a1\", + \"Alias\": \"d-93671bd1a1\", + \"DirectoryName\": \"corp.amazonworkspaces.com\", + \"RegistrationCode\": \"wsdub+SATMNS\", + \"SubnetIds\": [ + \"subnet-02d62b601c121d0cd\", + \"subnet-08b608fe3a2ecc505\" + ], + \"DnsIpAddresses\": [ + \"172.16.1.36\", + \"172.16.0.49\" + ], + \"CustomerUserName\": \"Administrator\", + \"IamRoleId\": \"arn:aws:iam::352283894008:role/workspaces_DefaultRole\", + \"DirectoryType\": \"SIMPLE_AD\", + \"WorkspaceSecurityGroupId\": \"sg-064d3dbf40db978bd\", + \"State\": \"REGISTERED\", + \"WorkspaceCreationProperties\": { + \"EnableWorkDocs\": true, + \"EnableInternetAccess\": true, + \"UserEnabledAsLocalAdministrator\": true, + \"EnableMaintenanceMode\": true + }, + \"WorkspaceAccessProperties\": { + \"DeviceTypeWindows\": \"ALLOW\", + \"DeviceTypeOsx\": \"ALLOW\", + \"DeviceTypeWeb\": \"DENY\", + \"DeviceTypeIos\": \"ALLOW\", + \"DeviceTypeAndroid\": \"ALLOW\", + \"DeviceTypeChromeOs\": \"ALLOW\", + \"DeviceTypeZeroClient\": \"ALLOW\" + }, + \"Tenancy\": \"SHARED\", + \"SelfservicePermissions\": { + \"RestartWorkspace\": \"ENABLED\", + \"IncreaseVolumeSize\": \"ENABLED\", + \"ChangeComputeType\": \"ENABLED\", + \"SwitchRunningMode\": \"ENABLED\", + \"RebuildWorkspace\": \"ENABLED\" + } + } + ] +}", + input_tests(Response, Tests). + + +describe_workspace_directories_output_tests(_) -> + Tests = + [?_workspaces_test( + {"DescribeWorkspaceDirectories example response", " +{ + \"Directories\": [ + { + \"DirectoryId\": \"d-93671bd1a1\", + \"Alias\": \"d-93671bd1a1\", + \"DirectoryName\": \"corp.amazonworkspaces.com\", + \"RegistrationCode\": \"wsdub+SATMNS\", + \"SubnetIds\": [ + \"subnet-02d62b601c121d0cd\", + \"subnet-08b608fe3a2ecc505\" + ], + \"DnsIpAddresses\": [ + \"172.16.1.36\", + \"172.16.0.49\" + ], + \"CustomerUserName\": \"Administrator\", + \"IamRoleId\": \"arn:aws:iam::352283111111:role/workspaces_DefaultRole\", + \"DirectoryType\": \"SIMPLE_AD\", + \"WorkspaceSecurityGroupId\": \"sg-064d3dbf40db978bd\", + \"State\": \"REGISTERED\", + \"WorkspaceCreationProperties\": { + \"EnableWorkDocs\": true, + \"EnableInternetAccess\": true, + \"UserEnabledAsLocalAdministrator\": true, + \"EnableMaintenanceMode\": true + }, + \"WorkspaceAccessProperties\": { + \"DeviceTypeWindows\": \"ALLOW\", + \"DeviceTypeOsx\": \"ALLOW\", + \"DeviceTypeWeb\": \"DENY\", + \"DeviceTypeIos\": \"ALLOW\", + \"DeviceTypeAndroid\": \"ALLOW\", + \"DeviceTypeChromeOs\": \"ALLOW\", + \"DeviceTypeZeroClient\": \"ALLOW\" + }, + \"Tenancy\": \"SHARED\", + \"SelfservicePermissions\": { + \"RestartWorkspace\": \"ENABLED\", + \"IncreaseVolumeSize\": \"ENABLED\", + \"ChangeComputeType\": \"ENABLED\", + \"SwitchRunningMode\": \"ENABLED\", + \"RebuildWorkspace\": \"ENABLED\" + } + } + ] +}", + {ok,#describe_workspace_directories{ + next_token = undefined, + workspace_directories = + [#workspace_directory{ + alias = <<"d-93671bd1a1">>, + customer_user_name = <<"Administrator">>, + directory_id = <<"d-93671bd1a1">>, + directory_name = <<"corp.amazonworkspaces.com">>, + directory_type = <<"SIMPLE_AD">>, + dns_ip_address = [<<"172.16.1.36">>,<<"172.16.0.49">>], + iam_role_id = + <<"arn:aws:iam::352283111111:role/workspaces_DefaultRole">>, + ip_group_ids = undefined, + registration_code = <<"wsdub+SATMNS">>, + selfservice_permissions = + #workspaces_selfservice_permissions{ + change_compute_type = <<"ENABLED">>, + increase_volume_size = <<"ENABLED">>, + rebuild_workspace = <<"ENABLED">>, + restart_workspace = <<"ENABLED">>, + switch_running_mode = <<"ENABLED">>}, + state = <<"REGISTERED">>, + subnet_ids = + [<<"subnet-02d62b601c121d0cd">>, + <<"subnet-08b608fe3a2ecc505">>], + tenancy = <<"SHARED">>, + workspace_access_properties = + #workspace_access_properties{ + device_type_android = <<"ALLOW">>, + device_type_chrome_os = <<"ALLOW">>, + device_type_ios = <<"ALLOW">>,device_type_osx = <<"ALLOW">>, + device_type_web = <<"DENY">>, + device_type_windows = <<"ALLOW">>, + device_type_zero_client = <<"ALLOW">>}, + workspace_creation_properties = + #workspace_creation_properties{ + custom_security_group_id = undefined,default_ou = undefined, + enable_internet_access = true, + enable_maintenance_mode = true,enable_work_docs = true, + user_enabled_as_local_administrator = true}, + workspace_security_group_id = <<"sg-064d3dbf40db978bd">>}]}} + })], + output_tests(?_f(erlcloud_workspaces:describe_workspace_directories([{out, record}])), Tests). + + +%%%=================================================================== +%%% Input test helpers +%%%=================================================================== + +-type expected_body() :: string(). + +sort_json([{_, _} | _] = Json) -> + %% Value is an object + SortedChildren = [{K, sort_json(V)} || {K,V} <- Json], + lists:keysort(1, SortedChildren); +sort_json([_|_] = Json) -> + %% Value is an array + [sort_json(I) || I <- Json]; +sort_json(V) -> + V. + +%% verifies that the parameters in the body match the expected parameters +-spec validate_body(binary(), expected_body()) -> ok. +validate_body(Body, Expected) -> + Want = sort_json(jsx:decode(list_to_binary(Expected), [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), + case Want =:= Actual of + true -> ok; + false -> + ?debugFmt("~nEXPECTED~n~p~nACTUAL~n~p~n", [Want, Actual]) + end, + ?assertEqual(Want, Actual). + + +%% returns the mock of the erlcloud_httpc function input tests expect to be called. +%% Validates the request body and responds with the provided response. +-spec input_expect(string(), expected_body()) -> fun(). +input_expect(Response, Expected) -> + fun(_Url, post, _Headers, Body, _Timeout, _Config) -> + validate_body(Body, Expected), + {ok, {{200, "OK"}, [], list_to_binary(Response)}} + end. + + +%% input_test converts an input_test specifier into an eunit test generator +-type input_test_spec() :: {pos_integer(), {fun(), expected_body()} | {string(), fun(), expected_body()}}. +-spec input_test(string(), input_test_spec()) -> tuple(). +input_test(Response, {Line, {Description, Fun, Expected}}) + when is_list(Description) -> + {Description, + {Line, + fun() -> + meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), + erlcloud_workspaces:configure(string:copies("A", 20), string:copies("a", 40)), + Fun() + end}}. + + +%% input_tests converts a list of input_test specifiers into an eunit test generator +-spec input_tests(string(), [input_test_spec()]) -> [tuple()]. +input_tests(Response, Tests) -> + [input_test(Response, Test) || Test <- Tests]. + +%%%=================================================================== +%%% Output test helpers +%%%=================================================================== + +%% returns the mock of the erlcloud_httpc function output tests expect to be called. +-spec output_expect(binary()) -> fun(). +output_expect(Response) -> + fun(_Url, post, _Headers, _Body, _Timeout, _Config) -> + {ok, {{200, "OK"}, [], Response}} + end. + +%% output_test converts an output_test specifier into an eunit test generator +-type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string(), term()}}. +-spec output_test(fun(), output_test_spec()) -> tuple(). +output_test(Fun, {Line, {Description, Response, Result}}) -> + {Description, + {Line, + fun() -> + meck:expect(erlcloud_httpc, request, output_expect(list_to_binary(Response))), + erlcloud_workspaces:configure(string:copies("A", 20), string:copies("a", 40)), + Actual = Fun(), + case Result =:= Actual of + true -> ok; + false -> + ?debugFmt("~nEXPECTED~n~p~nACTUAL~n~p~n", [Result, Actual]) + end, + ?assertEqual(Result, Actual) + end}}. + + +%% output_tests converts a list of output_test specifiers into an eunit test generator +-spec output_tests(fun(), [output_test_spec()]) -> [term()]. +output_tests(Fun, Tests) -> + [output_test(Fun, Test) || Test <- Tests]. From 6a337beae67c475a5dfb7ad2a0cd2a0d46db3a29 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Tue, 12 Jan 2021 14:45:40 +0000 Subject: [PATCH 201/310] Allow for configurable metadata host and port --- src/erlcloud.app.src | 3 ++- src/erlcloud_ec2_meta.erl | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/erlcloud.app.src b/src/erlcloud.app.src index 6d834d26f..3599241a6 100644 --- a/src/erlcloud.app.src +++ b/src/erlcloud.app.src @@ -21,7 +21,8 @@ {env, [ % Example [{<<"kinesis">>, [<<"myAZ1.amazonaws.com">>, <<"myAZ2.amazonaws.com">>]}] % or via ENV [{<<"kinesis">>, {env, "KINESIS_VPC_ENDPOINTS"}] - {services_vpc_endpoints, []} + {services_vpc_endpoints, []}, + {ec2_meta_host_port, "169.254.169.254:80"} % allows for an alternative instance metadata service ]}, {licenses, ["MIT"]}, {links, [{"Github", "https://github.com/erlcloud/erlcloud"}]} diff --git a/src/erlcloud_ec2_meta.erl b/src/erlcloud_ec2_meta.erl index b40ed4422..aaae48c70 100644 --- a/src/erlcloud_ec2_meta.erl +++ b/src/erlcloud_ec2_meta.erl @@ -25,12 +25,13 @@ get_instance_metadata(Config) -> %% @doc Retrieve the instance meta data for the instance this code is running on. Will fail if not an EC2 instance. %% %% This convenience function will retrieve the instance id from the AWS metadata available at -%% http://169.254.169.254/latest/meta-data/* +%% http:///latest/meta-data/* %% ItemPath allows fetching specific pieces of metadata. +%% defaults to 169.254.169.254 %% %% get_instance_metadata(ItemPath, Config) -> - MetaDataPath = "http://169.254.169.254/latest/meta-data/" ++ ItemPath, + MetaDataPath = "http://" ++ ec2_meta_host_port() ++ "/latest/meta-data/" ++ ItemPath, erlcloud_aws:http_body(erlcloud_httpc:request(MetaDataPath, get, [], <<>>, erlcloud_aws:get_timeout(Config), Config)). @@ -44,11 +45,12 @@ get_instance_user_data() -> %% @doc Retrieve the user data for the instance this code is running on. Will fail if not an EC2 instance. %% %% This convenience function will retrieve the user data the instance was started with, i.e. what's available at -%% http://169.254.169.254/latest/user-data +%% http:///latest/user-data +%% defaults to 169.254.169.254 %% %% get_instance_user_data(Config) -> - UserDataPath = "http://169.254.169.254/latest/user-data/", + UserDataPath = "http://" ++ ec2_meta_host_port() ++ "/latest/user-data/", erlcloud_aws:http_body(erlcloud_httpc:request(UserDataPath, get, [], <<>>, erlcloud_aws:get_timeout(Config), Config)). @@ -66,6 +68,13 @@ get_instance_dynamic_data(Config) -> %%%--------------------------------------------------------------------------- get_instance_dynamic_data(ItemPath, Config) -> - DynamicDataPath = "http://169.254.169.254/latest/dynamic/" ++ ItemPath, + DynamicDataPath = "http://" ++ ec2_meta_host_port() ++ "/latest/dynamic/" ++ ItemPath, erlcloud_aws:http_body(erlcloud_httpc:request(DynamicDataPath, get, [], <<>>, erlcloud_aws:get_timeout(Config), Config)). +%%%------------------------------------------------------------------------------ +%%% Internal functions. +%%%------------------------------------------------------------------------------ + +ec2_meta_host_port() -> + {ok, EC2MetaHostPort} = application:get_env(erlcloud, ec2_meta_host_port), + EC2MetaHostPort. From 548d7756d96facb2d5139d72212c79d263c7e537 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Sun, 31 Jan 2021 16:02:14 +0000 Subject: [PATCH 202/310] Broaden API expectations to proper returned values --- src/erlcloud_cloudwatch_logs.erl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 5c932df30..ae631dc01 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -486,7 +486,8 @@ log_stream_order_by(last_event_time) -> <<"LastEventTime">>. -spec get_query_results(QueryId, Options) -> Results when QueryId :: string(), Options :: [{out, map}], - Results :: {ok, query_results()} | {error, erlcloud_aws:httpc_result_error()}. + Results :: {ok, query_results() | AWSAPIReturn} | {error, erlcloud_aws:httpc_result_error()}, + AWSAPIReturn :: [{binary(), term()}]. % as per #API_GetQueryResults_ResponseSyntax get_query_results(QueryId, Options) -> get_query_results(QueryId, Options, default_config()). @@ -494,7 +495,8 @@ get_query_results(QueryId, Options) -> when QueryId :: string(), Options :: [{out, map}], Config :: aws_config(), - Results :: {ok, query_results()} | {error, erlcloud_aws:httpc_result_error()}. + Results :: {ok, query_results() | AWSAPIReturn} | {error, erlcloud_aws:httpc_result_error()}, + AWSAPIReturn :: [{binary(), term()}]. % as per #API_GetQueryResults_ResponseSyntax get_query_results(QueryId, Options, Config) -> Result0 = cw_request(Config, "GetQueryResults", [{<<"queryId">>, QueryId}]), Out = proplists:get_value(out, Options, undefined), From 665287916198eec79d62e467e51c59780d8b5a10 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Thu, 4 Feb 2021 14:05:08 +0000 Subject: [PATCH 203/310] Properly interpret returned CloudWatch log events (in the map-based interface) --- src/erlcloud_cloudwatch_logs.erl | 19 +++++++++++-------- test/erlcloud_cloudwatch_logs_tests.erl | 8 ++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index ae631dc01..262837a13 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -58,8 +58,8 @@ | unknown. -export_type([query_status/0]). --type query_results() :: #{ results := [#{ field := string(), - value := string() }], +-type query_results() :: #{ results := [[#{ field := string(), + value := string() }]], statistics := #{ bytes_scanned := float(), records_matched := float(), records_scanned := float() }, @@ -516,14 +516,17 @@ get_query_results(QueryId, Options, Config) -> -spec results_from_get_query_results(In) -> Out when In :: [[[{binary(), binary()}]]], - Out :: [#{ field := string(), - value := string() }]. + Out :: [[#{ field := string(), + value := string() }]]. results_from_get_query_results([]) -> []; -results_from_get_query_results([In]) -> - lists:map(fun (Result) -> - #{ field => binary_to_list(proplists:get_value(<<"field">>, Result)), - value => binary_to_list(proplists:get_value(<<"value">>, Result)) } +results_from_get_query_results(In) -> + lists:map(fun (Results) -> + lists:map(fun (Result) -> + #{ field => binary_to_list(proplists:get_value(<<"field">>, Result)), + value => binary_to_list(proplists:get_value(<<"value">>, Result)) } + end, + Results) end, In). diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index c6c24e5d3..f83bb5d20 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -614,10 +614,10 @@ get_query_results_output_tests(_) -> {<<"recordsMatched">>, 360851.0}, {<<"recordsScanned">>, 610956.0}]}, {<<"status">>, <<"Complete">>}]), - {ok, #{ results => [#{ field => "LogEvent1-field1-name", - value => "LogEvent1-field1-value" }, - #{ field => "LogEvent1-field2-name", - value => "LogEvent1-field2-value" }], + {ok, #{ results => [[#{ field => "LogEvent1-field1-name", + value => "LogEvent1-field1-value" }, + #{ field => "LogEvent1-field2-name", + value => "LogEvent1-field2-value" }]], statistics => #{ bytes_scanned => 81349723.0, records_matched => 360851.0, records_scanned => 610956.0 }, From 162fb5cc7bd21accd732edc60025f96f20962202 Mon Sep 17 00:00:00 2001 From: Yakov Kozlov Date: Fri, 5 Feb 2021 15:16:27 +0300 Subject: [PATCH 204/310] Improve SQS throttling detection --- src/erlcloud_aws.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 4e114fea0..eee87558b 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -251,7 +251,7 @@ do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, Config, response_status = Status} = Request) when %% Retry for 400, Bad Request is needed due to Amazon %% returns it in case of throttling - Status == 400; Status == 429 -> + Status == 400; Status == 403; Status == 429 -> ShouldRetry = is_throttling_error_response(Request), Request#aws_request{should_retry = ShouldRetry}; (#aws_request{response_type = error} = Request) -> @@ -1217,7 +1217,7 @@ is_throttling_error_response(RequestResponse) -> error_type = aws, response_body = RespBody} = RequestResponse, - case binary:match(RespBody, <<"Throttling">>) of + case binary:match(RespBody, <<"Throttl">>) of nomatch -> false; _ -> From 45e3764fe4a12fd133811f894dcb8ec9263fd096 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Sat, 6 Feb 2021 04:22:32 +0000 Subject: [PATCH 205/310] Increase performance and readability --- src/erlcloud_cloudwatch_logs.erl | 26 +++++++++++-------------- test/erlcloud_cloudwatch_logs_tests.erl | 8 ++++---- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 262837a13..47642141c 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -58,8 +58,8 @@ | unknown. -export_type([query_status/0]). --type query_results() :: #{ results := [[#{ field := string(), - value := string() }]], +-type query_results() :: #{ results := [[#{ field := binary(), + value := binary() }]], statistics := #{ bytes_scanned := float(), records_matched := float(), records_scanned := float() }, @@ -514,21 +514,17 @@ get_query_results(QueryId, Options, Config) -> Result0 end. --spec results_from_get_query_results(In) -> Out - when In :: [[[{binary(), binary()}]]], - Out :: [[#{ field := string(), - value := string() }]]. +-spec results_from_get_query_results(ResultRows) -> Out + when ResultRows :: [[[{binary(), binary()}]]], + Out :: [[#{ field := binary(), + value := binary() }]]. results_from_get_query_results([]) -> []; -results_from_get_query_results(In) -> - lists:map(fun (Results) -> - lists:map(fun (Result) -> - #{ field => binary_to_list(proplists:get_value(<<"field">>, Result)), - value => binary_to_list(proplists:get_value(<<"value">>, Result)) } - end, - Results) - end, - In). +results_from_get_query_results(ResultRows) -> + [ [#{ field => proplists:get_value(<<"field">>, ResultField), + value => proplists:get_value(<<"value">>, ResultField) } + || ResultField <- ResultRow ] + || ResultRow <- ResultRows ]. -spec status_from_get_query_results(In) -> Out when In :: binary(), diff --git a/test/erlcloud_cloudwatch_logs_tests.erl b/test/erlcloud_cloudwatch_logs_tests.erl index f83bb5d20..7a81d6910 100644 --- a/test/erlcloud_cloudwatch_logs_tests.erl +++ b/test/erlcloud_cloudwatch_logs_tests.erl @@ -614,10 +614,10 @@ get_query_results_output_tests(_) -> {<<"recordsMatched">>, 360851.0}, {<<"recordsScanned">>, 610956.0}]}, {<<"status">>, <<"Complete">>}]), - {ok, #{ results => [[#{ field => "LogEvent1-field1-name", - value => "LogEvent1-field1-value" }, - #{ field => "LogEvent1-field2-name", - value => "LogEvent1-field2-value" }]], + {ok, #{ results => [[#{ field => <<"LogEvent1-field1-name">>, + value => <<"LogEvent1-field1-value">> }, + #{ field => <<"LogEvent1-field2-name">>, + value => <<"LogEvent1-field2-value">> }]], statistics => #{ bytes_scanned => 81349723.0, records_matched => 360851.0, records_scanned => 610956.0 }, From cb1d94f7faf707d7703f351e8170ce3202887cde Mon Sep 17 00:00:00 2001 From: Vitor Andriotti Date: Wed, 10 Feb 2021 16:44:00 -0300 Subject: [PATCH 206/310] Add erlcloud_ssm module (AWS Systems Manager support) * Update erlcloud with an erlcloud_ssm module, and support GetParameter, GetParametersByPath, GetParameters, PutParameter, and DeleteParameter. * Add unit tests for erlcloud_ssm --- README.md | 1 + include/erlcloud_aws.hrl | 3 + include/erlcloud_ssm.hrl | 43 +++ src/erlcloud_aws.erl | 5 +- src/erlcloud_ssm.erl | 620 ++++++++++++++++++++++++++++++++++++ test/erlcloud_ssm_tests.erl | 443 ++++++++++++++++++++++++++ 6 files changed, 1114 insertions(+), 1 deletion(-) create mode 100644 include/erlcloud_ssm.hrl create mode 100644 src/erlcloud_ssm.erl create mode 100644 test/erlcloud_ssm_tests.erl diff --git a/README.md b/README.md index afe370d85..81481e588 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ Service APIs implemented: - Simple Notification Service (SNS) - Web Application Firewall (WAF) - AWS Cost and Usage Report API +- AWS Systems Manager (SSM) - and more to come The majority of API functions have been implemented. diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index ee72c9ebf..b8229954c 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -121,6 +121,9 @@ sm_scheme="https://"::string(), sm_host="secretsmanager.us-east-1.amazonaws.com"::string(), sm_port=443::non_neg_integer(), + ssm_scheme="https://"::string(), + ssm_host="ssm.us-east-1.amazonaws.com"::string(), + ssm_port=443::non_neg_integer(), guardduty_scheme="https://"::string(), guardduty_host="guardduty.us-east-1.amazonaws.com"::string(), guardduty_port=443::non_neg_integer(), diff --git a/include/erlcloud_ssm.hrl b/include/erlcloud_ssm.hrl new file mode 100644 index 000000000..7a757f4cc --- /dev/null +++ b/include/erlcloud_ssm.hrl @@ -0,0 +1,43 @@ +-ifndef(erlcloud_ssm_hrl). +-define(erlcloud_ssm_hrl, 0). + +-include("erlcloud.hrl"). + +%%%------------------------------------------------------------------------------ +%% +%% Common data types +%% +%%%------------------------------------------------------------------------------ + +-record(ssm_parameter, { + arn :: undefined | binary(), + data_type :: undefined | binary(), + last_modified_date :: undefined | float(), + name :: undefined | binary(), + selector :: undefined | binary(), + source_result :: undefined | binary(), + type :: undefined | binary(), + value :: undefined | binary(), + version :: undefined | non_neg_integer() +}). + +-record(ssm_get_parameter, { + parameter :: undefined | #ssm_parameter{} +}). + +-record(ssm_get_parameters, { + invalid_parameters :: undefined | list(binary()), + parameters :: undefined | [#ssm_parameter{}] +}). + +-record(ssm_get_parameters_by_path, { + next_token :: undefined | binary(), + parameters :: undefined | [#ssm_parameter{}] +}). + +-record(ssm_put_parameter, { + tier :: undefined | binary(), + version :: undefined | non_neg_integer() +}). + +-endif. diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index eee87558b..d5f22b373 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -784,7 +784,10 @@ service_config( <<"application_autoscaling">>, Region, Config ) -> Config#aws_config{application_autoscaling_host = Host}; service_config( <<"workspaces">> = Service, Region, Config ) -> Host = service_host(Service, Region), - Config#aws_config{workspaces_host = Host}. + Config#aws_config{workspaces_host = Host}; +service_config( <<"ssm">> = Service, Region, Config ) -> + Host = service_host(Service, Region), + Config#aws_config{ssm_host = Host}. %%%--------------------------------------------------------------------------- diff --git a/src/erlcloud_ssm.erl b/src/erlcloud_ssm.erl new file mode 100644 index 000000000..ba15b5227 --- /dev/null +++ b/src/erlcloud_ssm.erl @@ -0,0 +1,620 @@ +%% @doc +%% An Erlang interface to AWS Systems Manager (SSM). +%% +%% Output is in the form of `{ok, Value}' or `{error, Reason}'. The +%% format of `Value' is controlled by the `out' option, which defaults +%% to `json'. The possible values are: +%% +%% * `json' - The output from Systems Manager as processed by `jsx:decode' +%% with no further manipulation. +%% +%% * `record' - A record containing all the information from the +%% Systems Manager. +%% +%% * `map' - Same output of `json` opt but in a map formatting. +%% +%% Systems Manager errors are returned in the form `{error, {ErrorCode, Message}}' +%% where `ErrorCode' and 'Message' are both binary +%% strings. + +%% See the unit tests for additional usage examples beyond what are +%% provided for each function. +%% +%% @end + +-module(erlcloud_ssm). + +-include("erlcloud_aws.hrl"). +-include("erlcloud_ssm.hrl"). + +%%% Library initialization. +-export([configure/2, configure/3, configure/4, new/2, new/3, new/4]). + +-define(API_VERSION, "20150408"). +-define(OUTPUT_CHOICES, [json, record, map]). + +-export([ + get_parameter/1, get_parameter/2, + get_parameters/1, get_parameters/2, + get_parameters_by_path/1, get_parameters_by_path/2, + put_parameter/1, put_parameter/2, + delete_parameter/1, delete_parameter/2 +]). + +-export_type([ + get_parameter_opt/0, get_parameter_opts/0, + get_parameters_by_path_opt/0, get_parameters_by_path_opts/0, + get_parameters_opt/0, get_parameters_opts/0, + put_parameter_opt/0, put_parameter_opts/0, + delete_parameter_opt/0, delete_parameter_opts/0 + ]). + +%%%------------------------------------------------------------------------------ +%% Library initialization. +%%%------------------------------------------------------------------------------ + +-spec new(string(), string()) -> aws_config(). + +new(AccessKeyID, SecretAccessKey) -> + #aws_config{ + access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey + }. + +-spec new(string(), string(), string()) -> aws_config(). + +new(AccessKeyID, SecretAccessKey, Host) -> + #aws_config{ + access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey, + ssm_host=Host + }. + +-spec new(string(), string(), string(), non_neg_integer()) -> aws_config(). + +new(AccessKeyID, SecretAccessKey, Host, Port) -> + #aws_config{ + access_key_id=AccessKeyID, + secret_access_key=SecretAccessKey, + ssm_host=Host, + ssm_port=Port + }. + +-spec configure(string(), string()) -> ok. + +configure(AccessKeyID, SecretAccessKey) -> + put(aws_config, new(AccessKeyID, SecretAccessKey)), + ok. + +-spec configure(string(), string(), string()) -> ok. + +configure(AccessKeyID, SecretAccessKey, Host) -> + put(aws_config, new(AccessKeyID, SecretAccessKey, Host)), + ok. + +-spec configure(string(), string(), string(), non_neg_integer()) -> ok. + +configure(AccessKeyID, SecretAccessKey, Host, Port) -> + put(aws_config, new(AccessKeyID, SecretAccessKey, Host, Port)), + ok. + +default_config() -> + erlcloud_aws:default_config(). + + +%%%------------------------------------------------------------------------------ +%% Shared types +%%%------------------------------------------------------------------------------ + +-type string_param() :: binary() | string(). + +-type json_pair() :: {binary() | atom(), jsx:json_term()}. +-type json_return() :: {ok, jsx:json_term()} | {error, term()}. +-type ssm_return(Record) :: {ok, jsx:json_term() | Record } | {error, term()}. +-type decode_fun() :: fun((jsx:json_term(), decode_opts()) -> tuple()). + +%%%------------------------------------------------------------------------------ +%% Shared Options +%%%------------------------------------------------------------------------------ +-type out_type() :: json | record | map. +-type out_opt() :: {out, out_type()}. +-type property() :: proplists:property(). + +-type aws_opts() :: [json_pair()]. +-type ssm_opts() :: [out_opt()]. +-type opts() :: {aws_opts(), ssm_opts()}. + +-spec verify_ssm_opt(atom(), term()) -> ok. +verify_ssm_opt(out, Value) -> + case lists:member(Value, ?OUTPUT_CHOICES) of + true -> + ok; + false -> + error({erlcloud_ssm, {invalid_opt, {out, Value}}}) + end; +verify_ssm_opt(Name, Value) -> + error({erlcloud_ssm, {invalid_opt, {Name, Value}}}). + +-type opt_table_entry() :: {atom(), binary(), fun((_) -> jsx:json_term())}. +-type opt_table() :: [opt_table_entry()]. +-spec opt_folder(opt_table(), property(), opts()) -> opts(). +opt_folder(_, {_, undefined}, Opts) -> + %% ignore options set to undefined + Opts; +opt_folder(Table, {Name, Value}, {AwsOpts, EcsOpts}) -> + case lists:keyfind(Name, 1, Table) of + {Name, Key, ValueFun} -> + {[{Key, ValueFun(Value)} | AwsOpts], EcsOpts}; + false -> + verify_ssm_opt(Name, Value), + {AwsOpts, [{Name, Value} | EcsOpts]} + end. + +-spec opts(opt_table(), proplist()) -> opts(). +opts(Table, Opts) when is_list(Opts) -> + %% remove duplicate options + Opts1 = lists:ukeysort(1, proplists:unfold(Opts)), + lists:foldl(fun(Opt, A) -> opt_folder(Table, Opt, A) end, {[], []}, Opts1); +opts(_, _) -> + error({erlcloud_ssm, opts_not_list}). + +%%%------------------------------------------------------------------------------ +%% Shared Decoders +%%%------------------------------------------------------------------------------ + +-type decode_opt() :: {typed, boolean()}. +-type decode_opts() :: [decode_opt()]. +-type record_desc() :: {tuple(), field_table()}. + +-spec id(X) -> X. +id(X) -> X. + +-spec id(X, decode_opts()) -> X. +id(X, _) -> X. + +-type field_table() :: [{binary(), pos_integer(), + fun((jsx:json_term(), decode_opts()) -> term())}]. + +-spec decode_folder(field_table(), json_pair(), decode_opts(), tuple()) -> tuple(). +decode_folder(Table, {Key, Value}, Opts, A) -> + case lists:keyfind(Key, 1, Table) of + {Key, Index, ValueFun} -> + setelement(Index, A, ValueFun(Value, Opts)); + false -> + A + end. + + +-spec decode_record(record_desc(), jsx:json_term(), decode_opts()) -> tuple(). +decode_record({Record, _}, [{}], _) -> + %% jsx returns [{}] for empty objects + Record; +decode_record({Record, Table}, Json, Opts) -> + lists:foldl(fun(Pair, A) -> decode_folder(Table, Pair, Opts, A) end, Record, Json). + +%%%------------------------------------------------------------------------------ +%% Output +%%%------------------------------------------------------------------------------ +-spec out(json_return(), decode_fun(), ssm_opts()) + -> {ok, jsx:json_term() | tuple()} | + {simple, term()} | + {error, term()}. +out({error, Reason}, _, _) -> + {error, Reason}; +out({ok, Json}, Decode, Opts) -> + case proplists:get_value(out, Opts, json) of + json -> + {ok, Json}; + record -> + {ok, Decode(Json, [])}; + map -> + {ok, erlcloud_util:proplists_to_map(Json)} + end. + +%%%------------------------------------------------------------------------------ +%% Shared Records +%%%------------------------------------------------------------------------------ + +-spec parameter_record() -> record_desc(). +parameter_record() -> + {#ssm_parameter{}, + [ + {<<"ARN">>, #ssm_parameter.arn, fun id/2}, + {<<"DataType">>, #ssm_parameter.data_type, fun id/2}, + {<<"LastModifiedDate">>, #ssm_parameter.last_modified_date, fun id/2}, + {<<"Name">>, #ssm_parameter.name, fun id/2}, + {<<"Selector">>, #ssm_parameter.selector, fun id/2}, + {<<"SourceResult">>, #ssm_parameter.source_result, fun id/2}, + {<<"Type">>, #ssm_parameter.type, fun id/2}, + {<<"Value">>, #ssm_parameter.value, fun id/2}, + {<<"Version">>, #ssm_parameter.version, fun id/2} + ] + }. + +-spec get_parameter_record() -> record_desc(). +get_parameter_record() -> + {#ssm_get_parameter{}, + [ + {<<"Parameter">>, #ssm_get_parameter.parameter, fun decode_parameter/2} + ] + }. + +-spec get_parameters_record() -> record_desc(). +get_parameters_record() -> + {#ssm_get_parameters{}, + [ + {<<"InvalidParameters">>, #ssm_get_parameters.invalid_parameters, fun id/2}, + {<<"Parameters">>, #ssm_get_parameters.parameters, fun decode_parameters/2} + ] + }. + +-spec get_parameters_by_path_record() -> record_desc(). +get_parameters_by_path_record() -> + {#ssm_get_parameters_by_path{}, + [ + {<<"NextToken">>, #ssm_get_parameters_by_path.next_token, fun id/2}, + {<<"Parameters">>, #ssm_get_parameters_by_path.parameters, fun decode_parameters/2} + ] + }. + +-spec put_parameter_record() -> record_desc(). +put_parameter_record() -> + {#ssm_put_parameter{}, + [ + {<<"Tier">>, #ssm_put_parameter.tier, fun id/2}, + {<<"Version">>, #ssm_put_parameter.version, fun id/2} + ] + }. + +decode_parameter(V, Opts) -> + decode_record(parameter_record(), V, Opts). + +decode_parameters(V, Opts) -> + [decode_record(parameter_record(), I, Opts) || I <- V]. + +decode_get_parameter(V, Opts) -> + decode_record(get_parameter_record(), V, Opts). + +decode_get_parameters(V, Opts) -> + decode_record(get_parameters_record(), V, Opts). + +decode_get_parameters_by_path(V, Opts) -> + decode_record(get_parameters_by_path_record(), V, Opts). + +decode_put_parameter(V, Opts) -> + decode_record(put_parameter_record(), V, Opts). + +%%%------------------------------------------------------------------------------ +%% AWS Systems Manager API Functions +%%%------------------------------------------------------------------------------ + +%%%------------------------------------------------------------------------------ +%% GetParameter +%%%------------------------------------------------------------------------------ +-type get_parameter_opt() :: {name, string_param()} | {with_decryption, boolean()} | + out_opt(). +-type get_parameter_opts() :: [get_parameter_opt()]. + +-spec get_parameter_opts() -> opt_table(). +get_parameter_opts() -> + [ + {name, <<"Name">>, fun encode_json_value/1}, + {with_decryption, <<"WithDecryption">>, fun encode_json_value/1} + ]. + +-spec get_parameter(Opts :: get_parameter_opts()) -> ssm_return(#ssm_get_parameter{}). +get_parameter(Opts) -> + get_parameter(Opts, default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% SSM API +%% [https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParameter.html] +%% +%% ===Example=== +%% +%% Get information about a parameter by using the parameter name. +%% +%% ` +%% {ok, Parameter} = erlcloud_ssm:get_parameter([{name, "some_parameter"}, {out, json}]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec get_parameter(Opts :: get_parameter_opts(), Config :: aws_config()) -> ssm_return(#ssm_get_parameter{}). +get_parameter(Opts, #aws_config{} = Config) -> + {AwsOpts, SSMOpts} = opts(get_parameter_opts(), Opts), + Return = ssm_request( + Config, + "GetParameter", + AwsOpts), + out(Return, fun(Json, UOpts) -> + decode_get_parameter(Json, UOpts) + end, + SSMOpts). + +%%%------------------------------------------------------------------------------ +%% GetParameters +%%%------------------------------------------------------------------------------ +-type get_parameters_opt() :: {names, [string_param()]} | {with_decryption, boolean()} | + out_opt(). +-type get_parameters_opts() :: [get_parameters_opt()]. + +-spec get_parameters_opts() -> opt_table(). +get_parameters_opts() -> + [ + {names, <<"Names">>, fun encode_json_value/1}, + {with_decryption, <<"WithDecryption">>, fun encode_json_value/1} + ]. + +-spec get_parameters(Opts :: get_parameters_opts()) -> ssm_return(#ssm_get_parameters{}). +get_parameters(Opts) -> + get_parameters(Opts, default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% SSM API +%% [https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParameters.html] +%% +%% ===Example=== +%% +%% Get information about parameters by using the parameters' names. +%% +%% ` +%% {ok, Parameter} = erlcloud_ssm:get_parameters([{names, ["some_parameter_1", "some_parameter_2"]}]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec get_parameters(Opts :: get_parameters_opts(), Config :: aws_config()) -> ssm_return(#ssm_get_parameters{}). +get_parameters(Opts, #aws_config{} = Config) -> + {AwsOpts, SSMOpts} = opts(get_parameters_opts(), Opts), + Return = ssm_request( + Config, + "GetParameters", + AwsOpts), + out(Return, fun(Json, UOpts) -> + decode_get_parameters(Json, UOpts) + end, + SSMOpts). + +%%%------------------------------------------------------------------------------ +%% GetParametersByPath +%%%------------------------------------------------------------------------------ +-type get_parameters_by_path_filter_opt() :: {key, string_param()} | {option, string_param()} | + {values, [string_param()]}. +-type get_parameters_by_path_filter_opts() :: [get_parameters_by_path_filter_opt()]. +-type get_parameters_by_path_opt() :: {max_results, non_neg_integer()} | {next_token, string_param()} | + {parameter_filters, get_parameters_by_path_filter_opts()} | {path, string_param()} | + {recursive, boolean()} | {with_decryption, boolean()} | + out_opt(). +-type get_parameters_by_path_opts() :: [get_parameters_by_path_opt()]. + +-spec get_parameters_by_path_opts() -> opt_table(). +get_parameters_by_path_opts() -> + [ + {max_results, <<"MaxResults">>, fun id/1}, + {next_token, <<"NextToken">>, fun encode_json_value/1}, + {parameter_filters, <<"ParameterFilters">>, fun encode_json_parameter_filters_value/1}, + {path, <<"Path">>, fun encode_json_value/1}, + {recursive, <<"Recursive">>, fun encode_json_value/1}, + {with_decryption, <<"WithDecryption">>, fun encode_json_value/1} + ]. + +-spec get_parameters_by_path(Opts :: get_parameters_by_path_opts()) -> ssm_return(#ssm_get_parameters_by_path{}). +get_parameters_by_path(Opts) -> + get_parameters_by_path(Opts, default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% SSM API +%% [https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParametersByPath.html] +%% +%% ===Example=== +%% +%% Retrieve information about one or more parameters in a specific hierarchy. +%% +%% ` +%% {ok, Parameters} = erlcloud_ssm:get_parameters_by_path([{path, "/desired/path"}]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec get_parameters_by_path(Opts :: get_parameters_by_path_opts(), Config :: aws_config()) -> ssm_return(#ssm_get_parameters_by_path{}). +get_parameters_by_path(Opts, #aws_config{} = Config) -> + {AwsOpts, SSMOpts} = opts(get_parameters_by_path_opts(), Opts), + Return = ssm_request( + Config, + "GetParametersByPath", + AwsOpts), + out(Return, fun(Json, UOpts) -> + decode_get_parameters_by_path(Json, UOpts) + end, + SSMOpts). + +%%%------------------------------------------------------------------------------ +%% PutParameter +%%%------------------------------------------------------------------------------ +-type tag_parameter_opt() :: {key, string_param()} | {value, string_param()}. +-type tag_parameter_opts() :: [tag_parameter_opt()]. + +-type put_parameter_opt() :: {allowed_pattern, string_param()} | {data_type, string_param()} | + {description, string_param()} | {key_id, string_param()} | + {name, string_param()} | {overwrite, boolean()} | + {policies, string_param()} | {tags, [tag_parameter_opts()]} | + {tier, string_param()} | {type, string_param()} | + {value, string_param()} | out_opt(). +-type put_parameter_opts() :: [put_parameter_opt()]. + +-spec put_parameter_opts() -> opt_table(). +put_parameter_opts() -> + [ + {allowed_pattern, <<"AllowedPattern">>, fun encode_json_value/1}, + {data_type, <<"DataType">>, fun encode_json_value/1}, + {description, <<"Description">>, fun encode_json_value/1}, + {key_id, <<"KeyId">>, fun encode_json_value/1}, + {name, <<"Name">>, fun encode_json_value/1}, + {overwrite, <<"Overwrite">>, fun encode_json_value/1}, + {policies, <<"Policies">>, fun encode_json_value/1}, + {tags, <<"Tags">>, fun encode_json_tags_value/1}, + {tier, <<"Tier">>, fun encode_json_value/1}, + {type, <<"Type">>, fun encode_json_value/1}, + {value, <<"Value">>, fun encode_json_value/1} + ]. + +-spec put_parameter(Opts :: put_parameter_opts()) -> ssm_return(#ssm_put_parameter{}). +put_parameter(Opts) -> + put_parameter(Opts, default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% SSM API +%% [https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_PutParameter.html] +%% +%% ===Example=== +%% +%% Add a parameter to the system. +%% +%% ` +%% {ok, Parameter} = erlcloud_ssm:put_parameter([{name, <<"password">>}, {value, <<"myP@ssw0rd">>}, {type, <<"String">>}]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec put_parameter(Opts :: put_parameter_opts(), Config :: aws_config()) -> ssm_return(#ssm_put_parameter{}). +put_parameter(Opts, #aws_config{} = Config) -> + {AwsOpts, SSMOpts} = opts(put_parameter_opts(), Opts), + Return = ssm_request( + Config, + "PutParameter", + AwsOpts), + out(Return, fun(Json, UOpts) -> + decode_put_parameter(Json, UOpts) + end, + SSMOpts). + +%%%------------------------------------------------------------------------------ +%% DeleteParameter +%%%------------------------------------------------------------------------------ +-type delete_parameter_opt() :: {name, string_param()} | out_opt(). +-type delete_parameter_opts() :: [delete_parameter_opt()]. + +-spec delete_parameter_opts() -> opt_table(). +delete_parameter_opts() -> + [ + {name, <<"Name">>, fun encode_json_value/1} + ]. + +-spec delete_parameter(Opts :: delete_parameter_opts()) -> ok | {error, term()}. +delete_parameter(Opts) -> + delete_parameter(Opts, default_config()). + +%%%------------------------------------------------------------------------------ +%% @doc +%% SSM API +%% [https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_DeleteParameter.html] +%% +%% ===Example=== +%% +%% Delete a parameter from the system. +%% +%% ` +%% ok = erlcloud_ssm:delete_parameter([{name, <<"password">>}]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec delete_parameter(Opts :: delete_parameter_opts(), Config :: aws_config()) -> ok | {error, term()}. +delete_parameter(Opts, #aws_config{} = Config) -> + {AwsOpts, _} = opts(delete_parameter_opts(), Opts), + case ssm_request(Config, "DeleteParameter", AwsOpts) of + {ok, _} -> ok; + {error, _} = Error -> Error + end. + +%%%------------------------------------------------------------------------------ +%% Internal Functions +%%%------------------------------------------------------------------------------ +ssm_request(Config, Operation, Body) -> + case erlcloud_aws:update_config(Config) of + {ok, Config1} -> + ssm_request_impl(Config1, Operation, Body); + {error, Reason} -> + {error, Reason} + end. + +ssm_request_impl(Config, Operation, Body) -> + Payload = case Body of + [] -> <<"{}">>; + _ -> jsx:encode(lists:flatten(Body)) + end, + Headers = headers(Config, Operation, Payload), + Request = #aws_request{service = ssm, + uri = uri(Config), + method = post, + request_headers = Headers, + request_body = Payload}, + case erlcloud_aws:request_to_return(erlcloud_retry:request(Config, Request, fun ssm_result_fun/1)) of + {ok, {_RespHeaders, <<>>}} -> {ok, []}; + {ok, {_RespHeaders, RespBody}} -> {ok, jsx:decode(RespBody, [{return_maps, false}])}; + {error, _} = Error -> Error + end. + +-spec ssm_result_fun(Request :: aws_request()) -> aws_request(). +ssm_result_fun(#aws_request{response_type = ok} = Request) -> + Request; +ssm_result_fun(#aws_request{response_type = error, + error_type = aws, + response_status = Status} = Request) when Status >= 500 -> + Request#aws_request{should_retry = true}; +ssm_result_fun(#aws_request{response_type = error, error_type = aws} = Request) -> + Request#aws_request{should_retry = false}. + +headers(Config, Operation, Body) -> + Headers = [{"host", Config#aws_config.ssm_host}, + {"x-amz-target", lists:append(["AmazonSSM.", Operation])}, + {"content-type", "application/x-amz-json-1.1"}], + Region = erlcloud_aws:aws_region_from_host(Config#aws_config.ssm_host), + erlcloud_aws:sign_v4_headers(Config, Headers, Body, Region, "ssm"). + +uri(#aws_config{ssm_scheme = Scheme, ssm_host = Host} = Config) -> + lists:flatten([Scheme, Host, port_spec(Config)]). + +port_spec(#aws_config{ssm_port=80}) -> + ""; +port_spec(#aws_config{ssm_port=Port}) -> + [":", erlang:integer_to_list(Port)]. + +encode_json_value(undefined) -> undefined; +encode_json_value(true) -> true; +encode_json_value(false) -> false; +encode_json_value(L) when is_list(L), is_list(hd(L)) -> [encode_json_value(V) || V <- L]; +encode_json_value(L) when is_list(L), is_binary(hd(L)) -> [encode_json_value(V) || V <- L]; +encode_json_value(L) when is_list(L) -> list_to_binary(L); +encode_json_value(B) when is_binary(B) -> B; +encode_json_value(A) when is_atom(A) -> atom_to_binary(A, latin1). + +encode_json_tags_value(Tags) -> + encode_json_tags_value(Tags, []). + +encode_json_tags_value([], Acc) -> + Acc; +encode_json_tags_value([Tag|Tags], Acc) -> + encode_json_tags_value(Tags, [encode_json_tag_value(Tag)|Acc]). + +encode_json_tag_value(Tag) -> + encode_json_tag_value(Tag, []). + +encode_json_tag_value([], Acc) -> + Acc; +encode_json_tag_value([{key, Key}|Tags], Acc) -> + encode_json_tag_value(Tags, [{<<"Key">>, encode_json_value(Key)}|Acc]); +encode_json_tag_value([{value, Key}|Tags], Acc) -> + encode_json_tag_value(Tags, [{<<"Value">>, encode_json_value(Key)}|Acc]). + +encode_json_parameter_filters_value(ParameterFilters) -> + encode_json_parameter_filters_value(ParameterFilters, []). + +encode_json_parameter_filters_value([], Acc) -> + Acc; +encode_json_parameter_filters_value([{key, Key}|Filters], Acc) -> + encode_json_parameter_filters_value(Filters, [{<<"Key">>, encode_json_value(Key)}|Acc]); +encode_json_parameter_filters_value([{option, Option}|Filters], Acc) -> + encode_json_parameter_filters_value(Filters, [{<<"Option">>, encode_json_value(Option)}|Acc]); +encode_json_parameter_filters_value([{values, Values}|Filters], Acc) -> + encode_json_parameter_filters_value(Filters, [{<<"Values">>, encode_json_value(Values)}|Acc]). diff --git a/test/erlcloud_ssm_tests.erl b/test/erlcloud_ssm_tests.erl new file mode 100644 index 000000000..967ace427 --- /dev/null +++ b/test/erlcloud_ssm_tests.erl @@ -0,0 +1,443 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +-module(erlcloud_ssm_tests). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("include/erlcloud_ssm.hrl"). + +%% Unit tests for erlcloud_ssm. +%% These tests work by using meck to mock erlcloud_httpc. There are two classes of test: input and output. +%% +%% Input tests verify that different function args produce the desired JSON request. +%% An input test list provides a list of funs and the JSON that is expected to result. +%% +%% Output tests verify that the http response produces the correct return from the fun. +%% An output test lists provides a list of response bodies and the expected return. + +%% The _ssm_test macro provides line number annotation to a test, similar to _test, but doesn't wrap in a fun +-define(_ssm_test(T), {?LINE, T}). +%% The _f macro is a terse way to wrap code in a fun. Similar to _test but doesn't annotate with a line number +-define(_f(F), fun() -> F end). + +-export([validate_body/2]). + +%%%=================================================================== +%%% Test entry points +%%%=================================================================== +operation_test_() -> + {foreach, + fun start/0, + fun stop/1, + [ + fun get_parameter_input_tests/1, + fun get_parameter_output_tests/1, + fun get_parameters_input_tests/1, + fun get_parameters_output_tests/1, + fun get_parameters_by_path_input_tests/1, + fun get_parameters_by_path_output_tests/1, + fun put_parameter_input_tests/1, + fun put_parameter_output_tests/1, + fun delete_parameter_input_tests/1, + fun delete_parameter_output_tests/1 + ] + }. + +start() -> + meck:new(erlcloud_httpc), + ok. + + +stop(_) -> + meck:unload(erlcloud_httpc). + +%%%=================================================================== +%% Actual test specifiers +%%%=================================================================== + +%% GetParameter test based on the API examples: +%% https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParameter.html +get_parameter_input_tests(_) -> + Tests = + [?_ssm_test( + {"GetParameter example request", + ?_f(erlcloud_ssm:get_parameter([{name, "/root/parameter_1"}])), " +{ + \"Name\": \"/root/parameter_1\" +}" + }) + ], + + Response = " +{ + \"Parameter\": [ + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.057, + \"Name\": \"/root/parameter_1\", + \"Type\": \"String\", + \"Value\": \"testvalue\", + \"Version\": 1 + } + ] +}", + input_tests(Response, Tests). + + +get_parameter_output_tests(_) -> + Tests = + [?_ssm_test( + {"GetParameter example response", " +{ + \"Parameter\": + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.057, + \"Name\": \"/root/parameter_1\", + \"Type\": \"String\", + \"Value\": \"testvalue\", + \"Version\": 1 + } +}", + {ok, #ssm_get_parameter{parameter = #ssm_parameter{arn = <<"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1">>, + data_type = <<"text">>, last_modified_date = 1612970194.057, + name = <<"/root/parameter_1">>, selector = undefined, + source_result = undefined, type = <<"String">>, + value = <<"testvalue">>, version = 1}}} + })], + output_tests(?_f(erlcloud_ssm:get_parameter([{name, "/root/parameter_1"}, {out, record}])), Tests). + + +%% GetParameters test based on the API examples: +%% https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParameters.html +get_parameters_input_tests(_) -> + Tests = + [?_ssm_test( + {"GetParameters example request", + ?_f(erlcloud_ssm:get_parameters([{names, ["/root/parameter_1", "/root/secured_parameter", "some_invalid_param"]}, + {with_decryption, true}])), " +{ + \"Names\": [\"/root/parameter_1\", \"/root/secured_parameter\", \"some_invalid_param\"], + \"WithDecryption\": true +}" + }) + ], + + Response = " +{ + \"InvalidParameters\": [\"some_invalid_param\"], + \"Parameters\": [ + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.057, + \"Name\": \"/root/parameter_1\", + \"Type\": \"String\", + \"Value\": \"testvalue\", + \"Version\": 1 + }, + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/secured_parameter\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.31, + \"Name\": \"/root/secured_parameter\", + \"Type\": \"SecureString\", + \"Value\": \"MyP@ssw0rd\", + \"Version\": 1 + } + ] +}", + input_tests(Response, Tests). + + +get_parameters_output_tests(_) -> + Tests = + [?_ssm_test( + {"GetParameters example response", " +{ + \"InvalidParameters\": [\"some_invalid_param\"], + \"Parameters\": [ + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.057, + \"Name\": \"/root/parameter_1\", + \"Type\": \"String\", + \"Value\": \"testvalue\", + \"Version\": 1 + }, + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/secured_parameter\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.31, + \"Name\": \"/root/secured_parameter\", + \"Type\": \"SecureString\", + \"Value\": \"MyP@ssw0rd\", + \"Version\": 1 + } + ] +}", + {ok, #ssm_get_parameters{invalid_parameters = [<<"some_invalid_param">>], + parameters = [#ssm_parameter{arn = <<"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1">>, + data_type = <<"text">>, last_modified_date = 1612970194.057, + name = <<"/root/parameter_1">>, selector = undefined, + source_result = undefined, type = <<"String">>, + value = <<"testvalue">>, version = 1}, + #ssm_parameter{arn = <<"arn:aws:ssm:us-west-2:000000000000:parameter/root/secured_parameter">>, + data_type = <<"text">>, last_modified_date = 1612970194.31, + name = <<"/root/secured_parameter">>, selector = undefined, + source_result = undefined, type = <<"SecureString">>, + value = <<"MyP@ssw0rd">>, + version = 1}]}} + })], + output_tests(?_f(erlcloud_ssm:get_parameters([{names, ["/root/parameter_1", "/root/secured_parameter", "some_invalid_param"]}, + {with_decryption, true}, + {out, record}])), Tests). + + +%% GetParametersByPath test based on the API examples: +%% https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_GetParametersByPath.html +get_parameters_by_path_input_tests(_) -> + Tests = + [?_ssm_test( + {"GetParametersByPath example request", + ?_f(erlcloud_ssm:get_parameters_by_path([{max_results, 2}, + {path, <<"/root">>}, + {recursive, true}, + {with_decryption, false}, + {out, record}])), " +{ + \"MaxResults\": 2, + \"Path\": \"/root\", + \"Recursive\": true, + \"WithDecryption\": false, +}" + }) + ], + + Response = " +{ + \"NextToken\": \"AAEAAe2HRCL1+nBkC3kuOVp0r3fpOpPOH+/c+hH5VIdx7vntLivo/iGhhR8yllmtqsCdoNiwS4EQP+QrRDX+T1NT9vN7X2Fj+/Gb/++F089cGZc6/+/OAaqe/==\", + \"Parameters\": [ + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.057, + \"Name\": \"/root/parameter_1\", + \"Type\": \"String\", + \"Value\": \"testvalue\", + \"Version\": 1 + }, + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/secured_parameter\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.31, + \"Name\": \"/root/secured_parameter\", + \"Type\": \"SecureString\", + \"Value\": \"6EzK+k13aK/VKMYk77pyiErjGpaoEDA==\", + \"Version\": 1 + } + ] +}", + input_tests(Response, Tests). + + +get_parameters_by_path_output_tests(_) -> + Tests = + [?_ssm_test( + {"GetParametersByPath example response", " +{ + \"NextToken\": \"AAEAAe2HRCL1+nBkC3kuOVp0r3fpOpPOH+/c+hH5VIdx7vntLivo/iGhhR8yllmtqsCdoNiwS4EQP+QrRDX+T1NT9vN7X2Fj+/Gb/++F089cGZc6/+/OAaqe/==\", + \"Parameters\": [ + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.057, + \"Name\": \"/root/parameter_1\", + \"Type\": \"String\", + \"Value\": \"testvalue\", + \"Version\": 1 + }, + { + \"ARN\": \"arn:aws:ssm:us-west-2:000000000000:parameter/root/secured_parameter\", + \"DataType\": \"text\", + \"LastModifiedDate\": 1612970194.31, + \"Name\": \"/root/secured_parameter\", + \"Type\": \"SecureString\", + \"Value\": \"6EzK+k13aK/VKMYk77pyiErjGpaoEDA==\", + \"Version\": 1 + } + ] +}", + {ok, #ssm_get_parameters_by_path{next_token = <<"AAEAAe2HRCL1+nBkC3kuOVp0r3fpOpPOH+/c+hH5VIdx7vntLivo/iGhhR8yllmtqsCdoNiwS4EQP+QrRDX+T1NT9vN7X2Fj+/Gb/++F089cGZc6/+/OAaqe/==">>, + parameters = [#ssm_parameter{arn = <<"arn:aws:ssm:us-west-2:000000000000:parameter/root/parameter_1">>, + data_type = <<"text">>, last_modified_date = 1612970194.057, + name = <<"/root/parameter_1">>, selector = undefined, + source_result = undefined, type = <<"String">>, + value = <<"testvalue">>, version = 1}, + #ssm_parameter{arn = <<"arn:aws:ssm:us-west-2:000000000000:parameter/root/secured_parameter">>, + data_type = <<"text">>, last_modified_date = 1612970194.31, + name = <<"/root/secured_parameter">>, selector = undefined, + source_result = undefined, type = <<"SecureString">>, + value = <<"6EzK+k13aK/VKMYk77pyiErjGpaoEDA==">>, + version = 1}]}} + })], + output_tests(?_f(erlcloud_ssm:get_parameters_by_path([{max_results, 2}, + {path, <<"/root">>}, + {recursive, true}, + {with_decryption, false}, + {out, record}])), Tests). + + +%% PutParameter test based on the API examples: +%% https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_PutParameter.html +put_parameter_input_tests(_) -> + Tests = + [?_ssm_test( + {"PutParameter example request", + ?_f(erlcloud_ssm:put_parameter([{name, <<"/root/parameter_1">>}, {value, <<"testvalue">>}, {type, <<"String">>}])), " +{ + \"Name\": \"/root/parameter_1\", + \"Type\": \"String\", + \"Value\": \"testvalue\" +}" + }) + ], + + Response = " +{ + \"Tier\": \"Standard\", + \"Version\": 1 +}", + input_tests(Response, Tests). + + +put_parameter_output_tests(_) -> + Tests = + [?_ssm_test( + {"PutParameter example response", " +{ + \"Tier\": \"Standard\", + \"Version\": 1 +}", + {ok, #ssm_put_parameter{tier = <<"Standard">>, + version = 1}} + })], + output_tests(?_f(erlcloud_ssm:put_parameter([{name, <<"/root/parameter_1">>}, {value, <<"testvalue">>}, {type, <<"String">>}, {out, record}])), Tests). + + +%% DeleteParameter test based on the API examples: +%% https://docs.aws.amazon.com/systems-manager/latest/APIReference/API_DeleteParameter.html +delete_parameter_input_tests(_) -> + Tests = + [?_ssm_test( + {"DeleteParameter example request", + ?_f(erlcloud_ssm:delete_parameter([{name, <<"/root/parameter_1">>}])), " +{ + \"Name\": \"/root/parameter_1\" +}" + }) + ], + Response = "{}", + input_tests(Response, Tests). + + +delete_parameter_output_tests(_) -> + Tests = + [?_ssm_test( + {"DeleteParameter example response", "{}", + ok + })], + output_tests(?_f(erlcloud_ssm:delete_parameter([{name, <<"/root/parameter_1">>}])), Tests). + +%%%=================================================================== +%%% Input test helpers +%%%=================================================================== + +-type expected_body() :: string(). + +sort_json([{_, _} | _] = Json) -> + %% Value is an object + SortedChildren = [{K, sort_json(V)} || {K,V} <- Json], + lists:keysort(1, SortedChildren); +sort_json([_|_] = Json) -> + %% Value is an array + [sort_json(I) || I <- Json]; +sort_json(V) -> + V. + +%% verifies that the parameters in the body match the expected parameters +-spec validate_body(binary(), expected_body()) -> ok. +validate_body(Body, Expected) -> + Want = sort_json(jsx:decode(list_to_binary(Expected), [{return_maps, false}])), + Actual = sort_json(jsx:decode(Body, [{return_maps, false}])), + case Want =:= Actual of + true -> ok; + false -> + ?debugFmt("~nEXPECTED~n~p~nACTUAL~n~p~n", [Want, Actual]) + end, + ?assertEqual(Want, Actual). + + +%% returns the mock of the erlcloud_httpc function input tests expect to be called. +%% Validates the request body and responds with the provided response. +-spec input_expect(string(), expected_body()) -> fun(). +input_expect(Response, Expected) -> + fun(_Url, post, _Headers, Body, _Timeout, _Config) -> + validate_body(Body, Expected), + {ok, {{200, "OK"}, [], list_to_binary(Response)}} + end. + + +%% input_test converts an input_test specifier into an eunit test generator +-type input_test_spec() :: {pos_integer(), {fun(), expected_body()} | {string(), fun(), expected_body()}}. +-spec input_test(string(), input_test_spec()) -> tuple(). +input_test(Response, {Line, {Description, Fun, Expected}}) + when is_list(Description) -> + {Description, + {Line, + fun() -> + meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), + erlcloud_ssm:configure(string:copies("A", 20), string:copies("a", 40)), + Fun() + end}}. + + +%% input_tests converts a list of input_test specifiers into an eunit test generator +-spec input_tests(string(), [input_test_spec()]) -> [tuple()]. +input_tests(Response, Tests) -> + [input_test(Response, Test) || Test <- Tests]. + +%%%=================================================================== +%%% Output test helpers +%%%=================================================================== + +%% returns the mock of the erlcloud_httpc function output tests expect to be called. +-spec output_expect(binary()) -> fun(). +output_expect(Response) -> + fun(_Url, post, _Headers, _Body, _Timeout, _Config) -> + {ok, {{200, "OK"}, [], Response}} + end. + +%% output_test converts an output_test specifier into an eunit test generator +-type output_test_spec() :: {pos_integer(), {string(), term()} | {string(), string(), term()}}. +-spec output_test(fun(), output_test_spec()) -> tuple(). +output_test(Fun, {Line, {Description, Response, Result}}) -> + {Description, + {Line, + fun() -> + meck:expect(erlcloud_httpc, request, output_expect(list_to_binary(Response))), + erlcloud_ssm:configure(string:copies("A", 20), string:copies("a", 40)), + Actual = Fun(), + case Result =:= Actual of + true -> ok; + false -> + ?debugFmt("~nEXPECTED~n~p~nACTUAL~n~p~n", [Result, Actual]) + end, + ?assertEqual(Result, Actual) + end}}. + + +%% output_tests converts a list of output_test specifiers into an eunit test generator +-spec output_tests(fun(), [output_test_spec()]) -> [term()]. +output_tests(Fun, Tests) -> + [output_test(Fun, Test) || Test <- Tests]. From d44957830e8308ae8678d9c8328474cb79fc12ce Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Mon, 15 Feb 2021 17:44:13 -0600 Subject: [PATCH 207/310] Export DDB Streams record undynamizer * Export DDB Streams record undynamizer to use with Kinesis Data Streaming for DynamoDB --- src/erlcloud_ddb_streams.erl | 29 +++++++++++++++++ test/erlcloud_ddb_streams_tests.erl | 48 ++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_ddb_streams.erl b/src/erlcloud_ddb_streams.erl index e61c0f041..83575709b 100644 --- a/src/erlcloud_ddb_streams.erl +++ b/src/erlcloud_ddb_streams.erl @@ -65,6 +65,9 @@ list_streams/0, list_streams/1, list_streams/2 ]). +%%% Exported DynamoDB Streams Undynamizers +-export([undynamize_ddb_streams_record/1, undynamize_ddb_streams_record/2]). + -export_type( [aws_region/0, event_id/0, @@ -795,6 +798,32 @@ list_streams(Opts, Config) -> fun(Json, UOpts) -> undynamize_record(list_streams_record(), Json, UOpts) end, DdbStreamsOpts, #ddb_streams_list_streams.streams). + +%%%------------------------------------------------------------------------------ +%%% Exported Undynamizers +%%%------------------------------------------------------------------------------ +-type undynamize_ddb_streams_record_return() :: ddb_streams_return(#ddb_streams_record{}, #ddb_streams_stream_record{}). + +-spec undynamize_ddb_streams_record(json_term()) -> undynamize_ddb_streams_record_return(). +undynamize_ddb_streams_record(Return) -> + undynamize_ddb_streams_record(Return, []). + +%%------------------------------------------------------------------------------ +%% @doc Undynamizes a DynamoDB streams record. +%% This function can be used to undynamize the jsx-decoded Data field of a +%% record retrieved from Kinesis, after enabling Kinesis Data Streaming for +%% a DynamoDB table. +%% +%% Change Data Capture for Kinesis Data Streams with DynamoDB: +%% [http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/kds.html] +%%------------------------------------------------------------------------------ +-spec undynamize_ddb_streams_record(json_term(), ddb_streams_opts()) -> undynamize_ddb_streams_record_return(). +undynamize_ddb_streams_record(Return, Opts) -> + out({ok, Return}, + fun(Json, UOpts) -> undynamize_record(record_record(), Json, UOpts) end, + Opts, #ddb_streams_record.dynamodb). + + %%%------------------------------------------------------------------------------ %%% Request %%%------------------------------------------------------------------------------ diff --git a/test/erlcloud_ddb_streams_tests.erl b/test/erlcloud_ddb_streams_tests.erl index 98150d2e3..735eade87 100644 --- a/test/erlcloud_ddb_streams_tests.erl +++ b/test/erlcloud_ddb_streams_tests.erl @@ -27,7 +27,8 @@ operation_test_() -> fun get_shard_iterator_input_tests/1, fun get_shard_iterator_output_tests/1, fun list_streams_input_tests/1, - fun list_streams_output_tests/1 + fun list_streams_output_tests/1, + fun undynamize_ddb_streams_record_output_tests/1 ]}. start() -> @@ -587,3 +588,48 @@ list_streams_output_tests(_) -> table_name = <<"Forum">>}]}}) ], output_tests(?_f(erlcloud_ddb_streams:list_streams()), Tests). + + +undynamize_ddb_streams_record_output_tests(_) -> + DecodedResponse = jsx:decode( + <<" + { + \"awsRegion\": \"us-west-2\", + \"dynamodb\": { + \"ApproximateCreationDateTime\": 1551727994, + \"Keys\": { + \"ForumName\": {\"S\": \"DynamoDB\"}, + \"Subject\": {\"S\": \"DynamoDB Thread 3\"} + }, + \"SequenceNumber\": \"300000000000000499659\", + \"SizeBytes\": 41, + \"StreamViewType\": \"KEYS_ONLY\" + }, + \"eventID\": \"e2fd9c34eff2d779b297b26f5fef4206\", + \"eventName\": \"INSERT\", + \"eventSource\": \"aws:dynamodb\", + \"eventVersion\": \"1.0\" + }">>, [{return_maps, false}]), + DDBStreamsStreamRecord = #ddb_streams_stream_record{approximate_creation_date_time = 1551727994, + keys = [{<<"ForumName">>, <<"DynamoDB">>}, + {<<"Subject">>, <<"DynamoDB Thread 3">>}], + new_image = undefined, + old_image = undefined, + sequence_number = <<"300000000000000499659">>, + size_bytes = 41, + stream_view_type = keys_only}, + DDBStreamsStreamTypedRecord = DDBStreamsStreamRecord#ddb_streams_stream_record{keys = [{<<"ForumName">>, {s, <<"DynamoDB">>}}, + {<<"Subject">>, {s, <<"DynamoDB Thread 3">>}}]}, + + DDBStreamsRecord = #ddb_streams_record{aws_region = <<"us-west-2">>, + dynamodb = DDBStreamsStreamRecord, + event_id = <<"e2fd9c34eff2d779b297b26f5fef4206">>, + event_name = insert, + event_source = <<"aws:dynamodb">>, + event_version = <<"1.0">>}, + DDBStreamsTypedRecord = DDBStreamsRecord#ddb_streams_record{dynamodb = DDBStreamsStreamTypedRecord}, + + [?_assertEqual({ok, DDBStreamsStreamRecord}, erlcloud_ddb_streams:undynamize_ddb_streams_record(DecodedResponse)), + ?_assertEqual({ok, DDBStreamsStreamRecord}, erlcloud_ddb_streams:undynamize_ddb_streams_record(DecodedResponse, [{out, simple}])), + ?_assertEqual({ok, DDBStreamsRecord}, erlcloud_ddb_streams:undynamize_ddb_streams_record(DecodedResponse, [{out, record}])), + ?_assertEqual({ok, DDBStreamsTypedRecord}, erlcloud_ddb_streams:undynamize_ddb_streams_record(DecodedResponse, [{out, typed_record}]))]. From 1c32f78b82f3cf7cab649591958da725af1d57ef Mon Sep 17 00:00:00 2001 From: Carl Isom III Date: Mon, 1 Mar 2021 13:37:34 -0600 Subject: [PATCH 208/310] Add support for new DeduplicationScope and FifoThroughputLimit sqs attributes --- src/erlcloud_sqs.erl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index d9a023733..8bfaea66e 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -46,7 +46,7 @@ kms_master_key_id | kms_data_key_reuse_period_seconds | approximate_number_of_messages_not_visible | visibility_timeout | created_timestamp | last_modified_timestamp | policy | - queue_arn). + queue_arn | deduplication_scope | fifo_throughput_limit). -type(batch_entry() :: {string(), string()} | {string(), string(), [message_attribute()]} @@ -238,7 +238,9 @@ encode_attribute_name(policy) -> "Policy"; encode_attribute_name(redrive_policy) -> "RedrivePolicy"; encode_attribute_name(kms_master_key_id) -> "KmsMasterKeyId"; encode_attribute_name(kms_data_key_reuse_period_seconds) -> "KmsDataKeyReusePeriodSeconds"; -encode_attribute_name(all) -> "All". +encode_attribute_name(all) -> "All"; +encode_attribute_name(deduplication_scope) -> "DeduplicationScope"; +encode_attribute_name(fifo_throughput_limit) -> "FifoThroughputLimit". decode_attribute_name("MessageRetentionPeriod") -> message_retention_period; @@ -257,13 +259,17 @@ decode_attribute_name("RedrivePolicy") -> redrive_policy; decode_attribute_name("ContentBasedDeduplication") -> content_based_deduplication; decode_attribute_name("KmsMasterKeyId") -> kms_master_key_id; decode_attribute_name("KmsDataKeyReusePeriodSeconds") -> kms_data_key_reuse_period_seconds; -decode_attribute_name("FifoQueue") -> fifo_queue. +decode_attribute_name("FifoQueue") -> fifo_queue; +decode_attribute_name("DeduplicationScope") -> deduplication_scope; +decode_attribute_name("FifoThroughputLimit") -> fifo_throughput_limit. decode_attribute_value("Policy", Value) -> Value; decode_attribute_value("QueueArn", Value) -> Value; decode_attribute_value("RedrivePolicy", Value) -> Value; decode_attribute_value("KmsMasterKeyId", Value) -> Value; +decode_attribute_value("DeduplicationScope", Value) -> Value; +decode_attribute_value("FifoThroughputLimit", Value) -> Value; decode_attribute_value(_, "true") -> true; decode_attribute_value(_, "false") -> false; decode_attribute_value(_, Value) -> list_to_integer(Value). From 76937ee281aa8d4fc30299432aa153ca9c57f078 Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Tue, 2 Mar 2021 14:54:48 -0600 Subject: [PATCH 209/310] erlcloud_aws: support explicit querystring in writes Fixes #690. Refactors `erlcloud_aws:[do_]_aws_request_form_raw` function to take a QueryString distinctly from the Form and _always_ use that as the query string fragment of the URI, never in form body. This allows callers to explicitly force values to go into the query string on POST/PUT method calls where the Form gets stored in the body. Also fixes the logic of `erlcloud_lambda:lambda_request_no_update` to explicitly encode the Body (Form) and QParams to be fully separate, so that QParams, which for lambda always must go in the query string fragment of the URI, is always explicitly passed there. This makes it so that the `erlcloud_lambda:invoke` call, which is a POST call, will pass its `Qualifier` parameter as a query string. --- src/erlcloud_aws.erl | 72 ++++++++++++++++++++------------ src/erlcloud_cloudsearch.erl | 2 +- src/erlcloud_cloudtrail.erl | 2 +- src/erlcloud_cloudwatch_logs.erl | 1 + src/erlcloud_emr.erl | 2 +- src/erlcloud_guardduty.erl | 2 +- src/erlcloud_lambda.erl | 29 +++++++------ src/erlcloud_mes.erl | 2 +- src/erlcloud_mms.erl | 2 +- test/erlcloud_lambda_tests.erl | 29 +++++++++---- 10 files changed, 87 insertions(+), 56 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index d5f22b373..8fb5d86d7 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -8,8 +8,8 @@ aws_request_xml4/6, aws_request_xml4/8, aws_region_from_host/1, aws_request_form/8, - aws_request_form_raw/8, - do_aws_request_form_raw/9, + aws_request_form_raw/9, + do_aws_request_form_raw/10, param_list/2, default_config/0, auto_config/0, auto_config/1, default_config_region/2, default_config_override/1, update_config/1,clear_config/1, clear_expired_configs/0, @@ -211,8 +211,8 @@ aws_request4_no_update(Method, Protocol, Host, Port, Path, Params, Service, -spec aws_request_form(Method :: atom(), Protocol :: undefined | string(), Host :: string(), - Port :: undefined | integer() | string(), Path :: string(), Form :: [string()], - Headers :: list(), Config :: aws_config()) -> {ok, Body :: binary()} | {error, httpc_result_error()}. + Port :: undefined | integer() | string(), Path :: string(), Form :: [string()], + Headers :: list(), Config :: aws_config()) -> {ok, Body :: binary()} | {error, httpc_result_error()}. aws_request_form(Method, Protocol, Host, Port, Path, Form, Headers, Config) -> RequestHeaders = case proplists:is_defined("content-type", Headers) of false -> [{"content-type", ?DEFAULT_CONTENT_TYPE} | Headers]; @@ -222,16 +222,16 @@ aws_request_form(Method, Protocol, Host, Port, Path, Form, Headers, Config) -> undefined -> "https://"; _ -> [Protocol, "://"] end, - aws_request_form_raw(Method, Scheme, Host, Port, Path, list_to_binary(Form), RequestHeaders, Config). + aws_request_form_raw(Method, Scheme, Host, Port, Path, list_to_binary(Form), RequestHeaders, [], Config). -spec aws_request_form_raw(Method :: atom(), Scheme :: string() | [string()], - Host :: string(), Port :: undefined | integer() | string(), - Path :: string(), Form :: iodata(), Headers :: list(), - Config :: aws_config()) -> {ok, Body :: binary()} | {error, httpc_result_error()}. -aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, Config) -> - do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, Config, false). + Host :: string(), Port :: undefined | integer() | string(), + Path :: string(), Form :: iodata(), Headers :: list(), QueryString :: string(), + Config :: aws_config()) -> {ok, Body :: binary()} | {error, httpc_result_error()}. +aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, QueryString, Config) -> + do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, QueryString, Config, false). -do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, Config, ShowRespHeaders) -> +do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, QueryString, Config, ShowRespHeaders) -> URL = case Port of undefined -> [Scheme, Host, Path]; _ -> [Scheme, Host, $:, port_to_str(Port), Path] @@ -258,25 +258,16 @@ do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, Config, Request#aws_request{should_retry = false} end, + {URI, Body} = aws_request_form_uri_and_body(Method, URL, Form, QueryString), + AwsRequest = #aws_request{uri = URI, + method = Method, + request_headers = Headers, + request_body = Body}, + %% Note: httpc MUST be used with {timeout, timeout()} option %% Many timeout related failures is observed at prod env %% when library is used in 24/7 manner - Response = - case Method of - M when M =:= get orelse M =:= head orelse M =:= delete -> - Req = lists:flatten([URL, $?, Form]), - AwsRequest = #aws_request{uri = Req, - method = M, - request_headers = Headers, - request_body = <<>>}, - erlcloud_retry:request(Config, AwsRequest, ResultFun); - _ -> - AwsRequest = #aws_request{uri = lists:flatten(URL), - method = Method, - request_headers = Headers, - request_body = Form}, - erlcloud_retry:request(Config, AwsRequest, ResultFun) - end, + Response = erlcloud_retry:request(Config, AwsRequest, ResultFun), show_headers(ShowRespHeaders, request_to_return(Response)). @@ -1041,6 +1032,33 @@ get_timeout(#aws_config{timeout = undefined}) -> get_timeout(#aws_config{timeout = Timeout}) -> Timeout. +%% Construct the URI and body for an AWS request based on the Method, Form, and +%% QueryString: if the request is a read/delete, join Form and QueryString in +%% the URI and give an empty body; otherwise, pass the Form in the Body and +%% the QueryString in the URI. +aws_request_form_uri_and_body(Method, URL, Form, QueryString) when Method =:= delete; + Method =:= get; + Method =:= head -> + URI = make_uri(URL, join_query_strings([Form, QueryString])), + {URI, <<>>}; +aws_request_form_uri_and_body(_M, URL, Form, QueryString) -> + URI = make_uri(URL, QueryString), + {URI, Form}. + + +%% Given a URL and a QueryString, combine them together appropriately (drop the +%% QueryString if empty). +make_uri(URL, QueryString) when QueryString =:= <<>>; QueryString == [] -> + lists:flatten(URL); +make_uri(URL, QueryString) -> + lists:flatten([URL, $?, QueryString]). + + +%% Join a list of query strings together with `&', filtering out empty ones. +join_query_strings(QueryStrings) -> + lists:join($&, [QS || QS <- QueryStrings, QS /= <<>>, QS /= []]). + + %% Convert an aws_request record to return value as returned by http_headers_body request_to_return(#aws_request{response_type = ok, response_headers = Headers, diff --git a/src/erlcloud_cloudsearch.erl b/src/erlcloud_cloudsearch.erl index 5d4ce1e84..44a36d88e 100644 --- a/src/erlcloud_cloudsearch.erl +++ b/src/erlcloud_cloudsearch.erl @@ -734,7 +734,7 @@ cloudsearch_post_json(Host, Path, Body, case erlcloud_aws:aws_request_form_raw( post, Scheme, Host, Port, Path, Body, [{"content-type", "application/json"} | Headers], - Config) of + [], Config) of {ok, RespBody} -> {ok, jsx:decode(RespBody, [{return_maps, false}])}; {error, Reason} -> diff --git a/src/erlcloud_cloudtrail.erl b/src/erlcloud_cloudtrail.erl index f20dae943..ab073d1f7 100644 --- a/src/erlcloud_cloudtrail.erl +++ b/src/erlcloud_cloudtrail.erl @@ -201,7 +201,7 @@ request_impl(Method, Scheme, Host, Port, Path, Operation, Params, Body, #aws_con case erlcloud_aws:aws_request_form_raw( Method, Scheme, Host, Port, Path, Body, [{"content-type", "application/x-amz-json-1.1"} | Headers], - Config) of + [], Config) of {ok, RespBody} -> case Config#aws_config.cloudtrail_raw_result of true -> {ok, RespBody}; diff --git a/src/erlcloud_cloudwatch_logs.erl b/src/erlcloud_cloudwatch_logs.erl index 47642141c..762422ac1 100644 --- a/src/erlcloud_cloudwatch_logs.erl +++ b/src/erlcloud_cloudwatch_logs.erl @@ -891,6 +891,7 @@ maybe_cw_request({ok, Config}, Action, Params) -> "/", Request, make_request_headers(Config, Action, Request), + [], Config ) ); diff --git a/src/erlcloud_emr.erl b/src/erlcloud_emr.erl index 936d8ffbe..ed716e4df 100644 --- a/src/erlcloud_emr.erl +++ b/src/erlcloud_emr.erl @@ -202,7 +202,7 @@ request_no_update(Action, Json, Scheme, Host, Port, Service, Opts, Cfg) -> Region = erlcloud_aws:aws_region_from_host(Host), Headers = erlcloud_aws:sign_v4_headers(Cfg, H1, ReqBody, Region, Service) ++ H2, case erlcloud_aws:aws_request_form_raw(post, Scheme, Host, Port, - "/", ReqBody, Headers, Cfg) of + "/", ReqBody, Headers, [], Cfg) of {ok, Body} -> case proplists:get_value(out, Opts, json) of raw -> {ok, Body}; _ -> case Body of diff --git a/src/erlcloud_guardduty.erl b/src/erlcloud_guardduty.erl index 834e7e4b8..1f8b8d9e9 100644 --- a/src/erlcloud_guardduty.erl +++ b/src/erlcloud_guardduty.erl @@ -115,7 +115,7 @@ guardduty_request_no_update(Config, Method, Path, Body, QParam) -> Headers = headers(Method, Path, Config, encode_body(Body), QParam), case erlcloud_aws:aws_request_form_raw( Method, Config#aws_config.guardduty_scheme, Config#aws_config.guardduty_host, - Config#aws_config.guardduty_port, Path, Form, Headers, Config) of + Config#aws_config.guardduty_port, Path, Form, Headers, [], Config) of {ok, Data} -> {ok, jsx:decode(Data, [{return_maps, false}])}; E -> diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index 570060ce2..f67e3755d 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -379,7 +379,7 @@ invoke(FunctionName, Payload, Options, Qualifier) when is_binary(Qualifier) -> invoke(FunctionName, Payload, Options, Qualifier, Config = #aws_config{}) -> Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName) ++ "/invocations", QParams = filter_undef([{"Qualifier", Qualifier}]), - lambda_request(Config, post, Path, Options, Payload, QParams). + lambda_request(Config, post, Path, Payload, QParams, Options). %%------------------------------------------------------------------------------ %% ListAliases @@ -722,31 +722,30 @@ base_path() -> "/" ++ ?API_VERSION ++ "/". lambda_request(Config, Method, Path, Body) -> - lambda_request(Config, Method, Path, [], Body, []). + lambda_request(Config, Method, Path, Body, []). + lambda_request(Config, Method, Path, Body, QParams) -> - lambda_request(Config, Method, Path, [], Body, QParams). + lambda_request(Config, Method, Path, Body, QParams, []). -lambda_request(Config, Method, Path, Options, Body, QParam) -> +lambda_request(Config, Method, Path, Body, QParams, Options) -> case erlcloud_aws:update_config(Config) of {ok, Config1} -> - lambda_request_no_update(Config1, Method, Path, Options, Body, QParam); + lambda_request_no_update(Config1, Method, Path, Options, Body, QParams); {error, Reason} -> {error, Reason} end. -lambda_request_no_update(Config, Method, Path, Options, Body, QParam) -> - Form = case encode_body(Body) of - <<>> -> erlcloud_http:make_query_string(QParam); - Value -> Value - end, +lambda_request_no_update(Config, Method, Path, Options, Body0, QParams) -> ShowRespHeaders = proplists:get_value(show_headers, Options, false), RawBody = proplists:get_value(raw_response_body, Options, false), Hdrs0 = proplists:delete(show_headers, Options), Hdrs = proplists:delete(raw_response_body, Hdrs0), - Headers = headers(Method, Path, Hdrs, Config, encode_body(Body), QParam), + Body = encode_body(Body0), + Headers = headers(Method, Path, Hdrs, Config, Body, QParams), + QueryString = erlcloud_http:make_query_string(QParams), case erlcloud_aws:do_aws_request_form_raw( - Method, Config#aws_config.lambda_scheme, Config#aws_config.lambda_host, - Config#aws_config.lambda_port, Path, Form, Headers, Config, ShowRespHeaders) of + Method, Config#aws_config.lambda_scheme, Config#aws_config.lambda_host, + Config#aws_config.lambda_port, Path, Body, Headers, QueryString, Config, ShowRespHeaders) of {ok, RespHeaders, RespBody} -> {ok, RespHeaders, decode_body(RespBody, RawBody)}; {ok, RespBody} -> @@ -771,9 +770,9 @@ encode_body([]) -> encode_body(Body) -> jsx:encode(Body). -headers(Method, Uri, Hdrs, Config, Body, QParam) -> +headers(Method, Uri, Hdrs, Config, Body, QParams) -> Headers = [{"host", Config#aws_config.lambda_host}, {"content-type", "application/json"} | Hdrs], Region = erlcloud_aws:aws_region_from_host(Config#aws_config.lambda_host), erlcloud_aws:sign_v4(Method, Uri, Config, - Headers, Body, Region, "lambda", QParam). + Headers, Body, Region, "lambda", QParams). diff --git a/src/erlcloud_mes.erl b/src/erlcloud_mes.erl index a2e8fe4b8..b14af2e99 100644 --- a/src/erlcloud_mes.erl +++ b/src/erlcloud_mes.erl @@ -169,7 +169,7 @@ mes_request(Config, Operation, Json) -> mes_request_no_update(#aws_config{mes_scheme = Scheme, mes_host = Host, mes_port = Port} = Config, Operation, Json) -> Body = jsx:encode(Json), Headers = headers(Config, Operation, Body), - case erlcloud_aws:aws_request_form_raw(post, Scheme, Host, Port, "/", Body, Headers, Config) of + case erlcloud_aws:aws_request_form_raw(post, Scheme, Host, Port, "/", Body, Headers, [], Config) of {ok, Response} -> {ok, jsx:decode(Response, [{return_maps, false}])}; {error, Reason} -> diff --git a/src/erlcloud_mms.erl b/src/erlcloud_mms.erl index adc937baf..b6c46c793 100644 --- a/src/erlcloud_mms.erl +++ b/src/erlcloud_mms.erl @@ -168,7 +168,7 @@ mms_request(Config, Operation, Json) -> mms_request_no_update(#aws_config{mms_scheme = Scheme, mms_host = Host, mms_port = Port} = Config, Operation, Json) -> Body = jsx:encode(Json), Headers = headers(Config, Operation, Body), - case erlcloud_aws:aws_request_form_raw(post, Scheme, Host, Port, "/", Body, Headers, Config) of + case erlcloud_aws:aws_request_form_raw(post, Scheme, Host, Port, "/", Body, Headers, [], Config) of {ok, Response} -> {ok, jsx:decode(Response, [{return_maps, false}])}; {error, Reason} -> diff --git a/test/erlcloud_lambda_tests.erl b/test/erlcloud_lambda_tests.erl index e8a9e7244..83162e98f 100644 --- a/test/erlcloud_lambda_tests.erl +++ b/test/erlcloud_lambda_tests.erl @@ -25,6 +25,7 @@ mocks() -> mocked_get_function(), mocked_get_function_configuration(), mocked_invoke(), + mocked_invoke_qualifier(), mocked_list_aliases(), mocked_list_event_source_mappings(), mocked_list_function(), @@ -87,7 +88,7 @@ mocked_create_function() -> }. mocked_delete_event_source_mapping() -> { - [?BASE_URL ++ "event-source-mappings/6554f300-551b-46a6-829c-41b6af6022c6?", + [?BASE_URL ++ "event-source-mappings/6554f300-551b-46a6-829c-41b6af6022c6", delete, '_', <<>>, '_', '_'], make_response(<<"{\"BatchSize\":100,\"EventSourceArn\":\"arn:aws:kinesi" "s:us-east-1:352283894008:stream/eholland-test\",\"FunctionArn\":\"arn:aws:lam" @@ -98,7 +99,7 @@ mocked_delete_event_source_mapping() -> }. mocked_get_alias() -> { - [?BASE_URL ++ "functions/name/aliases/aliasName?", + [?BASE_URL ++ "functions/name/aliases/aliasName", get, '_', <<>>, '_', '_'], make_response(<<"{\"AliasArn\":\"arn:aws:lambda:us-east-1:352283894008:" "function:name:aliasName\",\"Description\":\"\",\"FunctionVersion\":\"$LATEST\" @@ -106,7 +107,7 @@ mocked_get_alias() -> }. mocked_get_event_source_mapping() -> { - [?BASE_URL ++ "event-source-mappings/a45b58ec-a539-4c47-929e-174b4dd2d963?", + [?BASE_URL ++ "event-source-mappings/a45b58ec-a539-4c47-929e-174b4dd2d963", get, '_', <<>>, '_', '_'], make_response(<<"{\"BatchSize\":100,\"EventSourceArn\":\"arn:aws:kinesi" "s:us-east-1:352283894008:stream/eholland-test\",\"FunctionArn\":\"arn:aws:lam" @@ -117,7 +118,7 @@ mocked_get_event_source_mapping() -> }. mocked_get_function() -> { - [?BASE_URL ++ "functions/name?", + [?BASE_URL ++ "functions/name", get, '_', <<>>, '_', '_'], make_response(<<"{\"Code\":{\"Location\":\"https://awslambda-us-east-1-" "tasks.s3-us-east-1.amazonaws.com/snapshots/352283894008/name-69237aec-bae9-40" @@ -142,7 +143,7 @@ mocked_get_function() -> }. mocked_get_function_configuration() -> { - [?BASE_URL ++ "functions/name/configuration?", get, '_', <<>>, '_', '_'], + [?BASE_URL ++ "functions/name/configuration", get, '_', <<>>, '_', '_'], make_response(<<"{\"CodeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4Ic" "piPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"FunctionArn\":\"arn:aws:la" "mbda:us-east-1:352283894008:function:name\",\"FunctionName\":\"name\",\"Handl" @@ -158,10 +159,17 @@ mocked_invoke() -> make_response(<<"{\"message\":\"Hello World!\"}">>) }. +mocked_invoke_qualifier() -> + { + [?BASE_URL ++ "functions/name_qualifier/invocations?Qualifier=123", post, + '_', <<"{}">>, '_', '_'], + make_response(<<"{\"message\":\"Hello World!\"}">>) + }. + mocked_list_aliases() -> { - [?BASE_URL ++ "functions/name/aliases?", get, '_', <<>>, '_', '_'], + [?BASE_URL ++ "functions/name/aliases", get, '_', <<>>, '_', '_'], make_response(<<"{\"Aliases\":[{\"AliasArn\":\"arn:aws:lambda:us-east-1" ":352283894008:function:name:aliasName\",\"Description\":\"\",\"FunctionVersio" "n\":\"$LATEST\",\"Name\":\"aliasName\"},{\"AliasArn\":\"arn:aws:lambda:us-eas" @@ -184,7 +192,7 @@ mocked_list_event_source_mappings() -> mocked_list_function() -> { - [?BASE_URL ++ "functions/?", + [?BASE_URL ++ "functions/", get, '_', <<>>, '_', '_'], make_response(<<"{\"Functions\":[{\"CodeSha256\":\"XmLDAZXEkl5KbA8ezZpw" "FU+bjgTXBehUmWGOScl4F2A=\",\"CodeSize\":5561,\"Description\":\"\",\"FunctionA" @@ -265,7 +273,7 @@ mocked_list_function() -> mocked_list_versions_by_function() -> { - [?BASE_URL ++ "functions/name/versions?", get, '_', <<>>, '_', '_'], + [?BASE_URL ++ "functions/name/versions", get, '_', <<>>, '_', '_'], make_response(<<"{\"NextMarker\":null,\"Versions\":[{\"CodeSha256\":\"z" "eoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=\",\"CodeSize\":848,\"Description" "\":\"\",\"FunctionArn\":\"arn:aws:lambda:us-east-1:352283894008:function:name:" @@ -642,6 +650,11 @@ api_tests(_) -> Expected = {ok, <<"{\"message\":\"Hello World!\"}">>}, ?assertEqual(Expected, Result) end, + fun() -> + Result = erlcloud_lambda:invoke(<<"name_qualifier">>, [], [], <<"123">>), + Expected = {ok, [{<<"message">>, <<"Hello World!">>}]}, + ?assertEqual(Expected, Result) + end, fun() -> Result = erlcloud_lambda:list_aliases(<<"name">>), Expected = {ok, [{<<"Aliases">>, From 2c7ce673f76bda5121547e2872d2e07a169b1386 Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Thu, 4 Mar 2021 13:04:47 -0600 Subject: [PATCH 210/310] erlcloud_lambda: Add URL encoding of function name Adds URL encoding of function name before signing the request, to make sure that alias/ARN forms (which have `:` in them) are signed according to AWS spec (canonical URL form). --- src/erlcloud_lambda.erl | 3 ++- test/erlcloud_lambda_tests.erl | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index f67e3755d..4c2747623 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -377,7 +377,8 @@ invoke(FunctionName, Payload, Options, Qualifier) when is_binary(Qualifier) -> Qualifier :: binary()| undefined, Config :: aws_config()) -> return_val(). invoke(FunctionName, Payload, Options, Qualifier, Config = #aws_config{}) -> - Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName) ++ "/invocations", + URLFunctionName = erlcloud_http:url_encode(binary_to_list(FunctionName)), + Path = base_path() ++ "functions/" ++ URLFunctionName ++ "/invocations", QParams = filter_undef([{"Qualifier", Qualifier}]), lambda_request(Config, post, Path, Payload, QParams, Options). diff --git a/test/erlcloud_lambda_tests.erl b/test/erlcloud_lambda_tests.erl index 83162e98f..9231e369b 100644 --- a/test/erlcloud_lambda_tests.erl +++ b/test/erlcloud_lambda_tests.erl @@ -25,6 +25,7 @@ mocks() -> mocked_get_function(), mocked_get_function_configuration(), mocked_invoke(), + mocked_invoke_alias(), mocked_invoke_qualifier(), mocked_list_aliases(), mocked_list_event_source_mappings(), @@ -159,6 +160,13 @@ mocked_invoke() -> make_response(<<"{\"message\":\"Hello World!\"}">>) }. +mocked_invoke_alias() -> + { + [?BASE_URL ++ "functions/name%3Aalias/invocations", post, '_', <<"{}">>, '_', '_'], + make_response(<<"{\"message\":\"Hello World!\"}">>) + }. + + mocked_invoke_qualifier() -> { [?BASE_URL ++ "functions/name_qualifier/invocations?Qualifier=123", post, @@ -650,6 +658,11 @@ api_tests(_) -> Expected = {ok, <<"{\"message\":\"Hello World!\"}">>}, ?assertEqual(Expected, Result) end, + fun() -> + Result = erlcloud_lambda:invoke(<<"name:alias">>, [], [raw_response_body], #aws_config{}), + Expected = {ok, <<"{\"message\":\"Hello World!\"}">>}, + ?assertEqual(Expected, Result) + end, fun() -> Result = erlcloud_lambda:invoke(<<"name_qualifier">>, [], [], <<"123">>), Expected = {ok, [{<<"message">>, <<"Hello World!">>}]}, From 22aa0e18bf72de338b29250ca2b9709cf14fc40b Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Thu, 4 Mar 2021 17:42:33 -0600 Subject: [PATCH 211/310] erlcloud_lambda: Add DOUBLE URL encoding of function name in signature According to [Create a canonical request for Signature Version 4](https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html): > Each path segment must be URI-encoded twice (except for Amazon S3 which only gets URI-encoded once). > Example Canonical URI with encoding > `/documents%2520and%2520settings/` So, we URL-encode all parameters in the URL everywhere in this module, then loose-URL-encode the URI when passing to sign_v4. I also restored `erlcloud_aws: aws_request_form_raw/8` and `erlcloud_aws:do_aws_request_form_raw/9` --- src/erlcloud_aws.erl | 15 +++++++++++++-- src/erlcloud_lambda.erl | 34 ++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 8fb5d86d7..98e6592f0 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -8,8 +8,8 @@ aws_request_xml4/6, aws_request_xml4/8, aws_region_from_host/1, aws_request_form/8, - aws_request_form_raw/9, - do_aws_request_form_raw/10, + aws_request_form_raw/8, aws_request_form_raw/9, + do_aws_request_form_raw/9, do_aws_request_form_raw/10, param_list/2, default_config/0, auto_config/0, auto_config/1, default_config_region/2, default_config_override/1, update_config/1,clear_config/1, clear_expired_configs/0, @@ -224,6 +224,14 @@ aws_request_form(Method, Protocol, Host, Port, Path, Form, Headers, Config) -> end, aws_request_form_raw(Method, Scheme, Host, Port, Path, list_to_binary(Form), RequestHeaders, [], Config). + +-spec aws_request_form_raw(Method :: atom(), Scheme :: string() | [string()], + Host :: string(), Port :: undefined | integer() | string(), + Path :: string(), Form :: iodata(), Headers :: list(), + Config :: aws_config()) -> {ok, Body :: binary()} | {error, httpc_result_error()}. +aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, Config) -> + do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, [], Config, false). + -spec aws_request_form_raw(Method :: atom(), Scheme :: string() | [string()], Host :: string(), Port :: undefined | integer() | string(), Path :: string(), Form :: iodata(), Headers :: list(), QueryString :: string(), @@ -231,6 +239,9 @@ aws_request_form(Method, Protocol, Host, Port, Path, Form, Headers, Config) -> aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, QueryString, Config) -> do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, QueryString, Config, false). +do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, Config, ShowRespHeaders) -> + do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, [], Config, ShowRespHeaders). + do_aws_request_form_raw(Method, Scheme, Host, Port, Path, Form, Headers, QueryString, Config, ShowRespHeaders) -> URL = case Port of undefined -> [Scheme, Host, Path]; diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index 4c2747623..2f10f8564 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -107,7 +107,7 @@ create_alias(FunctionName, FunctionVersion, Options :: proplist(), Config :: aws_config()) -> return_val(). create_alias(FunctionName, FunctionVersion, AliasName, Options, Config) -> - Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName) ++ "/aliases", + Path = base_path() ++ "functions/" ++ url_parameter(FunctionName) ++ "/aliases", Json = [{<<"FunctionVersion">>, FunctionVersion}, {<<"Name">>, AliasName} | Options], @@ -212,7 +212,7 @@ delete_event_source_mapping(Uuid) -> -spec delete_event_source_mapping(Uuid :: binary(), Config :: aws_config()) -> return_val(). delete_event_source_mapping(Uuid, Config) -> - Path = base_path() ++ "event-source-mappings/" ++ binary_to_list(Uuid), + Path = base_path() ++ "event-source-mappings/" ++ url_parameter(Uuid), lambda_request(Config, delete, Path, undefined). @@ -239,7 +239,7 @@ get_alias(FunctionName, Name) -> Config :: aws_config()) -> return_val(). get_alias(FunctionName, Name, Config) -> Path = base_path() ++ "functions/" - ++ binary_to_list(FunctionName) ++ "/aliases/" ++ binary_to_list(Name), + ++ url_parameter(FunctionName) ++ "/aliases/" ++ url_parameter(Name), lambda_request(Config, get, Path, undefined). %%------------------------------------------------------------------------------ @@ -262,7 +262,7 @@ get_event_source_mapping(Uuid) -> -spec get_event_source_mapping(Uuid :: binary(), Config :: aws_config()) -> return_val(). get_event_source_mapping(Uuid, Config) -> - Path = base_path() ++ "event-source-mappings/" ++ binary_to_list(Uuid), + Path = base_path() ++ "event-source-mappings/" ++ url_parameter(Uuid), lambda_request(Config, get, Path, undefined). %%------------------------------------------------------------------------------ @@ -291,7 +291,7 @@ get_function(FunctionName, Config) -> Qualifier :: undefined | binary(), Config :: aws_config()) -> return_val(). get_function(FunctionName, Qualifier, Config) -> - Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName), + Path = base_path() ++ "functions/" ++ url_parameter(FunctionName), QParams = filter_undef([{"Qualifier", Qualifier}]), lambda_request(Config, get, Path, undefined, QParams). @@ -317,7 +317,7 @@ get_function_configuration(FunctionName, Config) -> get_function_configuration(FunctionName, undefined, Config). get_function_configuration(Function, Qualifier, Config) -> - Path = base_path() ++ "functions/" ++ binary_to_list(Function) ++ "/configuration", + Path = base_path() ++ "functions/" ++ url_parameter(Function) ++ "/configuration", QParams = filter_undef([{"Qualifier", Qualifier}]), lambda_request(Config, get, Path, undefined, QParams). @@ -377,8 +377,7 @@ invoke(FunctionName, Payload, Options, Qualifier) when is_binary(Qualifier) -> Qualifier :: binary()| undefined, Config :: aws_config()) -> return_val(). invoke(FunctionName, Payload, Options, Qualifier, Config = #aws_config{}) -> - URLFunctionName = erlcloud_http:url_encode(binary_to_list(FunctionName)), - Path = base_path() ++ "functions/" ++ URLFunctionName ++ "/invocations", + Path = base_path() ++ "functions/" ++ url_parameter(FunctionName) ++ "/invocations", QParams = filter_undef([{"Qualifier", Qualifier}]), lambda_request(Config, post, Path, Payload, QParams, Options). @@ -419,7 +418,7 @@ list_aliases(FunctionName, FunctionVersion, Marker, MaxItems) -> Config :: aws_config()) -> return_val(). list_aliases(FunctionName, FunctionVersion, Marker, MaxItems, Config) -> Path = base_path() ++ "functions/" - ++ binary_to_list(FunctionName) ++ "/aliases", + ++ url_parameter(FunctionName) ++ "/aliases", QParams = filter_undef([{"Marker", Marker}, {"MaxItems", MaxItems}, {"FunctionVersion", FunctionVersion}]), @@ -521,7 +520,7 @@ list_versions_by_function(Function, Marker, MaxItems) -> MaxItems :: integer() | undefined, Config :: aws_config()) -> return_val(). list_versions_by_function(Function, Marker, MaxItems, Config) -> - Path = base_path() ++ "functions/" ++ binary_to_list(Function) ++ "/versions", + Path = base_path() ++ "functions/" ++ url_parameter(Function) ++ "/versions", QParams = filter_undef([{"Marker", Marker}, {"MaxItems", MaxItems}]), lambda_request(Config, get, Path, undefined, QParams). @@ -554,7 +553,7 @@ publish_version(FunctionName, CodeSha, Description) -> Description :: binary() | undefined, Config :: aws_config()) -> return_val(). publish_version(FunctionName, CodeSha, Description, Config) -> - Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName) ++ "/versions", + Path = base_path() ++ "functions/" ++ url_parameter(FunctionName) ++ "/versions", Json = filter_undef([{<<"CodeSha">>, CodeSha}, {<<"Description">>, Description}]), lambda_request(Config, post, Path, Json). @@ -591,7 +590,7 @@ update_alias(FunctionName, AliasName, Description, FunctionVersion) -> Config :: aws_config()) -> return_val(). update_alias(FunctionName, AliasName, Description, FunctionVersion, Config) -> Path = base_path() ++ "functions/" - ++ binary_to_list(FunctionName) ++ "/aliases/" ++ binary_to_list(AliasName), + ++ url_parameter(FunctionName) ++ "/aliases/" ++ url_parameter(AliasName), Json = filter_undef([{"Description", Description}, {"FunctionVersion", FunctionVersion}]), lambda_request(Config, put, Path, Json). @@ -623,7 +622,7 @@ update_event_source_mapping(Uuid, BatchSize, Enabled, FunctionName) -> FunctionName :: binary() | undefined, Config :: aws_config()) -> return_val(). update_event_source_mapping(Uuid, BatchSize, Enabled, FunctionName, Config) -> - Path = base_path() ++ "event-source-mappings/" ++ binary_to_list(Uuid), + Path = base_path() ++ "event-source-mappings/" ++ url_parameter(Uuid), Json = filter_undef([{<<"BatchSize">>, BatchSize}, {<<"Enabled">>, Enabled}, {<<"FunctionName">>, FunctionName}]), @@ -653,7 +652,7 @@ update_function_code(FunctionName, Publish, Code) -> Code :: erlcloud_lambda_code(), Config :: aws_config()) -> return_val(). update_function_code(FunctionName, Publish, Code, Config) -> - Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName) ++ "/code", + Path = base_path() ++ "functions/" ++ url_parameter(FunctionName) ++ "/code", Json = [{<<"Publish">>, Publish} | from_record(Code)], lambda_request(Config, put, Path, Json). @@ -701,7 +700,7 @@ update_function_configuration(FunctionName, Description, Handler, Configuration :: list(tuple()), % JSX json object Config :: aws_config()) -> return_val(). update_function_configuration(FunctionName, Configuration, Config) when is_list(Configuration) -> - Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName) ++ "/configuration", + Path = base_path() ++ "functions/" ++ url_parameter(FunctionName) ++ "/configuration", Json = filter_undef(Configuration), lambda_request(Config, put, Path, Json). @@ -719,6 +718,9 @@ from_record(#erlcloud_lambda_code{s3Bucket = S3Bucket, {<<"ZipFile">>, ZipFile}], filter_undef(List). +url_parameter(Param) -> + erlcloud_http:url_encode(binary_to_list(Param)). + base_path() -> "/" ++ ?API_VERSION ++ "/". @@ -775,5 +777,5 @@ headers(Method, Uri, Hdrs, Config, Body, QParams) -> Headers = [{"host", Config#aws_config.lambda_host}, {"content-type", "application/json"} | Hdrs], Region = erlcloud_aws:aws_region_from_host(Config#aws_config.lambda_host), - erlcloud_aws:sign_v4(Method, Uri, Config, + erlcloud_aws:sign_v4(Method, erlcloud_http:url_encode_loose(Uri), Config, Headers, Body, Region, "lambda", QParams). From 10d014bd6ba93aadc9a1b39af52089c0ac380f3c Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 5 Mar 2021 00:19:42 +0000 Subject: [PATCH 212/310] Allow consumer-defined lhttpc_types -less lhttpc fork to compile without issues --- src/erlcloud_aws.erl | 7 +++---- src/erlcloud_httpc.erl | 13 +++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index d5f22b373..2f3b3ce9b 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -29,7 +29,6 @@ -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). --include_lib("lhttpc/include/lhttpc_types.hrl"). -define(ERLCLOUD_RETRY_TIMEOUT, 10000). -define(GREGORIAN_EPOCH_OFFSET, 62167219200). @@ -43,7 +42,7 @@ -define(AWS_REGION, ["AWS_DEFAULT_REGION", "AWS_REGION"]). %% types --type http_client_result() :: result(). % from lhttpc_types.hrl +-type http_client_result() :: erlcloud_httpc:result(). -type http_client_headers() :: [{string(), string()}]. -type httpc_result_ok() :: {http_client_headers(), binary()}. -type httpc_result_error() :: {http_error, Status :: pos_integer(), StatusLine :: string(), Body :: binary()} @@ -1059,11 +1058,11 @@ request_to_return(#aws_request{response_type = error, {error, {http_error, Status, StatusLine, Body, Headers}}. %% http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html --spec sign_v4_headers(aws_config(), headers(), string() | binary(), string(), string()) -> headers(). +-spec sign_v4_headers(aws_config(), erlcloud_httpc:headers(), string() | binary(), string(), string()) -> erlcloud_httpc:headers(). sign_v4_headers(Config, Headers, Payload, Region, Service) -> sign_v4(post, "/", Config, Headers, Payload, Region, Service, []). --spec sign_v4(atom(), list(), aws_config(), headers(), string() | binary(), string(), string(), list()) -> headers(). +-spec sign_v4(atom(), list(), aws_config(), erlcloud_httpc:headers(), string() | binary(), string(), string(), list()) -> erlcloud_httpc:headers(). sign_v4(Method, Uri, Config, Headers, Payload, Region, Service, QueryParams) -> Date = iso_8601_basic_time(), {PayloadHash, Headers1} = diff --git a/src/erlcloud_httpc.erl b/src/erlcloud_httpc.erl index 0132d3f87..6408e5822 100644 --- a/src/erlcloud_httpc.erl +++ b/src/erlcloud_httpc.erl @@ -25,6 +25,19 @@ {error, any()}). -export_type([request_fun/0]). +% Imported from lhttpc_types.hrl +-type body() :: binary() + | undefined % HEAD request. + | pid(). % When partial_download option is used. + +-type headers() :: [{atom() | string(), iodata()}]. % atom is of type 'Cache-Control' | 'Connection' | 'Date' | ... +-export_type([headers/0]). + +-type result() :: {ok, {{StatusCode :: pos_integer(), StatusMsg :: string()}, headers(), body()}} + | {ok, {pid(), WindowSize :: non_neg_integer() | infinity}} + | {error, atom()}. +-export_type([result/0]). + request(URL, Method, Hdrs, Body, Timeout, #aws_config{http_client = lhttpc} = Config) -> request_lhttpc(URL, Method, Hdrs, Body, Timeout, Config); From b446430faee0c1de34c0d4532cfd0ec2f9ab2d44 Mon Sep 17 00:00:00 2001 From: Andrey Veselov Date: Fri, 5 Mar 2021 12:38:00 +0300 Subject: [PATCH 213/310] DeleteFunction API added --- src/erlcloud_lambda.erl | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index 2f10f8564..edaa90614 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -14,6 +14,7 @@ create_event_source_mapping/4, create_event_source_mapping/5, create_function/6, create_function/7, delete_event_source_mapping/1, delete_event_source_mapping/2, + delete_function/1, delete_function/2, delete_function/3, get_alias/2, get_alias/3, get_event_source_mapping/1, get_event_source_mapping/2, get_function/1, get_function/2, get_function/3, @@ -215,6 +216,34 @@ delete_event_source_mapping(Uuid, Config) -> Path = base_path() ++ "event-source-mappings/" ++ url_parameter(Uuid), lambda_request(Config, delete, Path, undefined). +%%------------------------------------------------------------------------------ +%% DeleteFunction +%%------------------------------------------------------------------------------ + +%%------------------------------------------------------------------------------ +%% @doc +%% Lambda API: +%% [https://docs.aws.amazon.com/lambda/latest/dg/API_DeleteFunction.html] +%% +%% ===Example=== +%% +%%------------------------------------------------------------------------------ +-spec delete_function(FunctionName :: binary()) -> return_val(). +delete_function(FunctionName) -> + delete_function(FunctionName, default_config()). + +-spec delete_function(FunctionName :: binary(), + Config :: aws_config()) -> return_val(). +delete_function(FunctionName, Config) -> + delete_function(FunctionName, undefined, Config). + +-spec delete_function(FunctionName :: binary(), + Qualifier :: undefined | binary(), + Config :: aws_config()) -> return_val(). +delete_function(FunctionName, Qualifier, Config) -> + Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName), + QParams = filter_undef([{"Qualifier", Qualifier}]), + lambda_request(Config, delete, Path, undefined, QParams). %%------------------------------------------------------------------------------ %% GetAlias From a4f36eb09a601cfca36887f65f8d42f50dc54581 Mon Sep 17 00:00:00 2001 From: Andrey Veselov Date: Fri, 5 Mar 2021 12:38:16 +0300 Subject: [PATCH 214/310] DeleteFunction API tests added --- test/erlcloud_lambda_tests.erl | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/test/erlcloud_lambda_tests.erl b/test/erlcloud_lambda_tests.erl index 9231e369b..027546e36 100644 --- a/test/erlcloud_lambda_tests.erl +++ b/test/erlcloud_lambda_tests.erl @@ -20,6 +20,8 @@ mocks() -> mocked_create_event_source_mapping(), mocked_create_function(), mocked_delete_event_source_mapping(), + mocked_delete_function(), + mocked_delete_function_qualifier(), mocked_get_alias(), mocked_get_event_source_mapping(), mocked_get_function(), @@ -98,6 +100,18 @@ mocked_delete_event_source_mapping() -> "ransitionReason\":\"User action\",\"UUID\":\"a45b58ec-a539-4c47-929e-174b4dd2" "d963\"}">>) }. +mocked_delete_function() -> + { + [?BASE_URL ++ "functions/name", + delete, '_', <<>>, '_', '_'], + make_response({204,"No Content"}, <<"">>) + }. +mocked_delete_function_qualifier() -> + { + [?BASE_URL ++ "functions/name_qualifier?Qualifier=123", + delete, '_', <<>>, '_', '_'], + make_response({204,"No Content"}, <<"">>) + }. mocked_get_alias() -> { [?BASE_URL ++ "functions/name/aliases/aliasName", @@ -581,6 +595,18 @@ api_tests(_) -> {<<"UUID">>,<<"a45b58ec-a539-4c47-929e-174b4dd2d963">>}]}, ?assertEqual(Expected, Result) end, + fun() -> + Result = erlcloud_lambda:delete_function( + <<"name">>), + Expected = {ok, []}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:delete_function( + <<"name_qualifier">>, <<"123">>, #aws_config{}), + Expected = {ok, []}, + ?assertEqual(Expected, Result) + end, fun() -> Result = erlcloud_lambda:get_alias(<<"name">>, <<"aliasName">>), Expected = {ok, [{<<"AliasArn">>, @@ -826,4 +852,7 @@ api_tests(_) -> ]. make_response(Value) -> - {ok, {{200, <<"OK">>}, [], Value}}. + make_response({200, <<"OK">>}, Value). + +make_response(Status, Value) -> + {ok, {Status, [], Value}}. From 7e260705d335aefeccdc684d2fe66e4e3fe42f9c Mon Sep 17 00:00:00 2001 From: Andrey Veselov Date: Mon, 15 Mar 2021 12:39:41 +0300 Subject: [PATCH 215/310] Delete function updated with url_parameter --- src/erlcloud_lambda.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index edaa90614..83c2d4cbb 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -241,7 +241,7 @@ delete_function(FunctionName, Config) -> Qualifier :: undefined | binary(), Config :: aws_config()) -> return_val(). delete_function(FunctionName, Qualifier, Config) -> - Path = base_path() ++ "functions/" ++ binary_to_list(FunctionName), + Path = base_path() ++ "functions/" ++ url_parameter(FunctionName), QParams = filter_undef([{"Qualifier", Qualifier}]), lambda_request(Config, delete, Path, undefined, QParams). From 7aec4aec0916247f2a7252bd8c1648384ad432cb Mon Sep 17 00:00:00 2001 From: Vitor Andriotti Date: Tue, 23 Mar 2021 09:10:31 -0300 Subject: [PATCH 216/310] Fix application autoscaler result fun to retry on ThrottlingException * Fix application autoscaler result fun to retry on ThrottlingException * Don't retry on 409 since it is not on Application Auto Scaling list of [common errors](https://docs.aws.amazon.com/autoscaling/application/APIReference/CommonErrors.html) --- src/erlcloud_application_autoscaler.erl | 28 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/erlcloud_application_autoscaler.erl b/src/erlcloud_application_autoscaler.erl index 5ee7cb7e7..abe90f6af 100644 --- a/src/erlcloud_application_autoscaler.erl +++ b/src/erlcloud_application_autoscaler.erl @@ -685,11 +685,29 @@ register_scalable_target(Configuration, BodyConfiguration) -> aas_result_fun(#aws_request{response_type = ok} = Request) -> Request; aas_result_fun(#aws_request{response_type = error, - error_type = aws, - response_status = Status} = Request) when -%% Retry conflicting operations 409,Conflict and 500s -%% including 503, SlowDown, Reduce your request rate. - Status =:= 409; Status >= 500 -> + error_type = aws, + response_status = 400, + response_body = Body} = Request) -> + %% Retry on ThrottlingException, ConcurrentUpdateException + try jsx:decode(Body, [{return_maps, false}]) of + Json -> + case proplists:get_value(<<"__type">>, Json) of + <<"ThrottlingException">> -> + Request#aws_request{should_retry = true}; + <<"ConcurrentUpdateException">> -> + Request#aws_request{should_retry = true}; + _Other -> + Request#aws_request{should_retry = false} + end + catch + error:badarg -> + Request#aws_request{should_retry = false} + end; +aas_result_fun(#aws_request{response_type = error, + error_type = aws, + response_status = Status} = Request) when + Status >= 500 -> + %% Retry on InternalFailure (500) or ServiceUnavailable (503). Request#aws_request{should_retry = true}; aas_result_fun(#aws_request{response_type = error, error_type = aws} = Request) -> Request#aws_request{should_retry = false}; From 860a0889775ca9b339a297c33e630aa274e82fc6 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Wed, 28 Apr 2021 21:12:20 +0100 Subject: [PATCH 217/310] Fix DAX-related -spec(_).s --- src/erlcloud_ddb2.erl | 47 ++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index a6d4177c5..e3d3d555d 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -1467,7 +1467,8 @@ table_description_record() -> %%%------------------------------------------------------------------------------ -type batch_get_item_opt() :: return_consumed_capacity_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type batch_get_item_opts() :: [batch_get_item_opt()]. -spec batch_get_item_opts() -> opt_table(). @@ -1541,7 +1542,7 @@ batch_get_item_record() -> end} ]}. --type batch_get_item_return() :: ddb_return(#ddb2_batch_get_item{}, [out_item()]). +-type batch_get_item_return() :: ddb_return(#ddb2_batch_get_item{} | #ddb2_request{}, [out_item()]). -spec batch_get_item(batch_get_item_request_items()) -> batch_get_item_return(). batch_get_item(RequestItems) -> @@ -1609,7 +1610,8 @@ batch_get_item(RequestItems, Opts, Config) -> -type batch_write_item_opt() :: return_consumed_capacity_opt() | return_item_collection_metrics_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type batch_write_item_opts() :: [batch_write_item_opt()]. -spec batch_write_item_opts() -> opt_table(). @@ -1669,7 +1671,7 @@ batch_write_item_record() -> end} ]}. --type batch_write_item_return() :: ddb_return(#ddb2_batch_write_item{}, #ddb2_batch_write_item{}). +-type batch_write_item_return() :: ddb_return(#ddb2_batch_write_item{} | #ddb2_request{}, #ddb2_batch_write_item{}). -spec batch_write_item(batch_write_item_request_items()) -> batch_write_item_return(). batch_write_item(RequestItems) -> @@ -2108,7 +2110,8 @@ delete_backup(BackupArn, Opts, Config) {return_values, none | all_old} | return_consumed_capacity_opt() | return_item_collection_metrics_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type delete_item_opts() :: [delete_item_opt()]. -spec delete_item_opts() -> opt_table(). @@ -2131,7 +2134,7 @@ delete_item_record() -> fun undynamize_item_collection_metrics/2} ]}. --type delete_item_return() :: ddb_return(#ddb2_delete_item{}, out_item()). +-type delete_item_return() :: ddb_return(#ddb2_delete_item{} | #ddb2_request{}, out_item()). -spec delete_item(table_name(), key()) -> delete_item_return(). delete_item(Table, Key) -> @@ -2670,7 +2673,8 @@ describe_time_to_live(Table, DbOpts, Config) -> attributes_to_get_opt() | consistent_read_opt() | return_consumed_capacity_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type get_item_opts() :: [get_item_opt()]. -spec get_item_opts() -> opt_table(). @@ -2688,7 +2692,7 @@ get_item_record() -> {<<"ConsumedCapacity">>, #ddb2_get_item.consumed_capacity, fun undynamize_consumed_capacity/2} ]}. --type get_item_return() :: ddb_return(#ddb2_get_item{}, out_item()). +-type get_item_return() :: ddb_return(#ddb2_get_item{} | #ddb2_request{}, out_item()). -spec get_item(table_name(), key()) -> get_item_return(). get_item(Table, Key) -> @@ -3024,7 +3028,8 @@ list_tags_of_resource(ResourceArn, Opts, Config) -> {return_values, none | all_old} | return_consumed_capacity_opt() | return_item_collection_metrics_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type put_item_opts() :: [put_item_opt()]. -spec put_item_opts() -> opt_table(). @@ -3047,7 +3052,7 @@ put_item_record() -> fun undynamize_item_collection_metrics/2} ]}. --type put_item_return() :: ddb_return(#ddb2_put_item{}, out_item()). +-type put_item_return() :: ddb_return(#ddb2_put_item{} | #ddb2_request{}, out_item()). -spec put_item(table_name(), in_item()) -> put_item_return(). put_item(Table, Item) -> @@ -3171,7 +3176,7 @@ q_record() -> {<<"ScannedCount">>, #ddb2_q.scanned_count, fun id/2} ]}. --type q_return() :: ddb_return(#ddb2_q{}, [out_item()]). +-type q_return() :: ddb_return(#ddb2_q{} | #ddb2_request{}, [out_item()]). -spec q(table_name(), conditions() | expression()) -> q_return(). q(Table, KeyConditionsOrExpression) -> @@ -3358,7 +3363,8 @@ restore_table_to_point_in_time(SourceTableName, TargetTableName, Opts, Config) {index_name, index_name()} | {select, select()} | return_consumed_capacity_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type scan_opts() :: [scan_opt()]. -spec scan_opts() -> opt_table(). @@ -3390,7 +3396,7 @@ scan_record() -> {<<"ScannedCount">>, #ddb2_scan.scanned_count, fun id/2} ]}. --type scan_return() :: ddb_return(#ddb2_scan{}, [out_item()]). +-type scan_return() :: ddb_return(#ddb2_scan{} | #ddb2_request{}, [out_item()]). -spec scan(table_name()) -> scan_return(). scan(Table) -> @@ -3482,7 +3488,8 @@ tag_resource(ResourceArn, Tags, Config) -> -type transact_get_items_transact_item_opts() :: expression_attribute_names_opt() | projection_expression_opt() | return_consumed_capacity_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type transact_get_items_opts() :: [transact_get_items_transact_item_opts()]. -type transact_get_items_get_item() :: {table_name(), key()} @@ -3493,7 +3500,7 @@ tag_resource(ResourceArn, Tags, Config) -> -type transact_get_items_transact_item() :: transact_get_items_get(). -type transact_get_items_transact_items() :: maybe_list(transact_get_items_transact_item()). --type transact_get_items_return() :: ddb_return(#ddb2_transact_get_items{}, out_item()). +-type transact_get_items_return() :: ddb_return(#ddb2_transact_get_items{} | #ddb2_request{}, out_item()). -spec dynamize_transact_get_items_transact_items(transact_get_items_transact_items()) -> [jsx:json_term()]. @@ -3590,7 +3597,8 @@ transact_get_items(TransactItems, Opts, Config) -> -type transact_write_items_opt() :: client_request_token_opt() | return_consumed_capacity_opt() | return_item_collection_metrics_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type transact_write_items_opts() :: [transact_write_items_opt()]. -type return_value_on_condition_check_failure_opt() :: {return_values_on_condition_check_failure, return_value()}. @@ -3672,7 +3680,7 @@ transact_write_items_record() -> end} ]}. --type transact_write_items_return() :: ddb_return(#ddb2_transact_write_items{}, out_item()). +-type transact_write_items_return() :: ddb_return(#ddb2_transact_write_items{} | #ddb2_request{}, out_item()). -spec transact_write_items(transact_write_items_transact_items()) -> transact_write_items_return(). transact_write_items(RequestItems) -> @@ -3856,7 +3864,8 @@ dynamize_update_item_updates_or_expression(Updates) -> {return_values, return_value()} | return_consumed_capacity_opt() | return_item_collection_metrics_opt() | - out_opt(). + out_opt() | + no_request_opt(). -type update_item_opts() :: [update_item_opt()]. -spec update_item_opts() -> opt_table(). @@ -3879,7 +3888,7 @@ update_item_record() -> fun undynamize_item_collection_metrics/2} ]}. --type update_item_return() :: ddb_return(#ddb2_update_item{}, out_item()). +-type update_item_return() :: ddb_return(#ddb2_update_item{} | #ddb2_request{}, out_item()). -spec update_item(table_name(), key(), in_updates() | expression()) -> update_item_return(). update_item(Table, Key, UpdatesOrExpression) -> From ae1596d493317efe1239df91670ee7fabe6ec5e0 Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 30 Apr 2021 17:28:19 +0100 Subject: [PATCH 218/310] Ease consumption (no need to consume a whole .hrl for a single type) --- src/erlcloud.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/erlcloud.erl b/src/erlcloud.erl index 11138d0e2..1e0666359 100644 --- a/src/erlcloud.erl +++ b/src/erlcloud.erl @@ -5,6 +5,8 @@ -include("erlcloud_aws.hrl"). -define(APP, erlcloud). +-export_type([aws_config/0]). + start() -> application:load(?APP), {ok, Apps} = application:get_key(?APP, applications), From ece8c2f68e504a7b5808ae6b546590aefaf48ca0 Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Tue, 4 May 2021 08:38:21 -0500 Subject: [PATCH 219/310] Update eini to 1.2.9 * Updates eini to [1.2.9](https://github.com/erlcloud/eini/releases/tag/1.2.9) (see support nested properties, https://github.com/erlcloud/eini/pull/19). --- rebar.config | 2 +- rebar.config.script | 2 +- rebar.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index 753c99996..fb7392b2d 100644 --- a/rebar.config +++ b/rebar.config @@ -19,7 +19,7 @@ {deps, [ {jsx, "2.11.0"}, {lhttpc, "1.6.2"}, - {eini, "1.2.7"}, + {eini, "1.2.9"}, {base16, "1.0.0"} ]}. diff --git a/rebar.config.script b/rebar.config.script index 1897c844e..732697f03 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -9,7 +9,7 @@ case erlang:function_exported(rebar3, main, 1) of [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.9.0"}}}, {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "v2.11.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, - {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.7"}}}, + {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.9"}}}, {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, {base16, ".*", {git, "https://github.com/goj/base16.git", {tag, "1.0.0"}}}]} | lists:keydelete(deps, 1, CONFIG)], diff --git a/rebar.lock b/rebar.lock index 134d2c922..a955d139a 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,12 +1,12 @@ {"1.1.0", [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, - {<<"eini">>,{pkg,<<"eini">>,<<"1.2.7">>},0}, + {<<"eini">>,{pkg,<<"eini">>,<<"1.2.9">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.11.0">>},0}, {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.6.2">>},0}]}. [ {pkg_hash,[ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, - {<<"eini">>, <<"EFC9D836E88591A47550BD34CE964E21CA1369F8716B24F73CFEA513FA99F666">>}, + {<<"eini">>, <<"FCC3CBD49BBDD9A1D9735C7365DAFFCD84481CCE81E6CB80537883AA44AC4895">>}, {<<"jsx">>, <<"08154624050333919B4AC1B789667D5F4DB166DC50E190C4D778D1587F102EE0">>}, {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} ]. From c3367e73ec864165b17275c8c8d8e13a9291d5ad Mon Sep 17 00:00:00 2001 From: "Paulo F. Oliveira" Date: Fri, 14 May 2021 10:10:08 +0100 Subject: [PATCH 220/310] Tweak CI versions (add 24.0, move 23.0 to 23.3) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e3d817be..f78254c4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: image: erlang:${{matrix.otp_vsn}} strategy: matrix: - otp_vsn: [19.3, 20.3, 21.3, 22.3, 23.0] + otp_vsn: [19.3, 20.3, 21.3, 22.3, 23.3, 24.0] os: [ubuntu-latest] force_rebar2: [true, false] env: From 0eba3a5c4a9ad983c7b644cb50d7b21c92151146 Mon Sep 17 00:00:00 2001 From: Hugues Martel Date: Sat, 3 Jul 2021 23:46:57 -0500 Subject: [PATCH 221/310] Expose erlcloud_ddb_util:get_all/5 --- src/erlcloud_ddb_util.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_ddb_util.erl b/src/erlcloud_ddb_util.erl index 24a22da64..24af5b189 100644 --- a/src/erlcloud_ddb_util.erl +++ b/src/erlcloud_ddb_util.erl @@ -21,7 +21,7 @@ %%% DynamoDB Higher Layer API -export([delete_all/2, delete_all/3, delete_all/4, delete_hash_key/3, delete_hash_key/4, delete_hash_key/5, - get_all/2, get_all/3, get_all/4, + get_all/2, get_all/3, get_all/4, get_all/5, put_all/2, put_all/3, put_all/4, list_tables_all/0, list_tables_all/1, q_all/2, q_all/3, q_all/4, From 5200636776e14d8c0b867c4d595d21639c246065 Mon Sep 17 00:00:00 2001 From: Pankaj Soni Date: Tue, 31 Aug 2021 17:37:30 +0530 Subject: [PATCH 222/310] Update erlcloud_aws.erl --- src/erlcloud_aws.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 3c5c67325..43591c0e5 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -746,8 +746,8 @@ service_config( <<"sdb">> = Service, Region, Config ) -> service_config( <<"ses">>, Region, Config ) -> Host = service_host( <<"email">>, Region ), Config#aws_config{ ses_host = Host }; -service_config( <<"sm">> = Service, Region, Config) -> - Host = service_host( Service, Region ), +service_config( <<"sm">>, Region, Config) -> + Host = service_host( <<"secretsmanager">>, Region ), Config#aws_config{ sm_host = Host }; service_config( <<"sns">> = Service, Region, Config ) -> Host = service_host( Service, Region ), From bf10f94bcd6a6d29f382222f612b8d9c0cb417e8 Mon Sep 17 00:00:00 2001 From: Pavel Puchkin Date: Thu, 9 Sep 2021 11:42:44 +0200 Subject: [PATCH 223/310] Add methods to get and set SNS platform application attrs --- src/erlcloud_sns.erl | 58 ++++++++++++++++++----- test/erlcloud_sns_tests.erl | 94 +++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 11 deletions(-) diff --git a/src/erlcloud_sns.erl b/src/erlcloud_sns.erl index 76debede8..b29493c0a 100644 --- a/src/erlcloud_sns.erl +++ b/src/erlcloud_sns.erl @@ -5,13 +5,22 @@ -author('elbrujohalcon@inaka.net'). -export([add_permission/3, add_permission/4, - create_platform_application/2, - create_platform_application/3, - create_platform_application/4, - create_platform_application/5, + create_platform_application/2, create_platform_application/3, + create_platform_application/4, create_platform_application/5, + + list_platform_applications/0, list_platform_applications/1, + list_platform_applications/2, list_platform_applications/3, + + get_platform_application_attributes/1, get_platform_application_attributes/2, + set_platform_application_attributes/2, set_platform_application_attributes/3, + create_platform_endpoint/2, create_platform_endpoint/3, create_platform_endpoint/4, create_platform_endpoint/5, create_platform_endpoint/6, + + list_endpoints_by_platform_application/1, list_endpoints_by_platform_application/2, + list_endpoints_by_platform_application/3, list_endpoints_by_platform_application/4, + create_topic/1, create_topic/2, delete_endpoint/1, delete_endpoint/2, delete_endpoint/3, delete_topic/1, delete_topic/2, @@ -28,10 +37,6 @@ list_subscriptions_by_topic_all/1, list_subscriptions_by_topic_all/2, - list_endpoints_by_platform_application/1, - list_endpoints_by_platform_application/2, - list_endpoints_by_platform_application/3, - list_endpoints_by_platform_application/4, get_endpoint_attributes/1, get_endpoint_attributes/2, get_endpoint_attributes/3, @@ -41,8 +46,6 @@ publish_to_topic/5, publish_to_target/2, publish_to_target/3, publish_to_target/4, publish_to_target/5, publish_to_phone/2, publish_to_phone/3, publish_to_phone/4, publish/5, publish/6, - list_platform_applications/0, list_platform_applications/1, - list_platform_applications/2, list_platform_applications/3, confirm_subscription/1, confirm_subscription/2, confirm_subscription/3, confirm_subscription2/2, confirm_subscription2/3, confirm_subscription2/4, set_topic_attributes/3, set_topic_attributes/4, @@ -106,7 +109,12 @@ -type sns_application_attribute() :: event_endpoint_created | event_endpoint_deleted | event_endpoint_updated - | event_delivery_failure. + | event_delivery_failure + | platform_credential + | platform_principal + | success_feedback_role_arn + | failure_feedback_role_arn + | success_feedback_sample_rate. -type sns_application() :: [{arn, string()} | {attributes, [{arn|sns_application_attribute(), string()}]}]. -type(sns_topic_attribute_name () :: 'Policy' | 'DisplayName' | 'DeliveryPolicy'). @@ -190,6 +198,34 @@ create_platform_application(Name, Platform, Attributes, AccessKeyID, SecretAcces create_platform_application(Name, Platform, Attributes, new_config(AccessKeyID, SecretAccessKey)). +-spec get_platform_application_attributes(string()) -> sns_application(). +get_platform_application_attributes(PlatformApplicationArn) -> + get_platform_application_attributes(PlatformApplicationArn, default_config()). + +-spec get_platform_application_attributes(string(), aws_config()) -> sns_application(). +get_platform_application_attributes(PlatformApplicationArn, Config) -> + Params = [{"PlatformApplicationArn", PlatformApplicationArn}], + Doc = sns_xml_request(Config, "GetPlatformApplicationAttributes", Params), + Decoded = + erlcloud_xml:decode( + [{attributes, "GetPlatformApplicationAttributesResult/Attributes/entry", + fun extract_attribute/1 + }], + Doc), + [{arn, PlatformApplicationArn} | Decoded]. + + +-spec set_platform_application_attributes(string(), [{sns_application_attribute(), string()}]) -> string(). +set_platform_application_attributes(PlatformApplicationArn, Attributes) -> + set_platform_application_attributes(PlatformApplicationArn, Attributes, default_config()). + +-spec set_platform_application_attributes(string(), [{sns_application_attribute(), string()}], aws_config()) -> string(). +set_platform_application_attributes(PlatformApplicationArn, Attributes, Config) -> + Params = [{"PlatformApplicationArn", PlatformApplicationArn}] ++ encode_attributes(Attributes), + Doc = sns_xml_request(Config, "SetPlatformApplicationAttributes", Params), + erlcloud_xml:get_text("ResponseMetadata/RequestId", Doc). + + -spec create_topic(string()) -> string(). create_topic(TopicName) -> create_topic(TopicName, default_config()). diff --git a/test/erlcloud_sns_tests.erl b/test/erlcloud_sns_tests.erl index be5634de5..6281433a5 100644 --- a/test/erlcloud_sns_tests.erl +++ b/test/erlcloud_sns_tests.erl @@ -39,6 +39,10 @@ sns_api_test_() -> fun subscribe_output_tests/1, fun create_platform_application_input_tests/1, fun create_platform_application_output_tests/1, + fun get_platform_application_attributes_input_tests/1, + fun get_platform_application_attributes_output_tests/1, + fun set_platform_application_attributes_input_tests/1, + fun set_platform_application_attributes_output_tests/1, fun set_topic_attributes_input_tests/1, fun set_topic_attributes_output_tests/1, fun set_subscription_attributes_input_tests/1, @@ -320,6 +324,96 @@ create_platform_application_output_tests(_) -> ], output_tests(?_f(erlcloud_sns:create_platform_application("ADM", "TestApp")), Tests). + +get_platform_application_attributes_input_tests(_) -> + Tests = + [?_sns_test( + {"Test to get platform application attributes.", + ?_f(erlcloud_sns:get_platform_application_attributes("TestAppArn")), + [ + {"Action", "GetPlatformApplicationAttributes"}, + {"PlatformApplicationArn", "TestAppArn"} + ]}) + ], + + Response = " + + + + + EventDeliveryFailure + arn:aws:sns:us-west-2:123456789012:topicarn + + + + + b6f0e78b-e9d4-5a0e-b973-adc04e8a4ff9 + + ", + + input_tests(Response, Tests). + +get_platform_application_attributes_output_tests(_) -> + Tests = [?_sns_test( + {"This is a get platform application attributes test.", + " + + + + EventDeliveryFailure + arn:aws:sns:us-west-2:123456789012:topicarn + + + + + b6f0e78b-e9d4-5a0e-b973-adc04e8a4ff9 + + ", + [{arn, "TestAppArn"}, + {attributes, [{event_delivery_failure, "arn:aws:sns:us-west-2:123456789012:topicarn"}]}] + }) + ], + output_tests(?_f(erlcloud_sns:get_platform_application_attributes("TestAppArn")), Tests). + + +set_platform_application_attributes_input_tests(_) -> + Tests = + [?_sns_test( + {"Test to set platform application attributes.", + ?_f(erlcloud_sns:set_platform_application_attributes( + "TestAppArn", + [{platform_principal, "some-api-key"}])), + [ + {"Action", "SetPlatformApplicationAttributes"}, + {"PlatformApplicationArn", "TestAppArn"}, + {"Attributes.entry.1.key", "PlatformPrincipal"}, + {"Attributes.entry.1.value", "some-api-key"} + ]}) + ], + + Response = " + + + cf577bcc-b3dc-5463-88f1-3180b9412395 + + ", + input_tests(Response, Tests). + +set_platform_application_attributes_output_tests(_) -> + Tests = [?_sns_test( + {"This is a set platform application attributes test.", + " + + cf577bcc-b3dc-5463-88f1-3180b9412395 + + ", + "cf577bcc-b3dc-5463-88f1-3180b9412395" + }) + ], + output_tests(?_f( + erlcloud_sns:set_platform_application_attributes("TestAppArn", [{platform_principal, "some-api_key"}])), Tests). + + %% Set topic attributes test based on the API examples: %% http://docs.aws.amazon.com/sns/latest/APIReference/API_SetTopicAttributes.html set_topic_attributes_input_tests(_) -> From 5f488c162a66e8086b25b0c3c7355f08f875781e Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Sun, 31 Oct 2021 11:04:50 -0500 Subject: [PATCH 224/310] VPC endpoint configuration: cache availability zone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow up to https://github.com/erlcloud/erlcloud/pull/638 (see [comment](https://github.com/erlcloud/erlcloud/pull/638#discussion_r407576219)). We have a service that uses erlcloud to generate an AWS configuration for accessing the AWS backend based on HTTP requests. It probably should use a shared cache for those configuration structures, but did not need them originally. When we added a VPC Endpoint configuration, the service immediately became degraded, and testing confirmed that it was because we were calling the instance metadata endpoint in a DOS fashion, causing `erlcloud_ec2_meta:get_instance_metadata/2` to mostly take 5 seconds and return `{error, {socket_error, timeout}}`, breaking the VPC endpoint configuration in addition to making most requests to the service take 5+ seconds. This seems like a hard edge for at-scale uses. It's very hard to understand this problem if you don't understand how this code works, because it doesn't break per se—it just becomes unbearably slow. To solve the problem, I propose we cache `ok` results from the AZ lookup permanently in the application environment. It's not possible for this value to change during run-time, so this should be totally fine. I also added some unit tests for this logic. --- src/erlcloud_aws.erl | 29 +++++++++++--- test/erlcloud_aws_tests.erl | 76 +++++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 43591c0e5..230eb6166 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -824,7 +824,7 @@ service_host( Service, Region ) when is_binary(Service) andalso is_binary(Region % magic can be done via EC2 DescribeVpcEndpoints/filter by VPC/filter by AZ. % however, permissions and describe* API throttling is not what we want to deal with here. get_host_vpc_endpoint(Service, Default) when is_binary(Service) -> - VPCEndpointsByService = application:get_env(erlcloud, services_vpc_endpoints, []), + VPCEndpointsByService = get_vpc_endpoints(), ConfiguredEndpoints = proplists:get_value(Service, VPCEndpointsByService, []), %% resolve through ENV if any Endpoints = case ConfiguredEndpoints of @@ -861,9 +861,7 @@ string_split(String, Char) -> pick_vpc_endpoint([], Default) -> Default; pick_vpc_endpoint(Endpoints, Default) -> - % it fine to use default here - no IAM is used, only for http client - % one cannot use auto_config()/default_cfg() as it creates an infinite recursion. - case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", #aws_config{}) of + case get_availability_zone() of {ok, AZ} -> lists:foldl( fun (E , Acc) -> @@ -882,10 +880,31 @@ pick_vpc_endpoint(Endpoints, Default) -> Default end. --spec get_vpc_endpoints () -> list({binary(), binary()}). +-spec get_vpc_endpoints() -> list({binary(), binary()}). get_vpc_endpoints() -> application:get_env(erlcloud, services_vpc_endpoints, []). +-spec get_availability_zone() -> {ok, binary()} | {error, term()}. +get_availability_zone() -> + case application:get_env(erlcloud, availability_zone) of + {ok, AZ} = OkResult when is_binary(AZ) -> + OkResult; + _ -> + cache_instance_metadata_availability_zone() + end. + +-spec cache_instance_metadata_availability_zone() -> {ok, binary()} | {error, term()}. +cache_instance_metadata_availability_zone() -> + % it fine to use default here - no IAM is used, only for http client + % one cannot use auto_config()/default_cfg() as it creates an infinite recursion. + case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", #aws_config{}) of + {ok, AZ} = OkResult -> + application:set_env(erlcloud, availability_zone, AZ), + OkResult; + {error, _} = Error -> + Error + end. + -spec configure(aws_config()) -> {ok, aws_config()}. configure(#aws_config{} = Config) -> diff --git a/test/erlcloud_aws_tests.erl b/test/erlcloud_aws_tests.erl index 3d584b495..5e454f7e6 100644 --- a/test/erlcloud_aws_tests.erl +++ b/test/erlcloud_aws_tests.erl @@ -912,3 +912,79 @@ service_config_waf_test() -> [erlcloud_aws:service_config( Service, Region, #aws_config{} ) || Region <- Regions]] ). + +get_host_vpc_endpoint_setup_fun() -> + meck:new(erlcloud_ec2_meta), + meck:expect(erlcloud_ec2_meta, get_instance_metadata, + fun("placement/availability-zone", #aws_config{}) -> + {ok, <<"us-east-1a">>} + end). + +get_host_vpc_endpoint_teardown(_) -> + application:unset_env(erlcloud, availability_zone), + application:unset_env(erlcloud, services_vpc_endpoints), + meck:unload(erlcloud_ec2_meta). + + +get_host_vpc_endpoint_test_() -> + {foreach, + fun get_host_vpc_endpoint_setup_fun/0, + fun get_host_vpc_endpoint_teardown/1, + [fun test_get_host_vpc_endpoint_no_settings/0, + fun test_get_host_vpc_endpoint_no_os_env/0, + fun test_get_host_vpc_endpoint_invalid_os_env/0, + fun test_get_host_vpc_endpoint_valid_os_env/0 + ]}. + +test_get_host_vpc_endpoint_no_settings() -> + Default = <<"kinesis.us-east-1.amazonaws.com">>, + ?assertEqual(Default, + erlcloud_aws:get_host_vpc_endpoint(<<"kinesis">>, Default)), + ?assertEqual(0, meck:num_calls(erlcloud_ec2_meta, get_instance_metadata, '_')). + + +test_get_host_vpc_endpoint_no_os_env() -> + Service = <<"kinesis">>, + EnvVar = "KINESIS_VPC_ENDPOINTS", + Default = <<"kinesis.us-east-1.amazonaws.com">>, + application:set_env(erlcloud, services_vpc_endpoints, [{Service, {env, EnvVar}}]), + ?assertEqual(Default, + erlcloud_aws:get_host_vpc_endpoint(<<"kinesis">>, Default)), + ?assertEqual(0, meck:num_calls(erlcloud_ec2_meta, get_instance_metadata, '_')). + +test_get_host_vpc_endpoint_invalid_os_env() -> + Service = <<"kinesis">>, + EnvVar = "KINESIS_VPC_ENDPOINTS", + EnvSetting = "vpc-xyz123-us-east-foobar.kinesis.us-east-1.vpce.amazonaws.com", + true = os:putenv("KINESIS_VPC_ENDPOINTS", EnvSetting), + Default = <<"kinesis.us-east-1.amazonaws.com">>, + application:set_env(erlcloud, services_vpc_endpoints, [{Service, {env, EnvVar}}]), + try + ?assertEqual(Default, + erlcloud_aws:get_host_vpc_endpoint(<<"kinesis">>, Default)), + ?assertEqual(1, meck:num_calls(erlcloud_ec2_meta, get_instance_metadata, '_')) + after + true = os:unsetenv(EnvVar) + end. + +test_get_host_vpc_endpoint_valid_os_env() -> + Service = <<"kinesis">>, + EnvVar = "KINESIS_VPC_ENDPOINTS", + EnvSetting = "ABC:vpc-xyz123-us-east-1a.kinesis.us-east-1.vpce.amazonaws.com," + "DEF:vpc-xyz123-us-east-1b.kinesis.us-east-1.vpce.amazonaws.com", + true = os:putenv("KINESIS_VPC_ENDPOINTS", EnvSetting), + Default = <<"kinesis.us-east-1.amazonaws.com">>, + application:set_env(erlcloud, services_vpc_endpoints, [{Service, {env, EnvVar}}]), + try + ?assertEqual(undefined, + application:get_env(erlcloud, availability_zone)), + ?assertEqual(<<"vpc-xyz123-us-east-1a.kinesis.us-east-1.vpce.amazonaws.com">>, + erlcloud_aws:get_host_vpc_endpoint(<<"kinesis">>, Default)), + ?assertEqual({ok, <<"us-east-1a">>}, + application:get_env(erlcloud, availability_zone)), + ?assertEqual(<<"vpc-xyz123-us-east-1a.kinesis.us-east-1.vpce.amazonaws.com">>, + erlcloud_aws:get_host_vpc_endpoint(<<"kinesis">>, Default)), + ?assertEqual(1, meck:num_calls(erlcloud_ec2_meta, get_instance_metadata, '_')) + after + true = os:unsetenv(EnvVar) + end. From 5a82f5e80a6829ce97241ee4a26a7a6865dd3496 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Tue, 9 Nov 2021 16:03:12 +0000 Subject: [PATCH 225/310] Allow for EC2-invoking consumers to test their code against LocalStack Make EC2 protocol & port number configurable under `#aws_config{}`. --- include/erlcloud_aws.hrl | 2 ++ src/erlcloud_ec2.erl | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index b8229954c..05bebaddc 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -20,7 +20,9 @@ -record(aws_config, { as_host="autoscaling.amazonaws.com"::string(), + ec2_protocol="https"::string(), ec2_host="ec2.amazonaws.com"::string(), + ec2_port=433::non_neg_integer(), iam_host="iam.amazonaws.com"::string(), sts_host="sts.amazonaws.com"::string(), s3_scheme="https://"::string(), diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index c5a0373bc..a5bc09244 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -3536,7 +3536,9 @@ ec2_query(Config, Action, Params) -> ec2_query(Config, Action, Params, ApiVersion) -> QParams = [{"Action", Action}, {"Version", ApiVersion}|Params], - erlcloud_aws:aws_request_xml4(post, Config#aws_config.ec2_host, + erlcloud_aws:aws_request_xml4(post, Config#aws_config.ec2_protocol, + Config#aws_config.ec2_host, + Config#aws_config.ec2_port, "/", QParams, "ec2", Config). default_config() -> erlcloud_aws:default_config(). From 6b9cbb4e1f0f103df9b2290fc875d5a5df11c307 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Wed, 10 Nov 2021 16:59:06 +0000 Subject: [PATCH 226/310] Fix default EC2 port --- include/erlcloud_aws.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 05bebaddc..3ac0b6e15 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -22,7 +22,7 @@ as_host="autoscaling.amazonaws.com"::string(), ec2_protocol="https"::string(), ec2_host="ec2.amazonaws.com"::string(), - ec2_port=433::non_neg_integer(), + ec2_port=443::non_neg_integer(), iam_host="iam.amazonaws.com"::string(), sts_host="sts.amazonaws.com"::string(), s3_scheme="https://"::string(), From 241123ea7455d89af298af9152aa9ef75ff2f591 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Wed, 10 Nov 2021 17:07:27 +0000 Subject: [PATCH 227/310] Fix broken EC2 tests --- test/erlcloud_ec2_tests.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 7ebe2f149..7e18a71c9 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -859,7 +859,7 @@ describe_account_attributes_test() -> ]}, meck:new(erlcloud_aws, [passthrough]), meck:expect(erlcloud_aws, aws_request_xml4, - fun(_,_,_,_,_,_) -> + fun(_,_,_,_,_,_,_,_) -> XMERL end), Result = erlcloud_ec2:describe_account_attributes(), @@ -918,7 +918,7 @@ describe_nat_gateways_test() -> }, meck:new(erlcloud_aws, [passthrough]), meck:expect(erlcloud_aws, aws_request_xml4, - fun(_,_,_,_,_,_) -> + fun(_,_,_,_,_,_,_,_) -> XMERL end), Result = erlcloud_ec2:describe_nat_gateways(), @@ -991,7 +991,7 @@ describe_vpc_peering_connections_test() -> }, meck:new(erlcloud_aws, [passthrough]), meck:expect(erlcloud_aws, aws_request_xml4, - fun(_,_,_,_,_,_) -> + fun(_,_,_,_,_,_,_,_) -> XMERL end), Result = erlcloud_ec2:describe_vpc_peering_connections(), @@ -1456,7 +1456,7 @@ test_pagination([], _, _, _, _) -> ok; test_pagination([{TotalResults, ResultsPerPage} | Rest], ResponseGenerator, OriginalFunction, NormalParams, PagedParams) -> meck:new(erlcloud_aws, [passthrough]), meck:expect(erlcloud_aws, aws_request_xml4, - fun(_,_,_,Params,_,_) -> + fun(_,_,_,_,_,Params,_,_) -> NextTokenString = proplists:get_value("NextToken", Params), MaxResults = proplists:get_value("MaxResults", Params), {Start, End, NT} = From 07108f5d89d29b75a356e174aaac580c5792046f Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Wed, 10 Nov 2021 17:07:38 +0000 Subject: [PATCH 228/310] Update `rebar.lock` to latest format IIRC introduced in rebar3.14 --- rebar.lock | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rebar.lock b/rebar.lock index a955d139a..e7ebd2084 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,4 +1,4 @@ -{"1.1.0", +{"1.2.0", [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.9">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.11.0">>},0}, @@ -8,5 +8,10 @@ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, {<<"eini">>, <<"FCC3CBD49BBDD9A1D9735C7365DAFFCD84481CCE81E6CB80537883AA44AC4895">>}, {<<"jsx">>, <<"08154624050333919B4AC1B789667D5F4DB166DC50E190C4D778D1587F102EE0">>}, - {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} + {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]}, +{pkg_hash_ext,[ + {<<"base16">>, <<"02AFD0827E61A7B07093873E063575CA3A2B07520567C7F8CEC7C5D42F052D76">>}, + {<<"eini">>, <<"DA64AE8DB7C2F502E6F20CDF44CD3D9BE364412B87FF49FEBF282540F673DFCB">>}, + {<<"jsx">>, <<"EED26A0D04D217F9EECEFFFB89714452556CF90EB38F290A27A4D45B9988F8C0">>}, + {<<"lhttpc">>, <<"76B5FA6149D1E10D4B1FBC4EBD51D371DB19C1AB9F0A9ECF5B526440DF064E97">>}]} ]. From eb5351cf641a4ec6c84c714910b85f2e7006dca8 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Wed, 10 Nov 2021 17:33:13 +0000 Subject: [PATCH 229/310] Revert "Update `rebar.lock` to latest format" This reverts commit 07108f5d89d29b75a356e174aaac580c5792046f. --- rebar.lock | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/rebar.lock b/rebar.lock index e7ebd2084..a955d139a 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,4 +1,4 @@ -{"1.2.0", +{"1.1.0", [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.9">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.11.0">>},0}, @@ -8,10 +8,5 @@ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, {<<"eini">>, <<"FCC3CBD49BBDD9A1D9735C7365DAFFCD84481CCE81E6CB80537883AA44AC4895">>}, {<<"jsx">>, <<"08154624050333919B4AC1B789667D5F4DB166DC50E190C4D778D1587F102EE0">>}, - {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]}, -{pkg_hash_ext,[ - {<<"base16">>, <<"02AFD0827E61A7B07093873E063575CA3A2B07520567C7F8CEC7C5D42F052D76">>}, - {<<"eini">>, <<"DA64AE8DB7C2F502E6F20CDF44CD3D9BE364412B87FF49FEBF282540F673DFCB">>}, - {<<"jsx">>, <<"EED26A0D04D217F9EECEFFFB89714452556CF90EB38F290A27A4D45B9988F8C0">>}, - {<<"lhttpc">>, <<"76B5FA6149D1E10D4B1FBC4EBD51D371DB19C1AB9F0A9ECF5B526440DF064E97">>}]} + {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} ]. From eff118fc7bd38a55873a9fc35d6860d7d3d9d3e4 Mon Sep 17 00:00:00 2001 From: Guilherme Andrade Date: Wed, 10 Nov 2021 19:51:03 +0000 Subject: [PATCH 230/310] Keep ports away from EC2 call URLs unless explicitly overridden --- include/erlcloud_aws.hrl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 3ac0b6e15..dd647683b 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -22,7 +22,7 @@ as_host="autoscaling.amazonaws.com"::string(), ec2_protocol="https"::string(), ec2_host="ec2.amazonaws.com"::string(), - ec2_port=443::non_neg_integer(), + ec2_port=undefined::non_neg_integer()|undefined, iam_host="iam.amazonaws.com"::string(), sts_host="sts.amazonaws.com"::string(), s3_scheme="https://"::string(), From d03a9602bee706924221b978f809b38ae08d00fb Mon Sep 17 00:00:00 2001 From: Pavel Puchkin Date: Fri, 19 Nov 2021 20:24:08 +0200 Subject: [PATCH 231/310] Fallback on unknown attribute names --- src/erlcloud_sqs.erl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index 8bfaea66e..42582bcb5 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -46,7 +46,7 @@ kms_master_key_id | kms_data_key_reuse_period_seconds | approximate_number_of_messages_not_visible | visibility_timeout | created_timestamp | last_modified_timestamp | policy | - queue_arn | deduplication_scope | fifo_throughput_limit). + queue_arn | deduplication_scope | fifo_throughput_limit | sqs_managed_sse_enabled). -type(batch_entry() :: {string(), string()} | {string(), string(), [message_attribute()]} @@ -240,7 +240,8 @@ encode_attribute_name(kms_master_key_id) -> "KmsMasterKeyId"; encode_attribute_name(kms_data_key_reuse_period_seconds) -> "KmsDataKeyReusePeriodSeconds"; encode_attribute_name(all) -> "All"; encode_attribute_name(deduplication_scope) -> "DeduplicationScope"; -encode_attribute_name(fifo_throughput_limit) -> "FifoThroughputLimit". +encode_attribute_name(fifo_throughput_limit) -> "FifoThroughputLimit"; +encode_attribute_name(sqs_managed_sse_enabled) -> "SqsManagedSseEnabled". decode_attribute_name("MessageRetentionPeriod") -> message_retention_period; @@ -261,7 +262,9 @@ decode_attribute_name("KmsMasterKeyId") -> kms_master_key_id; decode_attribute_name("KmsDataKeyReusePeriodSeconds") -> kms_data_key_reuse_period_seconds; decode_attribute_name("FifoQueue") -> fifo_queue; decode_attribute_name("DeduplicationScope") -> deduplication_scope; -decode_attribute_name("FifoThroughputLimit") -> fifo_throughput_limit. +decode_attribute_name("FifoThroughputLimit") -> fifo_throughput_limit; +decode_attribute_name("SqsManagedSseEnabled") -> sqs_managed_sse_enabled; +decode_attribute_name(Name) -> Name. decode_attribute_value("Policy", Value) -> Value; From e67d709372092d22500a71da92266adb117405bf Mon Sep 17 00:00:00 2001 From: Pavel Puchkin Date: Thu, 25 Nov 2021 18:13:25 +0200 Subject: [PATCH 232/310] Fix SNS message attribute Name field As per https://docs.aws.amazon.com/sns/latest/api/API_Publish.html --- src/erlcloud_sns.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_sns.erl b/src/erlcloud_sns.erl index b29493c0a..1ba5f1817 100644 --- a/src/erlcloud_sns.erl +++ b/src/erlcloud_sns.erl @@ -916,7 +916,7 @@ format_attribute_field(Num, {Key, Value}) -> -spec fields_for_attribute(string(), string() | binary() | number()) -> [{string(), string()}]. fields_for_attribute(Name, Value) -> - [{"Key", Name} | fields_for_attribute(Value)]. + [{"Name", Name} | fields_for_attribute(Value)]. -spec fields_for_attribute(string() | binary() | number()) -> [{string(), string()}]. fields_for_attribute(Value) when is_list(Value) -> From 2cea7e3628928fb14b21ccd83dd5fec6508bb05b Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Thu, 2 Dec 2021 09:46:41 -0600 Subject: [PATCH 233/310] Fix Hex.pm license to match COPYRIGHT Fixes https://github.com/erlcloud/erlcloud/issues/715. Erlcloud never used the MIT license, and has always been the BSD 2-clause license. It looks like this mistake in the Hex application properties was present from when hex publishing was implemented initially. Following https://hex.pm/docs/publish, we are amending the `licenses` from `["MIT"]` to `["BSD-2-Clause"]`, the [SPDX License identifier](https://spdx.org/licenses/) for [BSD 2-Clause](https://spdx.org/licenses/BSD-2-Clause.html). --- src/erlcloud.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud.app.src b/src/erlcloud.app.src index 3599241a6..6c54e36a3 100644 --- a/src/erlcloud.app.src +++ b/src/erlcloud.app.src @@ -24,7 +24,7 @@ {services_vpc_endpoints, []}, {ec2_meta_host_port, "169.254.169.254:80"} % allows for an alternative instance metadata service ]}, - {licenses, ["MIT"]}, + {licenses, ["BSD-2-Clause"]}, {links, [{"Github", "https://github.com/erlcloud/erlcloud"}]} ] }. From 897d1c3050bb79ee483c526484cd4152ecc7de63 Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Tue, 4 Jan 2022 20:37:16 -0600 Subject: [PATCH 234/310] erlcloud_sqs: get_queue_attributes improvement, test Follow-up to https://github.com/erlcloud/erlcloud/pull/713. When AWS added `SqsManagedSseEnabled`, we were lucky that it was a Boolean value, so it didn't break `erlcloud_sqs:decode_attribute_value/1`, which assumes that all attribute values that aren't specifically known are Boolean or integer values. This change anticipates a future when they add a string attribute value (like `QueueArn` or `KmsMasterKeyId `), so that we don't crash decoding THAT. Finally, adding some `get_queue_attributes` testing to `erlcloud_sqs_tests` to cover this functionality. --- src/erlcloud_sqs.erl | 8 +- test/erlcloud_sqs_tests.erl | 192 ++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index 42582bcb5..c93fd95b8 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -275,7 +275,13 @@ decode_attribute_value("DeduplicationScope", Value) -> Value; decode_attribute_value("FifoThroughputLimit", Value) -> Value; decode_attribute_value(_, "true") -> true; decode_attribute_value(_, "false") -> false; -decode_attribute_value(_, Value) -> list_to_integer(Value). +decode_attribute_value(_, Value) -> + try + list_to_integer(Value) + catch + error:badarg -> + Value + end. -spec list_queues() -> [string()]. diff --git a/test/erlcloud_sqs_tests.erl b/test/erlcloud_sqs_tests.erl index f98efd596..d9fcf3a7e 100644 --- a/test/erlcloud_sqs_tests.erl +++ b/test/erlcloud_sqs_tests.erl @@ -18,6 +18,10 @@ erlcloud_api_test_() -> fun stop/1, [ fun set_queue_attributes/1, + fun get_queue_attributes_all_output/1, + fun get_queue_attributes_all_unknown_output/1, + fun get_queue_attributes_all_input/1, + fun get_queue_attributes_specific_input/1, fun get_queue_url/1, fun send_message_with_message_opts/1, fun send_message_with_message_attributes/1, @@ -163,6 +167,194 @@ set_queue_attributes(_) -> ", input_tests(Response, Tests). +get_queue_attributes_all_output(_) -> + GetQueueAttributesResponse = " + + + + ReceiveMessageWaitTimeSeconds + 2 + + + VisibilityTimeout + 30 + + + ApproximateNumberOfMessages + 0 + + + ApproximateNumberOfMessagesNotVisible + 0 + + + CreatedTimestamp + 1286771522 + + + LastModifiedTimestamp + 1286771522 + + + QueueArn + arn:aws:sqs:us-east-2:123456789012:MyQueue + + + MaximumMessageSize + 8192 + + + MessageRetentionPeriod + 345600 + + + + 1ea71be5-b5a2-4f9d-b85a-945d8d08cd0b + +", + + Expected = [{receive_message_wait_time_seconds, 2}, + {visibility_timeout, 30}, + {approximate_number_of_messages, 0}, + {approximate_number_of_messages_not_visible, 0}, + {created_timestamp, 1286771522}, + {last_modified_timestamp, 1286771522}, + {queue_arn, "arn:aws:sqs:us-east-2:123456789012:MyQueue"}, + {maximum_message_size, 8192}, + {message_retention_period, 345600}], + Tests = + [?_sqs_test( + {"Test receives a get queue attributes result with all attributes (default).", + GetQueueAttributesResponse, Expected})], + + output_tests(?_f(erlcloud_sqs:get_queue_attributes("MyQueue")), Tests). + +get_queue_attributes_all_unknown_output(_) -> + GetQueueAttributesResponse = " + + + + ReceiveMessageWaitTimeSeconds + 2 + + + VisibilityTimeout + 30 + + + ApproximateNumberOfMessages + 0 + + + ApproximateNumberOfMessagesNotVisible + 0 + + + CreatedTimestamp + 1286771522 + + + LastModifiedTimestamp + 1286771522 + + + QueueArn + arn:aws:sqs:us-east-2:123456789012:MyQueue + + + MaximumMessageSize + 8192 + + + MessageRetentionPeriod + 345600 + + + UnrecognizedAttribute + UnrecognizedValue + + + + 1ea71be5-b5a2-4f9d-b85a-945d8d08cd0b + +", + Expected = [{receive_message_wait_time_seconds,2}, + {visibility_timeout, 30}, + {approximate_number_of_messages, 0}, + {approximate_number_of_messages_not_visible, 0}, + {created_timestamp, 1286771522}, + {last_modified_timestamp, 1286771522}, + {queue_arn, "arn:aws:sqs:us-east-2:123456789012:MyQueue"}, + {maximum_message_size, 8192}, + {message_retention_period, 345600}, + {"UnrecognizedAttribute", "UnrecognizedValue"}], + Tests = + [?_sqs_test( + {"Test receives a get queue attributes result with all attributes (default).", + GetQueueAttributesResponse, Expected})], + + output_tests(?_f(erlcloud_sqs:get_queue_attributes("MyQueue")), Tests). + +get_queue_attributes_all_input(_) -> + Expected = [ + {"Action", "GetQueueAttributes"}, + {"AttributeName.1", "All"} + ], + Tests = + [?_sqs_test( + {"Test getting queue attributes (specific).", + ?_f(erlcloud_sqs:get_queue_attributes("MyQueue")), + Expected})], + Response = " + + + + QueueArn + arn:aws:sqs:us-east-2:123456789012:MyQueue + + + + 1ea71be5-b5a2-4f9d-b85a-945d8d08cd0b + +", + input_tests(Response, Tests). + +get_queue_attributes_specific_input(_) -> + Expected = [ + {"Action", "GetQueueAttributes"}, + {"AttributeName.1", "VisibilityTimeout"}, + {"AttributeName.2", "DelaySeconds"}, + {"AttributeName.3", "ReceiveMessageWaitTimeSeconds"} + ], + Tests = + [?_sqs_test( + {"Test getting queue attributes (specific).", + ?_f(erlcloud_sqs:get_queue_attributes("MyQueue", [visibility_timeout, + delay_seconds, + receive_message_wait_time_seconds])), + Expected})], + Response = " + + + + VisibilityTimeout + 30 + + + DelaySeconds + 0 + + + ReceiveMessageWaitTimeSeconds + 2 + + + + 1ea71be5-b5a2-4f9d-b85a-945d8d08cd0b + +", + input_tests(Response, Tests). + get_queue_url(_) -> Expected = [ {"Action", "GetQueueUrl"}, From dfecf78491a24eed2e2085c1d16c7b6b35013a5a Mon Sep 17 00:00:00 2001 From: Nicholas Lundgaard Date: Wed, 5 Jan 2022 13:01:43 -0600 Subject: [PATCH 235/310] erlcloud_sqs: extend sqs_queue_attribute_names * Reorder `encode_attribute_name` and `decode_attribute_name` to use the ordering from [GetQueueAttributes API Reference](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_GetQueueAttributes.html) `AttributeName.N` definition (alphabetical, with common params first, then SSE & FIFO specific attribute names in sections below). * Add `FifoQueue` and `ContentBasedDeduplication` to `encode_attribute_name` and `decode_attribute_name`, previously missed FIFO parameters that should be supported. * Refactor `sqs_queue_attribute_name` type to include all known types. --- src/erlcloud_sqs.erl | 53 ++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/erlcloud_sqs.erl b/src/erlcloud_sqs.erl index c93fd95b8..19a906761 100644 --- a/src/erlcloud_sqs.erl +++ b/src/erlcloud_sqs.erl @@ -42,11 +42,14 @@ approximate_first_receive_timestamp | wait_time_seconds | receive_message_wait_time_seconds). --type(sqs_queue_attribute_name() :: all | approximate_number_of_messages | - kms_master_key_id | kms_data_key_reuse_period_seconds | - approximate_number_of_messages_not_visible | visibility_timeout | - created_timestamp | last_modified_timestamp | policy | - queue_arn | deduplication_scope | fifo_throughput_limit | sqs_managed_sse_enabled). + +-type(sqs_queue_attribute_name() :: all | approximate_number_of_messages | approximate_number_of_messages_not_visible | + approximate_number_of_messages_delayed | created_timestamp | delay_seconds | + last_modified_timestamp | maximum_message_size | message_retention_period | + policy | queue_arn | receive_message_wait_time_seconds | redrive_policy | + visibility_timeout | kms_master_key_id | kms_data_key_reuse_period_seconds | + sqs_managed_sse_enabled | fifo_queue | content_cased_deduplication | + deduplication_scope | fifo_throughput_limit). -type(batch_entry() :: {string(), string()} | {string(), string(), [message_attribute()]} @@ -223,47 +226,53 @@ get_queue_attributes(QueueName, AttributeNames, Config) [{decode_attribute_name(Name), decode_attribute_value(Name, Value)} || {Name, Value} <- Attrs]. -encode_attribute_name(message_retention_period) -> "MessageRetentionPeriod"; -encode_attribute_name(queue_arn) -> "QueueArn"; -encode_attribute_name(maximum_message_size) -> "MaximumMessageSize"; -encode_attribute_name(visibility_timeout) -> "VisibilityTimeout"; +encode_attribute_name(all) -> "All"; encode_attribute_name(approximate_number_of_messages) -> "ApproximateNumberOfMessages"; encode_attribute_name(approximate_number_of_messages_not_visible) -> "ApproximateNumberOfMessagesNotVisible"; encode_attribute_name(approximate_number_of_messages_delayed) -> "ApproximateNumberOfMessagesDelayed"; -encode_attribute_name(last_modified_timestamp) -> "LastModifiedTimestamp"; encode_attribute_name(created_timestamp) -> "CreatedTimestamp"; encode_attribute_name(delay_seconds) -> "DelaySeconds"; -encode_attribute_name(receive_message_wait_time_seconds) -> "ReceiveMessageWaitTimeSeconds"; +encode_attribute_name(last_modified_timestamp) -> "LastModifiedTimestamp"; +encode_attribute_name(maximum_message_size) -> "MaximumMessageSize"; +encode_attribute_name(message_retention_period) -> "MessageRetentionPeriod"; encode_attribute_name(policy) -> "Policy"; +encode_attribute_name(queue_arn) -> "QueueArn"; +encode_attribute_name(receive_message_wait_time_seconds) -> "ReceiveMessageWaitTimeSeconds"; encode_attribute_name(redrive_policy) -> "RedrivePolicy"; +encode_attribute_name(visibility_timeout) -> "VisibilityTimeout"; +%% The following attributes apply only to server-side-encryption encode_attribute_name(kms_master_key_id) -> "KmsMasterKeyId"; encode_attribute_name(kms_data_key_reuse_period_seconds) -> "KmsDataKeyReusePeriodSeconds"; -encode_attribute_name(all) -> "All"; +encode_attribute_name(sqs_managed_sse_enabled) -> "SqsManagedSseEnabled"; +%% The following attributes apply only to FIFO (first-in-first-out) queues +encode_attribute_name(fifo_queue) -> "FifoQueue"; +encode_attribute_name(content_cased_deduplication) -> "ContentBasedDeduplication"; encode_attribute_name(deduplication_scope) -> "DeduplicationScope"; -encode_attribute_name(fifo_throughput_limit) -> "FifoThroughputLimit"; -encode_attribute_name(sqs_managed_sse_enabled) -> "SqsManagedSseEnabled". +encode_attribute_name(fifo_throughput_limit) -> "FifoThroughputLimit". -decode_attribute_name("MessageRetentionPeriod") -> message_retention_period; -decode_attribute_name("QueueArn") -> queue_arn; -decode_attribute_name("MaximumMessageSize") -> maximum_message_size; -decode_attribute_name("VisibilityTimeout") -> visibility_timeout; decode_attribute_name("ApproximateNumberOfMessages") -> approximate_number_of_messages; decode_attribute_name("ApproximateNumberOfMessagesNotVisible") -> approximate_number_of_messages_not_visible; decode_attribute_name("ApproximateNumberOfMessagesDelayed") -> approximate_number_of_messages_delayed; -decode_attribute_name("LastModifiedTimestamp") -> last_modified_timestamp; decode_attribute_name("CreatedTimestamp") -> created_timestamp; decode_attribute_name("DelaySeconds") -> delay_seconds; -decode_attribute_name("ReceiveMessageWaitTimeSeconds") -> receive_message_wait_time_seconds; +decode_attribute_name("LastModifiedTimestamp") -> last_modified_timestamp; +decode_attribute_name("MaximumMessageSize") -> maximum_message_size; +decode_attribute_name("MessageRetentionPeriod") -> message_retention_period; decode_attribute_name("Policy") -> policy; +decode_attribute_name("QueueArn") -> queue_arn; +decode_attribute_name("ReceiveMessageWaitTimeSeconds") -> receive_message_wait_time_seconds; decode_attribute_name("RedrivePolicy") -> redrive_policy; -decode_attribute_name("ContentBasedDeduplication") -> content_based_deduplication; +decode_attribute_name("VisibilityTimeout") -> visibility_timeout; +%% The following attributes apply only to server-side-encryption decode_attribute_name("KmsMasterKeyId") -> kms_master_key_id; decode_attribute_name("KmsDataKeyReusePeriodSeconds") -> kms_data_key_reuse_period_seconds; +decode_attribute_name("SqsManagedSseEnabled") -> sqs_managed_sse_enabled; +%% The following attributes apply only to FIFO (first-in-first-out) queues decode_attribute_name("FifoQueue") -> fifo_queue; +decode_attribute_name("ContentBasedDeduplication") -> content_cased_deduplication; decode_attribute_name("DeduplicationScope") -> deduplication_scope; decode_attribute_name("FifoThroughputLimit") -> fifo_throughput_limit; -decode_attribute_name("SqsManagedSseEnabled") -> sqs_managed_sse_enabled; decode_attribute_name(Name) -> Name. From 855f7337f5157037fc7114a7bc1179009b27440d Mon Sep 17 00:00:00 2001 From: cody-friedrichsen Date: Wed, 5 Jan 2022 20:11:53 -0600 Subject: [PATCH 236/310] Handle invalid xml response in erlcloud_sns (#717) * Handle invalid xml response in erlcloud_sns * Remove erlang 21.X try..catch pattern; Remove unnecessary clause in erlcloud_sns tests * handle XML decode error in erlcloud_aws returning generic aws_error --- .gitignore | 1 + Makefile | 2 +- eunit.config | 1 + rebar.config | 2 ++ src/erlcloud_aws.erl | 32 ++++++++++++++++++++++++++------ test/erlcloud_sns_tests.erl | 19 +++++++++++++++++-- 6 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 eunit.config diff --git a/.gitignore b/.gitignore index 758520476..c85b5f9b5 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ ercloud.iml *[#]*[#] *[#]* .dialyzer_plt +eunit.log diff --git a/Makefile b/Makefile index d8304bd53..d9bbad732 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ ifeq ($(REBAR_VSN),2) $(MAKE) compile @$(REBAR) eunit skip_deps=true else - @$(REBAR) eunit + @ERL_FLAGS="-config $(PWD)/eunit" $(REBAR) eunit endif .dialyzer_plt: diff --git a/eunit.config b/eunit.config new file mode 100644 index 000000000..18babf022 --- /dev/null +++ b/eunit.config @@ -0,0 +1 @@ +[{kernel, [{error_logger, {file, "eunit.log"}}]}]. diff --git a/rebar.config b/rebar.config index fb7392b2d..499ecd44c 100644 --- a/rebar.config +++ b/rebar.config @@ -37,3 +37,5 @@ ]}. {post_hooks, [{clean, "rm -f .dialyzer_plt"}]}. + +{pre_hooks, [{clean, "rm -rf erl_crash.dump *.log"}]}. diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 230eb6166..e58a154c2 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -73,23 +73,26 @@ aws_request_xml(Method, Host, Path, Params, #aws_config{} = Config) -> Body = aws_request(Method, Host, Path, Params, Config), - element(1, xmerl_scan:string(binary_to_list(Body))). + raw_xml_response(Body). aws_request_xml(Method, Host, Path, Params, AccessKeyID, SecretAccessKey) -> Body = aws_request(Method, Host, Path, Params, AccessKeyID, SecretAccessKey), - element(1, xmerl_scan:string(binary_to_list(Body))). + raw_xml_response(Body). aws_request_xml(Method, Protocol, Host, Port, Path, Params, #aws_config{} = Config) -> Body = aws_request(Method, Protocol, Host, Port, Path, Params, Config), - element(1, xmerl_scan:string(binary_to_list(Body))). + raw_xml_response(Body). aws_request_xml(Method, Protocol, Host, Port, Path, Params, AccessKeyID, SecretAccessKey) -> Body = aws_request(Method, Protocol, Host, Port, Path, Params, AccessKeyID, SecretAccessKey), - element(1, xmerl_scan:string(binary_to_list(Body))). + raw_xml_response(Body). aws_request_xml2(Method, Host, Path, Params, #aws_config{} = Config) -> aws_request_xml2(Method, undefined, Host, undefined, Path, Params, Config). aws_request_xml2(Method, Protocol, Host, Port, Path, Params, #aws_config{} = Config) -> case aws_request2(Method, Protocol, Host, Port, Path, Params, Config) of {ok, Body} -> - {ok, element(1, xmerl_scan:string(binary_to_list(Body)))}; + case format_xml_response(Body) of + {ok, XML} -> {ok, XML}; + Error -> {error, Error} + end; {error, Reason} -> {error, Reason} end. @@ -99,7 +102,10 @@ aws_request_xml4(Method, Host, Path, Params, Service, #aws_config{} = Config) -> aws_request_xml4(Method, Protocol, Host, Port, Path, Params, Service, #aws_config{} = Config) -> case aws_request4(Method, Protocol, Host, Port, Path, Params, Service, Config) of {ok, Body} -> - {ok, element(1, xmerl_scan:string(binary_to_list(Body)))}; + case format_xml_response(Body) of + {ok, XML} -> {ok, XML}; + Error -> {error, Error} + end; {error, Reason} -> {error, Reason} end. @@ -331,6 +337,20 @@ encode_params(Params, Headers) -> _ContentType -> {Params, LowerCaseHeaders} end. +raw_xml_response(Body) -> + case format_xml_response(Body) of + {ok, XML} -> XML; + Error -> erlang:error(Error) + end. + +format_xml_response(Body) -> + try + {ok, element(1, xmerl_scan:string(binary_to_list(Body)))} + catch + _:_ -> + {aws_error, {invalid_xml_response_document, Body}} + end. + %%%--------------------------------------------------------------------------- -spec default_config() -> aws_config(). %%%--------------------------------------------------------------------------- diff --git a/test/erlcloud_sns_tests.erl b/test/erlcloud_sns_tests.erl index 6281433a5..ab043beb5 100644 --- a/test/erlcloud_sns_tests.erl +++ b/test/erlcloud_sns_tests.erl @@ -52,7 +52,8 @@ sns_api_test_() -> fun list_subscriptions_input_tests/1, fun list_subscriptions_output_tests/1, fun list_subscriptions_by_topic_input_tests/1, - fun list_subscriptions_by_topic_output_tests/1 + fun list_subscriptions_by_topic_output_tests/1, + fun publish_invalid_xml_response_output_tests/1 ]}. start() -> @@ -168,7 +169,12 @@ output_test(Fun, {Line, {Description, Response, Result}}) -> fun() -> meck:expect(erlcloud_httpc, request, output_expect(Response)), erlcloud_ec2:configure(string:copies("A", 20), string:copies("a", 40)), - Actual = Fun(), + Actual = try + Fun() + catch + _Class:Error -> + Error + end, ?assertEqual(Result, Actual) end}}. @@ -870,6 +876,15 @@ list_subscriptions_by_topic_output_tests(_) -> ]}) ]). +publish_invalid_xml_response_output_tests(_) -> + Config = erlcloud_aws:default_config(), + output_tests(?_f(erlcloud_sns:publish(topic, "arn:aws:sns:us-east-1:123456789012:My-Topic", "test message", undefined, Config)), + [?_sns_test( + {"Test PublishTopic invalid XML return", + "", + {sns_error, {aws_error, {invalid_xml_response_document, <<>>}}}}) + ]). + defaults_to_https(_) -> From 14aa01b0eb410578c554de5ab1b1f0c011aadf2a Mon Sep 17 00:00:00 2001 From: steve-vsee <75398304+steve-vsee@users.noreply.github.com> Date: Tue, 11 Jan 2022 14:44:17 -0800 Subject: [PATCH 237/310] Use https to fetch jsx and lhttpc See https://github.blog/2021-09-01-improving-git-protocol-security-github/ for why this is necessary --- rebar.config.script | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config.script b/rebar.config.script index 732697f03..756f8bbd7 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -7,10 +7,10 @@ case erlang:function_exported(rebar3, main, 1) of %% profiles CONFIG_DEPS = [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.9.0"}}}, - {jsx, ".*", {git, "git://github.com/talentdeficit/jsx.git", {tag, "v2.11.0"}}}, + {jsx, ".*", {git, "https://github.com/talentdeficit/jsx.git", {tag, "v2.11.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.9"}}}, - {lhttpc, ".*", {git, "git://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, + {lhttpc, ".*", {git, "https://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, {base16, ".*", {git, "https://github.com/goj/base16.git", {tag, "1.0.0"}}}]} | lists:keydelete(deps, 1, CONFIG)], CONFIG_NEW = From eeb6f10c705b4419bdd1103f52f6c7470f0ce271 Mon Sep 17 00:00:00 2001 From: steve-vsee <75398304+steve-vsee@users.noreply.github.com> Date: Tue, 11 Jan 2022 15:58:49 -0800 Subject: [PATCH 238/310] Add .git suffix to lhttpc --- rebar.config.script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config.script b/rebar.config.script index 756f8bbd7..618083bf3 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -10,7 +10,7 @@ case erlang:function_exported(rebar3, main, 1) of {jsx, ".*", {git, "https://github.com/talentdeficit/jsx.git", {tag, "v2.11.0"}}}, %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.9"}}}, - {lhttpc, ".*", {git, "https://github.com/erlcloud/lhttpc", {tag, "1.6.2"}}}, + {lhttpc, ".*", {git, "https://github.com/erlcloud/lhttpc.git", {tag, "1.6.2"}}}, {base16, ".*", {git, "https://github.com/goj/base16.git", {tag, "1.0.0"}}}]} | lists:keydelete(deps, 1, CONFIG)], CONFIG_NEW = From c2a8fc72b4675fde964dcc22e4037c1035140c79 Mon Sep 17 00:00:00 2001 From: Jeremy Hood Date: Thu, 13 Jan 2022 10:12:33 -0500 Subject: [PATCH 239/310] Correct error-details lookup in response from "message" to "Message" --- src/erlcloud_ddb_impl.erl | 2 +- test/erlcloud_ddb2_tests.erl | 8 +-- test/erlcloud_ddb_tests.erl | 127 +++++++++++++++++------------------ 3 files changed, 68 insertions(+), 69 deletions(-) diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index b8d66ab4f..014dd7c6a 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -248,7 +248,7 @@ client_error(Body, DDBError) -> undefined -> DDBError#ddb2_error{error_type = http, should_retry = false}; FullType -> - Message = proplists:get_value(<<"message">>, Json, <<>>), + Message = proplists:get_value(<<"Message">>, Json, <<>>), case binary:split(FullType, <<"#">>) of [_, Type] when Type =:= <<"ProvisionedThroughputExceededException">> orelse diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index c82047f99..9f468bc8f 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -273,7 +273,7 @@ error_handling_tests(_) -> {"Test retry after ProvisionedThroughputExceededException", [httpc_response(400, " {\"__type\":\"com.amazonaws.dynamodb.v20111205#ProvisionedThroughputExceededException\", -\"message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\"}" +\"Message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\"}" ), OkResponse], OkResult}), @@ -281,7 +281,7 @@ error_handling_tests(_) -> {"Test ConditionalCheckFailed error", [httpc_response(400, " {\"__type\":\"com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException\", -\"message\":\"The expected value did not match what was stored in the system.\"}" +\"Message\":\"The expected value did not match what was stored in the system.\"}" )], {error, {<<"ConditionalCheckFailedException">>, <<"The expected value did not match what was stored in the system.">>}}}), ?_ddb_test( @@ -315,7 +315,7 @@ error_handling_tests(_) -> \"Message\":\"The conditional request failed\", } ], - \"message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]\" + \"Message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]\" }" )], TransactionErrorResult}), @@ -333,7 +333,7 @@ error_handling_tests(_) -> \"Message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\", } ], - \"message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ProvisionedThroughputExceeded]\" + \"Message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ProvisionedThroughputExceeded]\" }" ), TransactOkResponse], diff --git a/test/erlcloud_ddb_tests.erl b/test/erlcloud_ddb_tests.erl index b78839518..4b57c29b2 100644 --- a/test/erlcloud_ddb_tests.erl +++ b/test/erlcloud_ddb_tests.erl @@ -19,7 +19,7 @@ -define(_f(F), fun() -> F end). -export([validate_body/2]). - + %%%=================================================================== %%% Test entry points %%%=================================================================== @@ -86,9 +86,9 @@ validate_body(Body, Expected) -> %% Validates the request body and responds with the provided response. -spec input_expect(string(), expected_body()) -> fun(). input_expect(Response, Expected) -> - fun(_Url, post, _Headers, Body, _Timeout, _Config) -> + fun(_Url, post, _Headers, Body, _Timeout, _Config) -> validate_body(Body, Expected), - {ok, {{200, "OK"}, [], list_to_binary(Response)}} + {ok, {{200, "OK"}, [], list_to_binary(Response)}} end. %% input_test converts an input_test specifier into an eunit test generator @@ -96,7 +96,7 @@ input_expect(Response, Expected) -> -spec input_test(string(), input_test_spec()) -> tuple(). input_test(Response, {Line, {Description, Fun, Expected}}) when is_list(Description) -> - {Description, + {Description, {Line, fun() -> meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), @@ -118,8 +118,8 @@ input_tests(Response, Tests) -> %% returns the mock of the erlcloud_httpc function output tests expect to be called. -spec output_expect(string()) -> fun(). output_expect(Response) -> - fun(_Url, post, _Headers, _Body, _Timeout, _Config) -> - {ok, {{200, "OK"}, [], list_to_binary(Response)}} + fun(_Url, post, _Headers, _Body, _Timeout, _Config) -> + {ok, {{200, "OK"}, [], list_to_binary(Response)}} end. %% output_test converts an output_test specifier into an eunit test generator @@ -141,9 +141,9 @@ output_test(Fun, {Line, {Description, Response, Result}}) -> end}}. %% output_test(Fun, {Line, {Response, Result}}) -> %% output_test(Fun, {Line, {"", Response, Result}}). - + %% output_tests converts a list of output_test specifiers into an eunit test generator --spec output_tests(fun(), [output_test_spec()]) -> [term()]. +-spec output_tests(fun(), [output_test_spec()]) -> [term()]. output_tests(Fun, Tests) -> [output_test(Fun, Test) || Test <- Tests]. @@ -155,7 +155,7 @@ output_tests(Fun, Tests) -> -spec httpc_response(pos_integer(), string()) -> tuple(). httpc_response(Code, Body) -> {ok, {{Code, ""}, [], list_to_binary(Body)}}. - + -type error_test_spec() :: {pos_integer(), {string(), list(), term()}}. -spec error_test(fun(), error_test_spec()) -> tuple(). error_test(Fun, {Line, {Description, Responses, Result}}) -> @@ -169,7 +169,7 @@ error_test(Fun, {Line, {Description, Responses, Result}}) -> Actual = Fun(), ?assertEqual(Result, Actual) end}}. - + -spec error_tests(fun(), [error_test_spec()]) -> [term()]. error_tests(Fun, Tests) -> [error_test(Fun, Test) || Test <- Tests]. @@ -201,17 +201,17 @@ error_handling_tests(_) -> \"status\":{\"S\":\"online\"} }, \"ConsumedCapacityUnits\": 1 -}" +}" ), OkResult = {ok, [{<<"friends">>, [<<"Lynda">>, <<"Aaron">>]}, {<<"status">>, <<"online">>}]}, - Tests = + Tests = [?_ddb_test( {"Test retry after ProvisionedThroughputExceededException", [httpc_response(400, " {\"__type\":\"com.amazonaws.dynamodb.v20111205#ProvisionedThroughputExceededException\", -\"message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\"}" +\"Message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\"}" ), OkResponse], OkResult}), @@ -219,7 +219,7 @@ error_handling_tests(_) -> {"Test ConditionalCheckFailed error", [httpc_response(400, " {\"__type\":\"com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException\", -\"message\":\"The expected value did not match what was stored in the system.\"}" +\"Message\":\"The expected value did not match what was stored in the system.\"}" )], {error, {<<"ConditionalCheckFailedException">>, <<"The expected value did not match what was stored in the system.">>}}}), ?_ddb_test( @@ -228,7 +228,7 @@ error_handling_tests(_) -> OkResponse], OkResult}) ], - + error_tests(?_f(erlcloud_ddb:get_item(<<"table">>, <<"key">>)), Tests). @@ -239,8 +239,8 @@ batch_get_item_input_tests(_) -> [?_ddb_test( {"BatchGetItem example request", ?_f(erlcloud_ddb:batch_get_item( - [{<<"comp2">>, [<<"Julie">>, - <<"Mingus">>], + [{<<"comp2">>, [<<"Julie">>, + <<"Mingus">>], [{attributes_to_get, [<<"user">>, <<"friends">>]}]}, {<<"comp1">>, [{<<"Casey">>, 1319509152}, {<<"Dave">>, 1319509155}, @@ -281,7 +281,7 @@ batch_get_item_input_tests(_) -> input_tests(Response, Tests). batch_get_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"BatchGetItem example response", " {\"Responses\": @@ -300,7 +300,7 @@ batch_get_item_output_tests(_) -> \"UnprocessedKeys\":{} }", {ok, #ddb_batch_get_item - {responses = + {responses = [#ddb_batch_get_item_response {table = <<"comp1">>, items = [[{<<"status">>, <<"online">>}, @@ -335,17 +335,17 @@ batch_get_item_output_tests(_) -> } }", {ok, #ddb_batch_get_item - {responses = [], - unprocessed_keys = - [{<<"comp2">>, [{s, <<"Julie">>}, - {s, <<"Mingus">>}], + {responses = [], + unprocessed_keys = + [{<<"comp2">>, [{s, <<"Julie">>}, + {s, <<"Mingus">>}], [{attributes_to_get, [<<"user">>, <<"friends">>]}]}, {<<"comp1">>, [{{s, <<"Casey">>}, {n, 1319509152}}, {{s, <<"Dave">>}, {n, 1319509155}}, {{s, <<"Riley">>}, {n, 1319509158}}], [{attributes_to_get, [<<"user">>, <<"status">>]}]}]}}}) ], - + output_tests(?_f(erlcloud_ddb:batch_get_item([{<<"table">>, [<<"key">>]}], [{out, record}])), Tests). %% BatchWriteItem test based on the API examples: @@ -438,7 +438,7 @@ batch_write_item_input_tests(_) -> input_tests(Response, Tests). batch_write_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"BatchWriteItem example response", " { @@ -468,7 +468,7 @@ batch_write_item_output_tests(_) -> } }", {ok, #ddb_batch_write_item - {responses = + {responses = [#ddb_batch_write_item_response {table = <<"Thread">>, consumed_capacity_units = 1.0}, @@ -534,7 +534,7 @@ batch_write_item_output_tests(_) -> {<<"Thread">>, [{put, [{<<"ForumName">>, {s, <<"Amazon DynamoDB">>}}, {<<"Subject">>, {s, <<"DynamoDB Thread 5">>}}]}]}]}}}) ], - + output_tests(?_f(erlcloud_ddb:batch_write_item([], [{out, record}])), Tests). %% CreateTable test based on the API examples: @@ -567,7 +567,7 @@ create_table_input_tests(_) -> input_tests(Response, Tests). create_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"CreateTable example response", " {\"TableDescription\": @@ -591,7 +591,7 @@ create_table_output_tests(_) -> table_name = <<"comp-table">>, table_status = <<"CREATING">>}}}) ], - + output_tests(?_f(erlcloud_ddb:create_table(<<"name">>, {<<"key">>, s}, 5, 10)), Tests). %% DeleteItem test based on the API examples: @@ -625,7 +625,7 @@ delete_item_input_tests(_) -> input_tests(Response, Tests). delete_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"DeleteItem example response", " {\"Attributes\": @@ -641,7 +641,7 @@ delete_item_output_tests(_) -> {<<"time">>, 200}, {<<"user">>, <<"Mingus">>}]}}) ], - + output_tests(?_f(erlcloud_ddb:delete_item(<<"table">>, <<"key">>)), Tests). %% DeleteTable test based on the API examples: @@ -650,7 +650,7 @@ delete_table_input_tests(_) -> Tests = [?_ddb_test( {"DeleteTable example request", - ?_f(erlcloud_ddb:delete_table(<<"Table1">>)), + ?_f(erlcloud_ddb:delete_table(<<"Table1">>)), "{\"TableName\":\"Table1\"}" }) ], @@ -669,7 +669,7 @@ delete_table_input_tests(_) -> input_tests(Response, Tests). delete_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"DeleteTable example response", " {\"TableDescription\": @@ -693,7 +693,7 @@ delete_table_output_tests(_) -> table_name = <<"Table1">>, table_status = <<"DELETING">>}}}) ], - + output_tests(?_f(erlcloud_ddb:delete_table(<<"name">>)), Tests). %% DescribeTable test based on the API examples: @@ -702,7 +702,7 @@ describe_table_input_tests(_) -> Tests = [?_ddb_test( {"DescribeTable example request", - ?_f(erlcloud_ddb:describe_table(<<"Table1">>)), + ?_f(erlcloud_ddb:describe_table(<<"Table1">>)), "{\"TableName\":\"Table1\"}" }) ], @@ -723,7 +723,7 @@ describe_table_input_tests(_) -> input_tests(Response, Tests). describe_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"DescribeTable example response", " {\"Table\": @@ -751,7 +751,7 @@ describe_table_output_tests(_) -> table_size_bytes = 949, table_status = <<"ACTIVE">>}}}) ], - + output_tests(?_f(erlcloud_ddb:describe_table(<<"name">>)), Tests). %% GetItem test based on the API examples: @@ -769,14 +769,14 @@ get_item_input_tests(_) -> Tests = [?_ddb_test( {"GetItem example request, with fully specified keys", - ?_f(erlcloud_ddb:get_item(<<"comptable">>, {{s, <<"Julie">>}, {n, 1307654345}}, + ?_f(erlcloud_ddb:get_item(<<"comptable">>, {{s, <<"Julie">>}, {n, 1307654345}}, [consistent_read, {attributes_to_get, [<<"status">>, <<"friends">>]}])), Example1Response}), ?_ddb_test( {"GetItem example request, with inferred key types", - ?_f(erlcloud_ddb:get_item(<<"comptable">>, {"Julie", 1307654345}, - [consistent_read, + ?_f(erlcloud_ddb:get_item(<<"comptable">>, {"Julie", 1307654345}, + [consistent_read, {attributes_to_get, [<<"status">>, <<"friends">>]}])), Example1Response}), ?_ddb_test( @@ -799,7 +799,7 @@ get_item_input_tests(_) -> input_tests(Response, Tests). get_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"GetItem example response", " {\"Item\": @@ -835,15 +835,15 @@ get_item_output_tests(_) -> {<<"b">>, <<5,182>>}, {<<"empty">>, <<>>}]}}), ?_ddb_test( - {"GetItem item not found", + {"GetItem item not found", "{\"ConsumedCapacityUnits\": 0.5}", {ok, []}}), ?_ddb_test( - {"GetItem no attributes returned", + {"GetItem no attributes returned", "{\"ConsumedCapacityUnits\":0.5,\"Item\":{}}", {ok, []}}) ], - + output_tests(?_f(erlcloud_ddb:get_item(<<"table">>, <<"key">>)), Tests). %% ListTables test based on the API examples: @@ -852,12 +852,12 @@ list_tables_input_tests(_) -> Tests = [?_ddb_test( {"ListTables example request", - ?_f(erlcloud_ddb:list_tables([{limit, 3}, {exclusive_start_table_name, <<"comp2">>}])), + ?_f(erlcloud_ddb:list_tables([{limit, 3}, {exclusive_start_table_name, <<"comp2">>}])), "{\"ExclusiveStartTableName\":\"comp2\",\"Limit\":3}" }), ?_ddb_test( {"ListTables empty request", - ?_f(erlcloud_ddb:list_tables()), + ?_f(erlcloud_ddb:list_tables()), "{}" }) @@ -867,7 +867,7 @@ list_tables_input_tests(_) -> input_tests(Response, Tests). list_tables_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"ListTables example response", "{\"LastEvaluatedTableName\":\"comp5\",\"TableNames\":[\"comp3\",\"comp4\",\"comp5\"]}", @@ -875,7 +875,7 @@ list_tables_output_tests(_) -> {last_evaluated_table_name = <<"comp5">>, table_names = [<<"comp3">>, <<"comp4">>, <<"comp5">>]}}}) ], - + output_tests(?_f(erlcloud_ddb:list_tables([{out, record}])), Tests). %% PutItem test based on the API examples: @@ -884,8 +884,8 @@ put_item_input_tests(_) -> Tests = [?_ddb_test( {"PutItem example request", - ?_f(erlcloud_ddb:put_item(<<"comp5">>, - [{<<"time">>, 300}, + ?_f(erlcloud_ddb:put_item(<<"comp5">>, + [{<<"time">>, 300}, {<<"feeling">>, <<"not surprised">>}, {<<"user">>, <<"Riley">>}], [{return_values, all_old}, @@ -903,8 +903,8 @@ put_item_input_tests(_) -> }), ?_ddb_test( {"PutItem float inputs", - ?_f(erlcloud_ddb:put_item(<<"comp5">>, - [{<<"time">>, 300}, + ?_f(erlcloud_ddb:put_item(<<"comp5">>, + [{<<"time">>, 300}, {<<"typed float">>, {n, 1.2}}, {<<"untyped float">>, 3.456}, {<<"mixed set">>, {ns, [7.8, 9.0, 10]}}], @@ -930,7 +930,7 @@ put_item_input_tests(_) -> input_tests(Response, Tests). put_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"PutItem example response", " {\"Attributes\": @@ -943,7 +943,7 @@ put_item_output_tests(_) -> {<<"time">>, 300}, {<<"user">>, <<"Riley">>}]}}) ], - + output_tests(?_f(erlcloud_ddb:put_item(<<"table">>, [])), Tests). %% Query test based on the API examples: @@ -1002,7 +1002,7 @@ q_input_tests(_) -> input_tests(Response, Tests). q_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"Query example 1 response", " {\"Count\":2,\"Items\":[{ @@ -1048,7 +1048,7 @@ q_output_tests(_) -> last_evaluated_key = undefined, consumed_capacity_units = 1}}}) ], - + output_tests(?_f(erlcloud_ddb:q(<<"table">>, <<"key">>, [{out, record}])), Tests). %% Scan test based on the API examples: @@ -1057,7 +1057,7 @@ scan_input_tests(_) -> Tests = [?_ddb_test( {"Scan example 1 request", - ?_f(erlcloud_ddb:scan(<<"1-hash-rangetable">>)), + ?_f(erlcloud_ddb:scan(<<"1-hash-rangetable">>)), "{\"TableName\":\"1-hash-rangetable\"}" }), ?_ddb_test( @@ -1104,7 +1104,7 @@ scan_input_tests(_) -> input_tests(Response, Tests). scan_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"Scan example 1 response", " {\"Count\":4,\"Items\":[{ @@ -1202,7 +1202,7 @@ scan_output_tests(_) -> scanned_count = 2, consumed_capacity_units = 0.5}}}) ], - + output_tests(?_f(erlcloud_ddb:scan(<<"name">>, [{out, record}])), Tests). %% UpdateItem test based on the API examples: @@ -1258,7 +1258,7 @@ update_item_input_tests(_) -> input_tests(Response, Tests). update_item_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"UpdateItem example response", " {\"Attributes\": @@ -1273,7 +1273,7 @@ update_item_output_tests(_) -> {<<"time">>, 1307654350}, {<<"user">>, <<"Julie">>}]}}) ], - + output_tests(?_f(erlcloud_ddb:update_item(<<"table">>, <<"key">>, [])), Tests). %% UpdateTable test based on the API examples: @@ -1306,7 +1306,7 @@ update_table_input_tests(_) -> input_tests(Response, Tests). update_table_output_tests(_) -> - Tests = + Tests = [?_ddb_test( {"UpdateTable example response", " {\"TableDescription\": @@ -1333,6 +1333,5 @@ update_table_output_tests(_) -> table_name = <<"comp1">>, table_status = <<"UPDATING">>}}}) ], - - output_tests(?_f(erlcloud_ddb:update_table(<<"name">>, 5, 15)), Tests). + output_tests(?_f(erlcloud_ddb:update_table(<<"name">>, 5, 15)), Tests). From 100b605dee49fde96148b7814c4b569e9269e96c Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Sun, 16 Jan 2022 23:46:06 +0800 Subject: [PATCH 240/310] Fix typos --- include/erlcloud_aws.hrl | 4 ++-- src/erlcloud_aws.erl | 4 ++-- src/erlcloud_ddb2.erl | 2 +- src/erlcloud_ddb_impl.erl | 4 ++-- src/erlcloud_elb.erl | 2 +- src/erlcloud_kinesis_impl.erl | 2 +- src/erlcloud_lambda.erl | 2 +- src/erlcloud_mturk.erl | 4 ++-- src/erlcloud_retry.erl | 2 +- src/erlcloud_s3.erl | 2 +- test/erlcloud_cloudformation_tests.erl | 4 ++-- test/erlcloud_sns_tests.erl | 2 +- 12 files changed, 17 insertions(+), 17 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index dd647683b..92e3c4667 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -143,7 +143,7 @@ security_token=undefined::string()|undefined, %% epoch seconds when temporary credentials will expire expiration=undefined :: pos_integer()|undefined, - %% Network request timeout; if not specifed, the default timeout will be used: + %% Network request timeout; if not specified, the default timeout will be used: %% ddb: 1s for initial call, 10s for subsequence; %% s3:delete_objects_batch/{2,3}, cloudtrail: 1s; %% other services: 10s. @@ -155,7 +155,7 @@ %% Note: If the lhttpc pool does not exists it will be created one with the lhttpc %% default settings [{connection_timeout,300000},{pool_size,1000}] lhttpc_pool=undefined::atom(), - %% Default to not retry failures (for backwards compatability). + %% Default to not retry failures (for backwards compatibility). %% Recommended to be set to default_retry to provide recommended retry behavior. %% Currently only affects S3 and service modules which use erlcloud_aws %% for issuing HTTP request to AWS, but intent is to change other services to use this as well. diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index e58a154c2..bd8663640 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -519,7 +519,7 @@ auto_config() -> %% This function works the same as {@link auto_config/0}, but if credentials %% are developed from User Profile as the source, the %% Options parameter provided will be used to control the -%% behavior. The credential profile name choosen can be controlled by +%% behavior. The credential profile name chosen can be controlled by %% providing {profile, atom()} as part of the options, and if %% not specified the default profile will be used. %% @@ -656,7 +656,7 @@ clear_expired_configs() -> %% %% If an invalid service name is provided, then this will throw an error, %% presuming that this is just a coding error. This behavior allows the -%% chaining of calls to this interface to allow concise configuraiton of a +%% chaining of calls to this interface to allow concise configuration of a %% config for multiple services. %% service_config( Service, Region, Config ) when is_atom(Service) -> diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index e3d3d555d..9ee1f75ce 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -3196,7 +3196,7 @@ q(Table, KeyConditionsOrExpression, Opts) -> %% %% ===Example=== %% -%% Get up to 3 itesm from the "Thread" table with "ForumName" of +%% Get up to 3 items from the "Thread" table with "ForumName" of %% "Amazon DynamoDB" and "LastPostDateTime" between specified %% value. Use the "LastPostIndex". %% diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 014dd7c6a..60d076c39 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -27,7 +27,7 @@ %% @author Ransom Richardson %% @doc %% -%% Implementation of requests to DynamoDB. This code is shared accross +%% Implementation of requests to DynamoDB. This code is shared across %% all API versions. %% %% The pluggable retry function provides a way to customize the retry behavior, as well @@ -165,7 +165,7 @@ request_id_from_error(#ddb2_error{response_headers = Headers}) when is_list(Head request_id_from_error(#ddb2_error{}) -> undefined. -%% For backwards compatability the default reason does not include the request id. +%% For backwards compatibility the default reason does not include the request id. %% This function will update the error to have reason containing the request id. -spec error_reason2(#ddb2_error{}) -> #ddb2_error{}. error_reason2(#ddb2_error{error_type = http} = Error) -> diff --git a/src/erlcloud_elb.erl b/src/erlcloud_elb.erl index c07ba2f3e..37a0af8e4 100644 --- a/src/erlcloud_elb.erl +++ b/src/erlcloud_elb.erl @@ -316,7 +316,7 @@ describe_load_balancer_policies(Config = #aws_config{}) -> %% -------------------------------------------------------------------- %% @doc describe_load_balancer_policies for specified ELB -%% with specificied policy names using default config. +%% with specified policy names using default config. %% @end %% -------------------------------------------------------------------- -spec describe_load_balancer_policies(string(), list() | aws_config()) -> diff --git a/src/erlcloud_kinesis_impl.erl b/src/erlcloud_kinesis_impl.erl index e9af4bd1c..bdc36f42a 100644 --- a/src/erlcloud_kinesis_impl.erl +++ b/src/erlcloud_kinesis_impl.erl @@ -27,7 +27,7 @@ %% @author Ransom Richardson %% @doc %% -%% Implementation of requests to DynamoDB. This code is shared accross +%% Implementation of requests to DynamoDB. This code is shared across %% all API versions. %% %% @end diff --git a/src/erlcloud_lambda.erl b/src/erlcloud_lambda.erl index 83c2d4cbb..0b7b258fb 100644 --- a/src/erlcloud_lambda.erl +++ b/src/erlcloud_lambda.erl @@ -357,7 +357,7 @@ get_function_configuration(Function, Qualifier, Config) -> %% [http://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html] %% %% option {show_headers, true|false} may be used for invoking Lambdas -%% this option changes returned spec of successfull Lambda invocation +%% this option changes returned spec of successful Lambda invocation %% false(default): output spec is {ok, Data} %% true: output spec is {ok, Headers, Data} where additional headers of %% Lambda invocation is returned diff --git a/src/erlcloud_mturk.erl b/src/erlcloud_mturk.erl index a0a94097a..e11653c9c 100644 --- a/src/erlcloud_mturk.erl +++ b/src/erlcloud_mturk.erl @@ -1286,8 +1286,8 @@ search_qualification_types(Options, Config) -> ). -spec send_test_event_notification(proplist(), mturk_event_type()) -> ok. -send_test_event_notification(Notificaiton, TestEventType) -> - send_test_event_notification(Notificaiton, TestEventType, default_config()). +send_test_event_notification(Notification, TestEventType) -> + send_test_event_notification(Notification, TestEventType, default_config()). -spec send_test_event_notification(proplist(), mturk_event_type(), aws_config()) -> ok. send_test_event_notification(Notification, TestEventType, Config) -> diff --git a/src/erlcloud_retry.erl b/src/erlcloud_retry.erl index 4d2944584..7f0b1066d 100644 --- a/src/erlcloud_retry.erl +++ b/src/erlcloud_retry.erl @@ -5,7 +5,7 @@ %% %% Implementation of retry logic for AWS requests %% -%% Currently only used for S3, but will be extended to other services in the furture. +%% Currently only used for S3, but will be extended to other services in the future. %% %% The pluggable retry function provides a way to customize the retry behavior, as well %% as log and customize errors that are generated by erlcloud. diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index ff4599480..d6eb8cc33 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -441,7 +441,7 @@ get_bucket_policy(BucketName) -> % % Example request: erlcloud_s3:get_bucket_policy("bucket1234", Config). -% Example success repsonse: {ok, "{\"Version\":\"2012-10-17\",\"Statement\": ..........} +% Example success response: {ok, "{\"Version\":\"2012-10-17\",\"Statement\": ..........} % Example error response: {error,{http_error,404,"Not Found", % "\n % diff --git a/test/erlcloud_cloudformation_tests.erl b/test/erlcloud_cloudformation_tests.erl index ebed26a99..afb005f1f 100644 --- a/test/erlcloud_cloudformation_tests.erl +++ b/test/erlcloud_cloudformation_tests.erl @@ -789,7 +789,7 @@ describe_stack_resource_output_tests(_) -> MyDBInstance Resource ID Resource Type - Tiemstamp + Timestamp Status @@ -814,7 +814,7 @@ describe_stack_resource_output_tests(_) -> describe_stack_events_output_tests(_) -> - Test = ?_cloudformation_test({"Test describe alls tack events", + Test = ?_cloudformation_test({"Test describe all tack events", <<" diff --git a/test/erlcloud_sns_tests.erl b/test/erlcloud_sns_tests.erl index ab043beb5..60cd9e9f9 100644 --- a/test/erlcloud_sns_tests.erl +++ b/test/erlcloud_sns_tests.erl @@ -754,7 +754,7 @@ list_subscriptions_by_topic_input_tests(_) -> {"TopicArn", "Arn"} ]}), ?_sns_test( - {"Test lists Subscriptions toke.", + {"Test lists Subscriptions token.", ?_f(erlcloud_sns:list_subscriptions_by_topic("Arn", "Token")), [ {"Action","ListSubscriptionsByTopic"}, From 56a5e94a180d0833499845c606d2d7a96b577ea1 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Tue, 15 Feb 2022 11:33:46 -0500 Subject: [PATCH 241/310] Use new EC2 API version for regions discovery. --- src/erlcloud_ec2.erl | 15 +++++++++++---- test/erlcloud_ec2_tests.erl | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index a5bc09244..d50a96390 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -21,7 +21,7 @@ %% Availability Zones and Regions describe_availability_zones/0, describe_availability_zones/1, describe_availability_zones/2, - describe_regions/0, describe_regions/1, describe_regions/2, + describe_regions/0, describe_regions/1, describe_regions/2, describe_regions/3, %% Elastic Block Store attach_volume/3, attach_volume/4, @@ -1689,16 +1689,23 @@ describe_regions(Config) when is_record(Config, aws_config) -> describe_regions([], Config); describe_regions(RegionNames) -> - describe_regions(RegionNames, default_config()). + describe_regions(RegionNames, none, default_config()). -spec describe_regions([string()], aws_config()) -> ok_error(proplist()). describe_regions(RegionNames, Config) + when is_record(Config, aws_config) -> + describe_regions(RegionNames, none, Config). + +-spec describe_regions([string()], none | filter_list(), aws_config()) -> ok_error(proplist()). +describe_regions(RegionNames, Filter, Config) when is_list(RegionNames) -> - case ec2_query(Config, "DescribeRegions", erlcloud_aws:param_list(RegionNames, "RegionName")) of + Params = erlcloud_aws:param_list(RegionNames, "RegionName") ++ list_to_ec2_filter(Filter), + case ec2_query(Config, "DescribeRegions", Params, ?NEW_API_VERSION) of {ok, Doc} -> Items = xmerl_xpath:string("/DescribeRegionsResponse/regionInfo/item", Doc), {ok, [[{region_name, get_text("regionName", Item)}, - {region_endpoint, get_text("regionEndpoint", Item)} + {region_endpoint, get_text("regionEndpoint", Item)}, + {opt_in_status, get_text("optInStatus", Item)} ] || Item <- Items]}; {error, _} = Error -> Error diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 7e18a71c9..d24a6ecdb 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -39,6 +39,7 @@ describe_test_() -> fun stop/1, [ fun describe_vpcs_tests/1, + fun describe_regions_tests/1, fun describe_tags_input_tests/1, fun describe_tags_output_tests/1, fun request_spot_fleet_input_tests/1, @@ -252,6 +253,41 @@ describe_vpcs_tests(_) -> ], output_tests(?_f(erlcloud_ec2:describe_vpcs()), Tests). +describe_regions_tests(_) -> + Tests = [ + ?_ec2_test({ + "Describe all the regions", + " + 59dbff89-35bd-4eac-99ed-be587EXAMPLE + + + af-south-1 + ec2.af-south-1.amazonaws.com + opted-in + + + ap-northeast-3 + ec2.ap-northeast-3.amazonaws.com + opted-in + + + ", + {ok, [ + [ + {region_name,"af-south-1"}, + {region_endpoint,"ec2.af-south-1.amazonaws.com"}, + {opt_in_status,"opted-in"} + ], + [ + {region_name,"ap-northeast-3"}, + {region_endpoint,"ec2.ap-northeast-3.amazonaws.com"}, + {opt_in_status,"opted-in"} + ] + ]} + }) + ], + output_tests(?_f(erlcloud_ec2:describe_regions([], [{"opt-in-status", "opted-in"}], erlcloud_aws:default_config())), Tests). + %% DescribeTags test based on the API examples: %% http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeTags.html describe_tags_input_tests(_) -> From ca225f71a4d4f39465aa8f8bc7254b9d40345c15 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Wed, 16 Feb 2022 05:37:16 -0500 Subject: [PATCH 242/310] Address comments --- src/erlcloud_ec2.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index d50a96390..72c3529c3 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -1689,7 +1689,7 @@ describe_regions(Config) when is_record(Config, aws_config) -> describe_regions([], Config); describe_regions(RegionNames) -> - describe_regions(RegionNames, none, default_config()). + describe_regions(RegionNames, default_config()). -spec describe_regions([string()], aws_config()) -> ok_error(proplist()). describe_regions(RegionNames, Config) From 4be90404993e9a318fe164efd1f13988fea2af55 Mon Sep 17 00:00:00 2001 From: Kirill Bondarenko <32926637+key-master@users.noreply.github.com> Date: Fri, 15 Apr 2022 13:13:18 +0300 Subject: [PATCH 243/310] use dot in S3 host name --- src/erlcloud_aws.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index bd8663640..59bb4ced2 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -826,7 +826,7 @@ service_host( <<"s3">>, <<"us-gov-west-1">> ) -> "s3-fips-us-gov-west-1.amazonaw service_host( <<"s3">>, <<"cn-north-1">> ) -> "s3.cn-north-1.amazonaws.com.cn"; service_host( <<"s3">>, <<"cn-northwest-1">> ) -> "s3.cn-northwest-1.amazonaws.com.cn"; service_host( <<"s3">>, Region ) -> - binary_to_list( <<"s3-", Region/binary, ".amazonaws.com">> ); + binary_to_list( <<"s3.", Region/binary, ".amazonaws.com">> ); service_host( <<"iam">>, <<"cn-north-1">> ) -> "iam.amazonaws.com.cn"; service_host( <<"iam">>, <<"cn-northwest-1">> ) -> "iam.amazonaws.com.cn"; service_host( <<"sdb">>, <<"us-east-1">> ) -> "sdb.amazonaws.com"; From be671ce3ff7336dcf863f1d38d2adfe55b2f8cb4 Mon Sep 17 00:00:00 2001 From: Kirill Bondarenko Date: Fri, 15 Apr 2022 13:38:58 +0300 Subject: [PATCH 244/310] update tests --- test/erlcloud_aws_tests.erl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/erlcloud_aws_tests.erl b/test/erlcloud_aws_tests.erl index 5e454f7e6..b6e60d21e 100644 --- a/test/erlcloud_aws_tests.erl +++ b/test/erlcloud_aws_tests.erl @@ -747,15 +747,15 @@ service_config_s3_test() -> <<"sa-east-1">>, <<"us-gov-west-1">>, <<"cn-north-1">>, <<"cn-northwest-1">>], Expected = ["s3-external-1.amazonaws.com", - "s3-us-west-1.amazonaws.com", - "s3-us-west-2.amazonaws.com", - "s3-eu-west-1.amazonaws.com", - "s3-eu-central-1.amazonaws.com", - "s3-ap-northeast-1.amazonaws.com", - "s3-ap-northeast-2.amazonaws.com", - "s3-ap-southeast-1.amazonaws.com", - "s3-ap-southeast-2.amazonaws.com", - "s3-sa-east-1.amazonaws.com", + "s3.us-west-1.amazonaws.com", + "s3.us-west-2.amazonaws.com", + "s3.eu-west-1.amazonaws.com", + "s3.eu-central-1.amazonaws.com", + "s3.ap-northeast-1.amazonaws.com", + "s3.ap-northeast-2.amazonaws.com", + "s3.ap-southeast-1.amazonaws.com", + "s3.ap-southeast-2.amazonaws.com", + "s3.sa-east-1.amazonaws.com", "s3-fips-us-gov-west-1.amazonaws.com", "s3.cn-north-1.amazonaws.com.cn", "s3.cn-northwest-1.amazonaws.com.cn"], From 975173f1f1d5a499e595c81bb1c010cdaf0011b9 Mon Sep 17 00:00:00 2001 From: Konstantin Kuzmin Date: Mon, 23 May 2022 08:49:12 -0400 Subject: [PATCH 245/310] Add AllRegions option to describe_regions. --- src/erlcloud_ec2.erl | 13 ++++++++++--- test/erlcloud_ec2_tests.erl | 26 ++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index 72c3529c3..c45a67ba5 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -21,7 +21,7 @@ %% Availability Zones and Regions describe_availability_zones/0, describe_availability_zones/1, describe_availability_zones/2, - describe_regions/0, describe_regions/1, describe_regions/2, describe_regions/3, + describe_regions/0, describe_regions/1, describe_regions/2, describe_regions/3, describe_regions/4, %% Elastic Block Store attach_volume/3, attach_volume/4, @@ -1698,8 +1698,15 @@ describe_regions(RegionNames, Config) -spec describe_regions([string()], none | filter_list(), aws_config()) -> ok_error(proplist()). describe_regions(RegionNames, Filter, Config) - when is_list(RegionNames) -> - Params = erlcloud_aws:param_list(RegionNames, "RegionName") ++ list_to_ec2_filter(Filter), + when is_list(RegionNames), is_record(Config, aws_config) -> + describe_regions(RegionNames, Filter, false, Config). + +-spec describe_regions([string()], none | filter_list(), boolean(), aws_config()) -> ok_error(proplist()). +describe_regions(RegionNames, Filter, AllRegions, Config) + when is_list(RegionNames), is_boolean(AllRegions), is_record(Config, aws_config) -> + Params = erlcloud_aws:param_list(RegionNames, "RegionName") ++ + list_to_ec2_filter(Filter) ++ + [{"AllRegions", AllRegions}], case ec2_query(Config, "DescribeRegions", Params, ?NEW_API_VERSION) of {ok, Doc} -> Items = xmerl_xpath:string("/DescribeRegionsResponse/regionInfo/item", Doc), diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index d24a6ecdb..a79d099f8 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -39,7 +39,8 @@ describe_test_() -> fun stop/1, [ fun describe_vpcs_tests/1, - fun describe_regions_tests/1, + fun describe_regions_input_tests/1, + fun describe_regions_output_tests/1, fun describe_tags_input_tests/1, fun describe_tags_output_tests/1, fun request_spot_fleet_input_tests/1, @@ -253,7 +254,28 @@ describe_vpcs_tests(_) -> ], output_tests(?_f(erlcloud_ec2:describe_vpcs()), Tests). -describe_regions_tests(_) -> +describe_regions_input_tests(_) -> + Tests = + [?_ec2_test( + {"This example describes all regions.", + ?_f(erlcloud_ec2:describe_regions([], [{"opt-in-status", "opted-in"}], erlcloud_aws:default_config())), + [{"Action", "DescribeRegions"}, + {"Filter.1.Name","opt-in-status"}, + {"Filter.1.Value.1","opted-in"}, + {"AllRegions","false"}]}), + ?_ec2_test( + {"This example describes all regions.", + ?_f(erlcloud_ec2:describe_regions([], [], true, erlcloud_aws:default_config())), + [{"Action", "DescribeRegions"}, + {"AllRegions","true"}]})], + Response = " + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +", + input_tests(Response, Tests). + +describe_regions_output_tests(_) -> Tests = [ ?_ec2_test({ "Describe all the regions", From a080fe22ac0c8625c2506cdb36b12badf5469910 Mon Sep 17 00:00:00 2001 From: Amershan Date: Fri, 10 Jun 2022 12:31:13 +0100 Subject: [PATCH 246/310] Support AWS cognito (#727) * Support AWS cognito * Filter only optional fields * Add map guard to map filtering funs * Add version to the request headers * Rename erlcloud_congito -> erlcloud_cognito_user_pools * Fix specifications * Fix eunit tests for cognito user pools * Add cognito user pools to aws config funs Co-authored-by: arpad.budai --- include/erlcloud_aws.hrl | 12 +- src/erlcloud_aws.erl | 5 + src/erlcloud_cognito_user_pools.erl | 917 +++++++++++++++++ src/erlcloud_util.erl | 22 +- test/erlcloud_cognito_user_pools_tests.erl | 1059 ++++++++++++++++++++ 5 files changed, 2006 insertions(+), 9 deletions(-) create mode 100644 src/erlcloud_cognito_user_pools.erl create mode 100644 test/erlcloud_cognito_user_pools_tests.erl diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 92e3c4667..39cbe8072 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -1,11 +1,11 @@ -ifndef(erlcloud_aws_hrl). -define(erlcloud_aws_hrl, 0). --record(aws_assume_role,{ - role_arn :: string() | undefined, - session_name = "erlcloud" :: string(), - duration_secs = 900 :: 900..43200, - external_id :: string() | undefined +-record(aws_assume_role, { + role_arn :: string() | undefined, + session_name = "erlcloud" :: string(), + duration_secs = 900 :: 900..43200, + external_id :: string() | undefined }). -type(aws_assume_role() :: #aws_assume_role{}). @@ -37,6 +37,8 @@ s3_bucket_access_method=vhost::vhost|path|auto, s3_bucket_after_host=false::boolean(), sdb_host="sdb.amazonaws.com"::string(), + cognito_user_pools_host ="cognito-idp.eu-west-1.amazonaws.com"::string(), + cognito_user_pools_scheme ="https://"::string()|undefined, elb_host="elasticloadbalancing.amazonaws.com"::string(), rds_host="rds.us-east-1.amazonaws.com"::string(), ses_host="email.us-east-1.amazonaws.com"::string(), diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 59bb4ced2..14a4086d4 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -784,6 +784,9 @@ service_config( <<"glue">> = Service, Region, Config ) -> service_config( <<"athena">> = Service, Region, Config ) -> Host = service_host( Service, Region ), Config#aws_config{ athena_host = Host }; +service_config( <<"cognito_user_pools">> = Service, Region, Config ) -> + Host = service_host( Service, Region ), + Config#aws_config{ cognito_user_pools_host = Host }; service_config( <<"states">> = Service, Region, Config ) -> Host = service_host( Service, Region ), Config#aws_config{ states_host = Host }; @@ -821,6 +824,8 @@ service_config( <<"ssm">> = Service, Region, Config ) -> %% This function handles the special and general cases of service host %% names. %% +service_host( <<"cognito_user_pools">>, Region ) -> + binary_to_list(<<"cognito-idp.", Region/binary, ".amazonaws.com">>); service_host( <<"s3">>, <<"us-east-1">> ) -> "s3-external-1.amazonaws.com"; service_host( <<"s3">>, <<"us-gov-west-1">> ) -> "s3-fips-us-gov-west-1.amazonaws.com"; service_host( <<"s3">>, <<"cn-north-1">> ) -> "s3.cn-north-1.amazonaws.com.cn"; diff --git a/src/erlcloud_cognito_user_pools.erl b/src/erlcloud_cognito_user_pools.erl new file mode 100644 index 000000000..8ffa263ce --- /dev/null +++ b/src/erlcloud_cognito_user_pools.erl @@ -0,0 +1,917 @@ +-module(erlcloud_cognito_user_pools). + +-include("erlcloud_aws.hrl"). + +-export([configure/2, configure/3, new/2, new/3]). + +-export([ + list_users/1, + list_users/2, + list_users/5, + list_users/6, + list_all_users/1, + list_all_users/2, + list_all_users/3, + + admin_list_groups_for_user/2, + admin_list_groups_for_user/3, + admin_list_groups_for_user/4, + admin_list_groups_for_user/5, + + admin_get_user/2, + admin_get_user/3, + + admin_create_user/2, + admin_create_user/3, + admin_create_user/4, + admin_delete_user/2, + admin_delete_user/3, + + admin_add_user_to_group/3, + admin_add_user_to_group/4, + + admin_remove_user_from_group/3, + admin_remove_user_from_group/4, + + create_group/2, + create_group/3, + create_group/5, + create_group/6, + + delete_group/2, + delete_group/3, + + admin_reset_user_password/2, + admin_reset_user_password/3, + admin_reset_user_password/4, + + admin_update_user_attributes/3, + admin_update_user_attributes/4, + admin_update_user_attributes/5, + + change_password/3, + change_password/4, + + list_user_pools/0, + list_user_pools/1, + list_user_pools/2, + list_all_user_pools/0, + list_all_user_pools/1, + + admin_set_user_password/3, + admin_set_user_password/4, + admin_set_user_password/5, + + describe_user_pool/1, + describe_user_pool/2, + + get_user_pool_mfa_config/1, + get_user_pool_mfa_config/2, + + list_identity_providers/1, + list_identity_providers/3, + list_identity_providers/4, + list_all_identity_providers/1, + list_all_identity_providers/2, + + describe_identity_provider/2, + describe_identity_provider/3, + + describe_user_pool_client/2, + describe_user_pool_client/3, + + list_user_pool_clients/1, + list_user_pool_clients/3, + list_user_pool_clients/4, + list_all_user_pool_clients/1, + list_all_user_pool_clients/2, + + admin_list_devices/2, + admin_list_devices/3, + admin_list_devices/5, + admin_list_all_devices/2, + admin_list_all_devices/3, + + admin_forget_device/3, + admin_forget_device/4, + + admin_confirm_signup/2, + admin_confirm_signup/3, + admin_confirm_signup/4, + + admin_initiate_auth/4, + admin_initiate_auth/5, + admin_initiate_auth/8, + + respond_to_auth_challenge/4, + respond_to_auth_challenge/5, + respond_to_auth_challenge/8, + + create_identity_provider/4, + create_identity_provider/5, + create_identity_provider/6, + create_identity_provider/7, + + delete_identity_provider/2, + delete_identity_provider/3, + + update_identity_provider/2, + update_identity_provider/3, + update_identity_provider/4, + update_identity_provider/5, + update_identity_provider/6, + + request/2, + request/3 +]). + +-define(MAX_RESULTS, 60). +-define(API_VERSION, "2016-04-18"). + +-spec new(string(), string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey) -> + #aws_config{access_key_id = AccessKeyID, + secret_access_key = SecretAccessKey, + retry = fun erlcloud_retry:default_retry/1}. + +-spec new(string(), string(), string()) -> aws_config(). +new(AccessKeyID, SecretAccessKey, Host) -> + #aws_config{access_key_id = AccessKeyID, + secret_access_key = SecretAccessKey, + cognito_user_pools_host = Host, + retry = fun erlcloud_retry:default_retry/1}. + +-spec configure(string(), string()) -> ok. +configure(AccessKeyID, SecretAccessKey) -> + put(aws_config, new(AccessKeyID, SecretAccessKey)), + ok. + +-spec configure(string(), string(), string()) -> ok. +configure(AccessKeyID, SecretAccessKey, Host) -> + put(aws_config, new(AccessKeyID, SecretAccessKey, Host)), + ok. + +-spec list_users(binary()) -> {ok, map()} | {error, any()}. +list_users(UserPoolId) -> + list_users(UserPoolId, undefined, undefined, undefined, undefined). + +-spec list_users(binary(), aws_config()) -> {ok, map()} | {error, any()}. +list_users(UserPoolId, Config) -> + Body = #{ + <<"UserPoolId">> => unicode:characters_to_binary(UserPoolId) + }, + request(Config, "ListUsers", Body). + +-spec list_users(binary(), + [binary()] | undefined, + binary() | undefined, + number() | undefined, + binary() | undefined) -> {ok, map()} | {error, any()}. +list_users(UserPoolId, AttributesToGet, Filter, Limit, PaginationToken) -> + Config = erlcloud_aws:default_config(), + list_users(UserPoolId, AttributesToGet, Filter, Limit, PaginationToken, Config). + +list_users(UserPoolId, AttributesToGet, Filter, Limit, PaginationToken, Config) -> + BaseBody = #{ + <<"UserPoolId">> => UserPoolId, + <<"AttributesToGet">> => AttributesToGet, + <<"Filter">> => Filter, + <<"Limit">> => Limit, + <<"PaginationToken">> => PaginationToken + }, + Body = erlcloud_util:filter_undef(BaseBody), + request(Config, "ListUsers", Body). + +-spec list_all_users(binary()) -> {ok, map()} | {error, any()}. +list_all_users(UserPoolId) -> + list_all_users(UserPoolId, undefined). + +-spec list_all_users(binary(), binary() | undefined | aws_config()) -> + {ok, map()} | {error, any()}. +list_all_users(UserPoolId, Config) when is_record(Config, aws_config) -> + list_all_users(UserPoolId, undefined, Config); +list_all_users(UserPoolId, Filter) -> + Config = erlcloud_aws:default_config(), + list_all_users(UserPoolId, Filter, Config). + +-spec list_all_users(binary(), binary() | undefined, aws_config()) -> + {ok, map()} | {error, any()}. +list_all_users(UserPoolId, Filter, Config) -> + Fun = fun list_users/6, + Args = [UserPoolId, undefined, Filter], + list_all(Fun, Args, Config, <<"Users">>, <<"PaginationToken">>). + +-spec admin_list_groups_for_user(binary(), binary()) -> + {ok, map()} | {error, any()}. +admin_list_groups_for_user(UserName, UserPoolId) -> + Config = erlcloud_aws:default_config(), + admin_list_groups_for_user(UserName, UserPoolId, Config). + +-spec admin_list_groups_for_user(binary(), binary(), aws_config()) -> + {ok, map()} | {error, any()}. +admin_list_groups_for_user(UserName, UserPoolId, Config) -> + Body = #{ + <<"Username">> => UserName, + <<"UserPoolId">> => UserPoolId + }, + request(Config, "AdminListGroupsForUser", Body). + +-spec admin_list_groups_for_user(binary(), binary(), number(), + binary() | undefined) -> + {ok, map()} | {error, any()}. +admin_list_groups_for_user(UserName, UserPoolId, Limit, NextToken) -> + Config = erlcloud_aws:default_config(), + admin_list_groups_for_user(UserName, UserPoolId, Limit, NextToken, Config). + +-spec admin_list_groups_for_user(binary(), binary(), number(), + binary() | undefined, aws_config()) -> + {ok, map()} | {error, any()}. +admin_list_groups_for_user(UserName, UserPoolId, Limit, NextToken, Config) -> + Body = #{ + <<"Username">> => UserName, + <<"UserPoolId">> => UserPoolId, + <<"Limit">> => Limit, + <<"NextToken">> => NextToken + }, + request(Config, "AdminListGroupsForUser", Body). + +-spec admin_get_user(binary(), binary()) -> {ok, map()} | {error, any()}. +admin_get_user(UserName, UserPoolId) -> + Config = erlcloud_aws:default_config(), + admin_get_user(UserName, UserPoolId, Config). + +-spec admin_get_user(binary(), binary(), aws_config()) -> + {ok, map()} | {error, any()}. +admin_get_user(UserName, UserPoolId, Config) -> + Body = #{ + <<"Username">> => UserName, + <<"UserPoolId">> => UserPoolId + }, + request(Config, "AdminGetUser", Body). + +-spec admin_create_user(binary(), binary()) -> + {ok, map()} | {error, any()}. +admin_create_user(UserName, UserPoolId) -> + admin_create_user(UserName, UserPoolId, #{}). + +-spec admin_create_user(binary(), binary(), maps:maps()) -> + {ok, map()} | {error, any()}. +admin_create_user(UserName, UserPoolId, OptionalArgs) -> + Config = erlcloud_aws:default_config(), + admin_create_user(UserName, UserPoolId, OptionalArgs, Config). + +-spec admin_create_user(binary(), binary(), maps:maps(), aws_config()) -> + {ok, map()} | {error, any()}. +admin_create_user(UserName, UserPoolId, OptionalArgs, Config) -> + Body = OptionalArgs#{ + <<"Username">> => UserName, + <<"UserPoolId">> => UserPoolId + }, + request(Config, "AdminCreateUser", Body). + +-spec admin_delete_user(binary(), binary()) -> ok | {error, any()}. +admin_delete_user(UserName, UserPoolId) -> + Config = erlcloud_aws:default_config(), + admin_delete_user(UserName, UserPoolId, Config). + +-spec admin_delete_user(binary(), binary(), aws_config()) -> ok | {error, any()}. +admin_delete_user(UserName, UserPoolId, Config) -> + Body = #{ + <<"Username">> => UserName, + <<"UserPoolId">> => UserPoolId + }, + request_no_resp(Config, "AdminDeleteUser", Body). + +-spec admin_add_user_to_group(binary(), binary(), binary()) -> + ok | {error, any()}. +admin_add_user_to_group(GroupName, UserName, UserPoolId) -> + Config = erlcloud_aws:default_config(), + admin_add_user_to_group(GroupName, UserName, UserPoolId, Config). + +-spec admin_add_user_to_group(binary(), binary(), binary(), aws_config()) -> + ok | {error, any()}. +admin_add_user_to_group(GroupName, UserName, UserPoolId, Config) -> + Body = #{ + <<"Username">> => UserName, + <<"GroupName">> => GroupName, + <<"UserPoolId">> => UserPoolId + }, + request_no_resp(Config, "AdminAddUserToGroup", Body). + +-spec admin_remove_user_from_group(binary(), binary(), binary()) -> + ok | {error, any()}. +admin_remove_user_from_group(GroupName, UserName, UserPoolId) -> + Config = erlcloud_aws:default_config(), + admin_remove_user_from_group(GroupName, UserName, UserPoolId, Config). + +-spec admin_remove_user_from_group(binary(), binary(), binary(), aws_config()) -> + ok | {error, any()}. +admin_remove_user_from_group(GroupName, UserName, UserPoolId, Config) -> + Body = #{ + <<"Username">> => UserName, + <<"GroupName">> => GroupName, + <<"UserPoolId">> => UserPoolId + }, + request_no_resp(Config, "AdminRemoveUserFromGroup", Body). + +-spec create_group(binary(), binary()) -> {ok, map()} | {error, any()}. +create_group(GroupName, UserPoolId) -> + create_group(GroupName, UserPoolId, undefined, undefined, undefined). + +-spec create_group(binary(), binary(), aws_config()) -> + {ok, map()} | {error, any()}. +create_group(GroupName, UserPoolId, Config) -> + create_group(GroupName, UserPoolId, undefined, undefined, undefined, Config). + +-spec create_group(binary(), binary(), binary() | undefined, + number() | undefined, binary() | undefined) -> + {ok, map()} | {error, any()}. +create_group(GroupName, UserPoolId, Description, Precedence, RoleArn) -> + Config = erlcloud_aws:default_config(), + create_group(GroupName, UserPoolId, Description, Precedence, RoleArn, Config). + +-spec create_group(binary(), binary(), binary() | undefined, + number() | undefined, binary() | undefined, aws_config()) -> + {ok, map()} | {error, any()}. +create_group(GroupName, UserPoolId, Description, Precedence, RoleArn, Config) -> + Body0 = #{ + <<"GroupName">> => GroupName, + <<"UserPoolId">> => UserPoolId, + <<"Description">> => Description, + <<"Precedence">> => Precedence, + <<"RoleArn">> => RoleArn + }, + + Body = erlcloud_util:filter_undef(Body0), + request(Config, "CreateGroup", Body). + +-spec delete_group(binary(), binary()) -> ok | {error, any()}. +delete_group(GroupName, UserPoolId) -> + Config = erlcloud_aws:default_config(), + delete_group(GroupName, UserPoolId, Config). + +-spec delete_group(binary(), binary(), aws_config()) -> ok | {error, any()}. +delete_group(GroupName, UserPoolId, Config) -> + Body = #{ + <<"GroupName">> => unicode:characters_to_binary(GroupName), + <<"UserPoolId">> => unicode:characters_to_binary(UserPoolId) + }, + request_no_resp(Config, "DeleteGroup", Body). + +-spec admin_reset_user_password(binary(), binary()) -> + ok| {error, any()}. +admin_reset_user_password(UserName, UserPoolId) -> + admin_reset_user_password(UserName, UserPoolId, undefined). + +-spec admin_reset_user_password(binary(), binary(), map() | undefined) -> + ok | {error, any()}. +admin_reset_user_password(UserName, UserPoolId, MetaData) -> + Config = erlcloud_aws:default_config(), + admin_reset_user_password(UserName, UserPoolId, MetaData, Config). + +-spec admin_reset_user_password(binary(), binary(), + map() | undefined, aws_config()) -> + ok | {error, any()}. +admin_reset_user_password(UserName, UserPoolId, MetaData, Config) -> + BaseBody = #{ + <<"Username">> => UserName, + <<"UserPoolId">> => UserPoolId, + <<"ClientMetaData">> => MetaData + }, + Body = erlcloud_util:filter_undef(BaseBody), + request_no_resp(Config, "AdminResetUserPassword", Body). + +-spec admin_update_user_attributes(binary(), binary(), [map()]) -> + ok | {error, any()}. +admin_update_user_attributes(UserName, UserPoolId, Attributes) -> + admin_update_user_attributes(UserName, UserPoolId, Attributes, undefined). + +-spec admin_update_user_attributes(binary(), binary(), [map()], + map() | undefined) -> + ok | {error, any()}. +admin_update_user_attributes(UserName, UserPoolId, Attributes, MetaData) -> + Config = erlcloud_aws:default_config(), + admin_update_user_attributes(UserName, UserPoolId, Attributes, MetaData, Config). + +-spec admin_update_user_attributes(binary(), binary(), [map()], + map() | undefined, aws_config()) -> + ok | {error, any()}. +admin_update_user_attributes(UserName, UserPoolId, Attributes, MetaData, Config) -> + BaseBody = #{ + <<"Username">> => UserName, + <<"UserPoolId">> => UserPoolId, + <<"UserAttributes">> => Attributes, + <<"ClientMetaData">> => MetaData + }, + Body = erlcloud_util:filter_undef(BaseBody), + request_no_resp(Config, "AdminUpdateUserAttributes", Body). + +-spec change_password(binary(), binary(), binary()) -> + ok | {error, any()}. +change_password(OldPass, NewPass, AccessToken) -> + Config = erlcloud_aws:default_config(), + change_password(OldPass, NewPass, AccessToken, Config). + +-spec change_password(binary(), binary(), binary(), aws_config()) -> + ok | {error, any()}. +change_password(OldPass, NewPass, AccessToken, Config) -> + Body = #{ + <<"AccessToken">> => AccessToken, + <<"PreviousPassword">> => OldPass, + <<"ProposedPassword">> => NewPass + }, + request_no_resp(Config, "ChangePassword", Body). + +-spec list_user_pools() -> {ok, map()} | {error, any()}. +list_user_pools() -> + list_user_pools(?MAX_RESULTS, undefined). + +-spec list_user_pools(integer()) -> {ok, map()} | {error, any()}. +list_user_pools(MaxResult) -> + list_user_pools(MaxResult, undefined). + +-spec list_user_pools(integer(), binary() | undefined) -> + {ok, map()} | {error, any()}. +list_user_pools(MaxResult, NextToken) -> + Config = erlcloud_aws:default_config(), + list_user_pools(MaxResult, NextToken, Config). + +-spec list_user_pools(integer(), binary() | undefined, aws_config()) -> + {ok, map()} | {error, any()}. +list_user_pools(MaxResult, NextToken, Config) -> + Body0 = #{ + <<"MaxResults">> => MaxResult, + <<"NextToken">> => NextToken + }, + Body = erlcloud_util:filter_undef(Body0), + request(Config, "ListUserPools", Body). + +-spec list_all_user_pools() -> {ok, map()} | {error, any()}. +list_all_user_pools() -> + Config = erlcloud_aws:default_config(), + list_all_user_pools(Config). + +-spec list_all_user_pools(aws_config()) -> {ok, map()} | {error, any()}. +list_all_user_pools(Config) -> + Fun = fun list_user_pools/3, + list_all(Fun, [], Config, <<"UserPools">>, <<"NextToken">>). + +-spec admin_set_user_password(binary(), binary(), binary()) -> + {ok, map()} | {error, any()}. +admin_set_user_password(UserId, UserPoolId, Password) -> + admin_set_user_password(UserId, UserPoolId, Password, false). + +-spec admin_set_user_password(binary(), binary(), binary(), boolean()) -> + ok | {error, any()}. +admin_set_user_password(UserId, UserPoolId, Password, Permanent) -> + Config = erlcloud_aws:default_config(), + admin_set_user_password(UserId, UserPoolId, Password, Permanent, Config). + +-spec admin_set_user_password(binary(), binary(), binary(), boolean(), + aws_config()) -> + ok | {error, any()}. +admin_set_user_password(UserId, UserPoolId, Password, Permanent, Config) -> + Body = #{ + <<"Password">> => Password, + <<"Username">> => UserId, + <<"UserPoolId">> => UserPoolId, + <<"Permanent">> => Permanent + }, + request_no_resp(Config, "AdminSetUserPassword", Body). + +-spec describe_user_pool(binary()) -> {ok, map()} | {error, any()}. +describe_user_pool(UserPoolId) -> + Config = erlcloud_aws:default_config(), + describe_user_pool(UserPoolId, Config). + +-spec describe_user_pool(binary(), aws_config()) -> {ok, map()} | {error, any()}. +describe_user_pool(UserPoolId, Config) -> + Body = #{ + <<"UserPoolId">> => UserPoolId + }, + request(Config, "DescribeUserPool", Body). + +-spec get_user_pool_mfa_config(binary()) -> {ok, map()} | {error, any()}. +get_user_pool_mfa_config(UserPoolId) -> + Config = erlcloud_aws:default_config(), + get_user_pool_mfa_config(UserPoolId, Config). + +-spec get_user_pool_mfa_config(binary(), aws_config()) -> + {ok, map()} | {error, any()}. +get_user_pool_mfa_config(UserPoolId, Config) -> + Body = #{ + <<"UserPoolId">> => UserPoolId + }, + request(Config, "GetUserPoolMfaConfig", Body). + +-spec list_identity_providers(binary()) -> {ok, map()} | {error, any()}. +list_identity_providers(UserPoolId) -> + list_identity_providers(UserPoolId, ?MAX_RESULTS, undefined). + +-spec list_identity_providers(binary(), integer(), binary() | undefined) -> + {ok, map()} | {error, any()}. +list_identity_providers(UserPoolId, MaxResults, NextToken) -> + Config = erlcloud_aws:default_config(), + list_identity_providers(UserPoolId, MaxResults, NextToken, Config). + +-spec list_identity_providers(binary(), + integer(), + binary() | undefined, + aws_config()) -> + {ok, map()} | {error, any()}. +list_identity_providers(UserPoolId, MaxResults, NextToken, Config) -> + Body0 = #{ + <<"UserPoolId">> => UserPoolId, + <<"NextToken">> => NextToken, + <<"MaxResults">> => MaxResults + }, + Body = erlcloud_util:filter_undef(Body0), + request(Config, "ListIdentityProviders", Body). + +-spec list_all_identity_providers(binary()) -> + {ok, map()} | {error, any()}. +list_all_identity_providers(UserPoolId) -> + Config = erlcloud_aws:default_config(), + list_all_identity_providers(UserPoolId, Config). + +-spec list_all_identity_providers(binary(), aws_config()) -> + {ok, map()} | {error, any()}. +list_all_identity_providers(UserPoolId, Config) -> + Fun = fun list_identity_providers/4, + Args = [UserPoolId], + list_all(Fun, Args, Config, <<"Providers">>, <<"NextToken">>). + +-spec describe_identity_provider(binary(), binary()) -> + {ok, map()} | {error, any()}. +describe_identity_provider(UserPoolId, ProviderName) -> + Config = erlcloud_aws:default_config(), + describe_identity_provider(UserPoolId, ProviderName, Config). + +-spec describe_identity_provider(binary(), binary(), aws_config()) -> + {ok, map()} | {error, any()}. +describe_identity_provider(UserPoolId, ProviderName, Config) -> + Body = #{ + <<"ProviderName">> => ProviderName, + <<"UserPoolId">> => UserPoolId + }, + request(Config, "DescribeIdentityProvider", Body). + +-spec describe_user_pool_client(binary(), binary()) -> + {ok, map()} | {error, any()}. +describe_user_pool_client(UserPoolId, ClientId) -> + Config = erlcloud_aws:default_config(), + describe_user_pool_client(UserPoolId, ClientId, Config). + +describe_user_pool_client(UserPoolId, ClientId, Config) -> + Body = #{ + <<"ClientId">> => ClientId, + <<"UserPoolId">> => UserPoolId + }, + request(Config, "DescribeUserPoolClient", Body). + +-spec list_user_pool_clients(binary()) -> {ok, map()} | {error, any()}. +list_user_pool_clients(UserPoolId) -> + list_user_pool_clients(UserPoolId, ?MAX_RESULTS, undefined). + +-spec list_user_pool_clients(binary(), non_neg_integer(), binary() | undefined) -> + {ok, map()} | {error, any()}. +list_user_pool_clients(UserPoolId, MaxResults, NextToken) -> + Config = erlcloud_aws:default_config(), + list_user_pool_clients(UserPoolId, MaxResults, NextToken, Config). + +-spec list_user_pool_clients(binary(), non_neg_integer(), binary() | undefined, + aws_config()) -> + {ok, map()} | {error, any()}. +list_user_pool_clients(UserPoolId, MaxResults, NextToken, Config) -> + Body0 = #{ + <<"UserPoolId">> => UserPoolId, + <<"NextToken">> => NextToken, + <<"MaxResults">> => MaxResults + }, + Body = erlcloud_util:filter_undef(Body0), + request(Config, "ListUserPoolClients", Body). + +-spec list_all_user_pool_clients(binary()) -> + {ok, map()} | {error, any()}. +list_all_user_pool_clients(UserPoolId) -> + Config = erlcloud_aws:default_config(), + list_all_user_pool_clients(UserPoolId, Config). + +-spec list_all_user_pool_clients(binary(), aws_config()) -> + {ok, map()} | {error, any()}. +list_all_user_pool_clients(UserPoolId, Config) -> + Fun = fun list_user_pool_clients/4, + Args = [UserPoolId], + list_all(Fun, Args, Config, <<"UserPoolClients">>, <<"NextToken">>). + +-spec admin_list_devices(binary(), binary()) -> {ok, map()} | {error, any()}. +admin_list_devices(UserPoolId, Username) -> + Config = erlcloud_aws:default_config(), + admin_list_devices(UserPoolId, Username, Config). + +-spec admin_list_devices(binary(), binary(), aws_config()) -> + {ok, map()} | {error, any()}. +admin_list_devices(UserPoolId, Username, Config) -> + admin_list_devices(UserPoolId, Username, ?MAX_RESULTS, undefined, Config). + +-spec admin_list_devices(binary(), binary(), integer(), binary() | undefined, + aws_config()) -> + {ok, map()} | {error, any()}. +admin_list_devices(UserPoolId, Username, Limit, PaginationToken, Config) -> + Body0 = #{ + <<"UserPoolId">> => UserPoolId, + <<"Username">> => Username, + <<"Limit">> => Limit, + <<"PaginationToken">> => PaginationToken + }, + Body = erlcloud_util:filter_undef(Body0), + request(Config, "AdminListDevices", Body). + +-spec admin_list_all_devices(binary(), binary()) -> + {ok, map()} | {error, any()}. +admin_list_all_devices(UserPoolId, Username) -> + Config = erlcloud_aws:default_config(), + admin_list_all_devices(UserPoolId, Username, Config). + +-spec admin_list_all_devices(binary(), binary(), aws_config()) -> + {ok, map()} | {error, any()}. +admin_list_all_devices(UserPoolId, Username, Config) -> + Fun = fun admin_list_devices/5, + Args = [UserPoolId, Username], + list_all(Fun, Args, Config, <<"Devices">>, <<"PaginationToken">>). + +-spec admin_forget_device(binary(), binary(), binary()) -> + ok | {error, any()}. +admin_forget_device(UserPoolId, Username, DeviceKey) -> + Config = erlcloud_aws:default_config(), + admin_forget_device(UserPoolId, Username, DeviceKey, Config). + +-spec admin_forget_device(binary(), binary(), binary(), aws_config()) -> + ok | {error, any()}. +admin_forget_device(UserPoolId, Username, DeviceKey, Config) -> + Body = #{ + <<"UserPoolId">> => UserPoolId, + <<"Username">> => Username, + <<"DeviceKey">> => DeviceKey + }, + request_no_resp(Config, "AdminForgetDevice", Body). + +-spec admin_confirm_signup(binary(), binary()) -> + {ok, map()} | {error, any()}. +admin_confirm_signup(UserPoolId, Username) -> + admin_confirm_signup(UserPoolId, Username, #{}). + +-spec admin_confirm_signup(binary(), binary(), maps:map()) -> + {ok, map()} | {error, any()}. +admin_confirm_signup(UserPoolId, Username, ClientMetadata) -> + Config = erlcloud_aws:default_config(), + admin_confirm_signup(UserPoolId, Username, ClientMetadata, Config). + +-spec admin_confirm_signup(binary(), binary(), maps:map(), aws_config()) -> + {ok, map()} | {error, any()}. +admin_confirm_signup(UserPoolId, Username, ClientMetadata, Config) -> + Body = #{ + <<"UserPoolId">> => UserPoolId, + <<"Username">> => Username, + <<"ClientMetadata">> => ClientMetadata + }, + request(Config, "AdminConfirmSignUp", Body). + +-spec admin_initiate_auth(binary(), binary(), binary(), maps:map()) -> + {ok, map()} | {error, any()}. +admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams) -> + Cfg = erlcloud_aws:default_config(), + admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, Cfg). + +-spec admin_initiate_auth(binary(), binary(), binary(), + maps:map(), aws_config()) -> + {ok, map()} | {error, any()}. +admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, Cfg) -> + admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, + #{}, #{}, #{}, Cfg). + +-spec admin_initiate_auth(binary(), binary(), binary(), maps:map(), + maps:map(), maps:map(), maps:map(), aws_config()) -> + {ok, map()} | {error, any()}. +admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, + AnalyticsMeta, ClientMeta, ContextData, Cfg) -> + Mandatory = #{ + <<"AuthFlow">> => AuthFlow, + <<"ClientId">> => ClientId, + <<"UserPoolId">> => PoolId + }, + Optional = #{ + <<"AnalyticsMetadata">> => AnalyticsMeta, + <<"AuthParameters">> => AuthParams, + <<"ClientMetadata">> => ClientMeta, + <<"ContextData">> => ContextData + }, + request(Cfg, "AdminInitiateAuth", make_request_body(Mandatory, Optional)). + +-spec respond_to_auth_challenge(binary(), binary(), maps:map(), binary()) -> + {ok, map()} | {error, any()}. +respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, Session) -> + Cfg = erlcloud_aws:default_config(), + respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, + Session, Cfg). + +-spec respond_to_auth_challenge(binary(), binary(), maps:map(), binary(), + aws_config()) -> + {ok, map()} | {error, any()}. +respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, + Session, Cfg) -> + respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, + Session, #{}, #{}, #{}, Cfg). + +-spec respond_to_auth_challenge(binary(), binary(), maps:map(), binary(), + maps:map(), maps:map(), maps:map(), + aws_config()) -> + {ok, map()} | {error, any()}. +respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, + Session, AnalyticsMeta, ClientMeta, ContextData, Cfg) -> + Mandatory = #{ + <<"ChallengeName">> => ChallengeName, + <<"ChallengeResponses">> => ChallengeResponses, + <<"ClientId">> => ClientId + }, + Optional = #{ + <<"AnalyticsMetadata">> => AnalyticsMeta, + <<"ClientMetadata">> => ClientMeta, + <<"Session">> => Session, + <<"UserContextData">> => ContextData + }, + request(Cfg, "RespondToAuthChallenge", make_request_body(Mandatory, Optional)). + +-spec create_identity_provider(binary(), binary(), binary(), map()) -> + {ok, map()} | {error, any()}. +create_identity_provider(UserPoolId, ProviderName, ProviderType, + ProviderDetails) -> + create_identity_provider(UserPoolId, ProviderName, ProviderType, + ProviderDetails, #{}). + +-spec create_identity_provider(binary(), binary(), binary(), map(), map()) -> + {ok, map()} | {error, any()}. +create_identity_provider(UserPoolId, ProviderName, ProviderType, + ProviderDetails, AttributeMapping) -> + create_identity_provider(UserPoolId, ProviderName, ProviderType, + ProviderDetails, AttributeMapping, []). + +-spec create_identity_provider(binary(), binary(), binary(), + map(), map(), list()) -> + {ok, map()} | {error, any()}. +create_identity_provider(UserPoolId, ProviderName, ProviderType, + ProviderDetails, AttributeMapping, IdpIdentifiers) -> + Config = erlcloud_aws:default_config(), + create_identity_provider(UserPoolId, ProviderName, ProviderType, + ProviderDetails, AttributeMapping, IdpIdentifiers, Config). + +-spec create_identity_provider(binary(), binary(), binary(), map(), map(), + list(), aws_config()) -> + {ok, map()} | {error, any()}. +create_identity_provider(UserPoolId, ProviderName, ProviderType, ProviderDetails, + AttributeMapping, IdpIdentifiers, Config) -> + Mandatory = #{ + <<"UserPoolId">> => UserPoolId, + <<"ProviderName">> => ProviderName, + <<"ProviderType">> => ProviderType, + <<"ProviderDetails">> => ProviderDetails + }, + Optional = #{ + <<"AttributeMapping">> => AttributeMapping, + <<"IdpIdentifiers">> => IdpIdentifiers + }, + request(Config, "CreateIdentityProvider", make_request_body(Mandatory, Optional)). + +-spec delete_identity_provider(binary(), binary()) -> + ok | {error, any()}. +delete_identity_provider(UserPoolId, ProviderName) -> + Config = erlcloud_aws:default_config(), + delete_identity_provider(UserPoolId, ProviderName, Config). + +-spec delete_identity_provider(binary(), binary(), aws_config()) -> + ok | {error, any()}. +delete_identity_provider(UserPoolId, ProviderName, Config) -> + Body = #{ + <<"UserPoolId">> => UserPoolId, + <<"ProviderName">> => ProviderName + }, + request_no_resp(Config, "DeleteIdentityProvider", Body). + +-spec update_identity_provider(binary(), binary()) -> + {ok, map()} | {error, any()}. +update_identity_provider(UserPoolId, ProviderName) -> + update_identity_provider(UserPoolId, ProviderName, #{}). + +-spec update_identity_provider(binary(), binary(), map()) -> + {ok, map()} | {error, any()}. +update_identity_provider(UserPoolId, ProviderName, ProviderDetails) -> + update_identity_provider(UserPoolId, ProviderName, ProviderDetails, #{}). + +-spec update_identity_provider(binary(), binary(), map(), map()) -> + {ok, map()} | {error, any()}. +update_identity_provider(UserPoolId, ProviderName, + ProviderDetails, AttributeMapping) -> + update_identity_provider(UserPoolId, ProviderName, + ProviderDetails, AttributeMapping, []). + +-spec update_identity_provider(binary(), binary(), map(), map(), list()) -> + {ok, map()} | {error, any()}. +update_identity_provider(UserPoolId, ProviderName, + ProviderDetails, AttributeMapping, IdpIdentifiers) -> + Config = erlcloud_aws:default_config(), + update_identity_provider(UserPoolId, ProviderName, ProviderDetails, + AttributeMapping, IdpIdentifiers, Config). + +-spec update_identity_provider(binary(), binary(), map(), map(), + list(), aws_config()) -> + {ok, map()} | {error, any()}. +update_identity_provider(UserPoolId, ProviderName, ProviderDetails, + AttributeMapping, IdpIdentifiers, Config) -> + Mandatory = #{ + <<"UserPoolId">> => UserPoolId, + <<"ProviderName">> => ProviderName + }, + Optional = #{ + <<"ProviderDetails">> => ProviderDetails, + <<"AttributeMapping">> => AttributeMapping, + <<"IdpIdentifiers">> => IdpIdentifiers + }, + request(Config, "UpdateIdentityProvider", make_request_body(Mandatory, Optional)). + +%%------------------------------------------------------------------------------ +%% Internal Functions +%%------------------------------------------------------------------------------ +request(Config, Request) -> + Result = erlcloud_retry:request(Config, Request, fun handle_result/1), + case erlcloud_aws:request_to_return(Result) of + {ok, {_, <<>>}} -> {ok, #{}}; + {ok, {_, RespBody}} -> {ok, jsx:decode(RespBody, [return_maps])}; + {error, _} = Error -> Error + end. + +request(Config0, OperationName, Request) -> + case erlcloud_aws:update_config(Config0) of + {ok, Config} -> + Body = jsx:encode(Request), + Operation = "AWSCognitoIdentityProviderService." ++ OperationName, + Headers = get_headers(Config, Operation, Body), + AwsRequest = #aws_request{service = 'cognito-idp', + uri = get_url(Config), + method = post, + request_headers = Headers, + request_body = Body}, + request(Config, AwsRequest); + {error, Reason} -> + {error, Reason} + end. + +request_no_resp(Config, OperationName, Request) -> + case request(Config, OperationName, Request) of + {ok, _} -> ok; + Error -> Error + end. + +make_request_body(Mandatory, Optional) -> + maps:merge(Mandatory, erlcloud_util:filter_empty_map(Optional)). + +get_headers(#aws_config{cognito_user_pools_host = Host} = Config, Operation, Body) -> + Headers = [{"host", Host}, + {"x-amz-target", Operation}, + {"version", ?API_VERSION}, + {"content-type", "application/x-amz-json-1.1"}], + Region = erlcloud_aws:aws_region_from_host(Host), + erlcloud_aws:sign_v4_headers(Config, Headers, Body, Region, "cognito-idp"). + +handle_result(#aws_request{response_type = ok} = Request) -> + Request; +handle_result(#aws_request{response_type = error, + error_type = aws, + response_status = Status} = Request) + when Status >= 500 -> + Request#aws_request{should_retry = true}; +handle_result(#aws_request{response_type = error, + error_type = aws} = Request) -> + Request#aws_request{should_retry = false}. + +get_url(#aws_config{cognito_user_pools_scheme = Scheme, + cognito_user_pools_host = Host}) -> + Scheme ++ Host. + +list_all(Fun, Args, Config, Key, TokenAlias) -> + list_all(Fun, Args, Config, Key, TokenAlias, undefined, []). + +list_all(Fun, Args, Config, Key, TokenAlias, NextToken, Acc) -> + UpdArgs = Args ++ [?MAX_RESULTS, NextToken, Config], + case erlang:apply(Fun, UpdArgs) of + {ok, Map} -> + UpdAcc = Acc ++ maps:get(Key, Map), + NewToken = maps:get(TokenAlias, Map, undefined), + case NewToken of + undefined -> + {ok, #{Key => UpdAcc}}; + _ -> + list_all(Fun, Args, Config, Key, TokenAlias, NewToken, UpdAcc) + end; + Error -> + Error + end. diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index 7d9a45848..201803b96 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -17,10 +17,13 @@ encode_object_list/2, next_token/2, filter_undef/1, + filter_empty_map/1, + filter_empty_list/1, uri_parse/1, http_uri_decode/1, http_uri_encode/1, - proplists_to_map/1, proplists_to_map/2 + proplists_to_map/1, + proplists_to_map/2 ]). -define(MAX_ITEMS, 1000). @@ -195,9 +198,20 @@ next_token(Path, XML) -> ok end. --spec filter_undef(proplists:proplist()) -> proplists:proplist(). -filter_undef(List) -> - lists:filter(fun({_Name, Value}) -> Value =/= undefined end, List). +-spec filter_undef(proplists:proplist() | maps:map()) -> + proplists:proplist() | maps:map(). +filter_undef(List) when is_list(List) -> + lists:filter(fun({_Name, Value}) -> Value =/= undefined end, List); +filter_undef(Map) when is_map(Map) -> + maps:filter(fun(_Key, Value) -> Value =/= undefined end, Map). + +-spec filter_empty_map(maps:map()) -> maps:map(). +filter_empty_map(Map) when is_map(Map) -> + maps:filter(fun(_Key, Value) -> Value =/= #{} end, Map). + +-spec filter_empty_list(maps:map()) -> maps:map(). +filter_empty_list(Map) when is_map(Map) -> + maps:filter(fun(_Key, Value) -> Value =/= [] end, Map). -ifdef(OTP_RELEASE). -if(?OTP_RELEASE >= 23). diff --git a/test/erlcloud_cognito_user_pools_tests.erl b/test/erlcloud_cognito_user_pools_tests.erl new file mode 100644 index 000000000..e2f7aaec5 --- /dev/null +++ b/test/erlcloud_cognito_user_pools_tests.erl @@ -0,0 +1,1059 @@ +-module(erlcloud_cognito_user_pools_tests). + +-include_lib("eunit/include/eunit.hrl"). +-include("erlcloud_aws.hrl"). + +%% API +-export([]). + +%% define response macros +-define(EHTTPC, erlcloud_httpc). + +-define(USER_POOL_ID, <<"testpool">>). +-define(CLIENT_ID, <<"id-1">>). +-define(USERNAME, <<"test.user">>). + +-define(LIST_USERS_RESP, + #{<<"Users">> => + [ + #{<<"Attributes">> => + [ + #{ + <<"Name">> => <<"sub">>, + <<"Value">> => <<"ec64aac9-78da-4c0f-870c-6389a1ef38bf">>}, + #{ + <<"Name">> => <<"email_verified">>, + <<"Value">> => <<"true">>}, + #{ + <<"Name">> => <<"custom:test1">>, + <<"Value">> => <<"test">>}, + #{ + <<"Name">> => <<"email">>, + <<"Value">> => <<"test@fake.email">>}, + #{ + <<"Name">> => <<"custom:test2">>, + <<"Value">> => <<"test">>} + ], + <<"Enabled">> => true, + <<"UserCreateDate">> => 1632222392.623, + <<"UserLastModifiedDate">> => 1633509297.315, + <<"UserStatus">> => <<"CONFIRMED">>, + <<"Username">> => <<"test.user1">>}, + #{<<"Attributes">> => + [ + #{ + <<"Name">> => <<"sub">>, + <<"Value">> => <<"b23652f5-2340-4137-9a5e-4b40641420eb">>}, + #{ + <<"Name">> => <<"email_verified">>, + <<"Value">> => <<"true">>}, + #{ + <<"Name">> => <<"email">>, + <<"Value">> => <<"test2@fake.email">>} + ], + <<"Enabled">> => true, + <<"UserCreateDate">> => 1631791632.572, + <<"UserLastModifiedDate">> => 1631791778.698, + <<"UserStatus">> => <<"CONFIRMED">>, + <<"Username">> => <<"test.user2">>} + ]}). + +-define(LIST_ALL_USERS, + #{<<"Users">> => + [ + #{<<"Attributes">> => + [ + #{ + <<"Name">> => <<"sub">>, + <<"Value">> => <<"ec64aac9-78da-4c0f-870c-6389a1ef38bf">>}, + #{ + <<"Name">> => <<"email_verified">>, + <<"Value">> => <<"true">>}, + #{ + <<"Name">> => <<"custom:test1">>, + <<"Value">> => <<"test">>}, + #{ + <<"Name">> => <<"email">>, + <<"Value">> => <<"test@fake.email">>}, + #{ + <<"Name">> => <<"custom:test2">>, + <<"Value">> => <<"test">>} + ], + <<"Enabled">> => true, + <<"UserCreateDate">> => 1632222392.623, + <<"UserLastModifiedDate">> => 1633509297.315, + <<"UserStatus">> => <<"CONFIRMED">>, + <<"Username">> => <<"test.user1">>}, + #{<<"Attributes">> => + [ + #{ + <<"Name">> => <<"sub">>, + <<"Value">> => <<"b23652f5-2340-4137-9a5e-4b40641420eb">>}, + #{ + <<"Name">> => <<"email_verified">>, + <<"Value">> => <<"true">>}, + #{ + <<"Name">> => <<"email">>, + <<"Value">> => <<"test2@fake.email">>} + ], + <<"Enabled">> => true, + <<"UserCreateDate">> => 1631791632.572, + <<"UserLastModifiedDate">> => 1631791778.698, + <<"UserStatus">> => <<"CONFIRMED">>, + <<"Username">> => <<"test.user2">>}, + #{<<"Attributes">> => + [ + #{ + <<"Name">> => <<"sub">>, + <<"Value">> => <<"b23652f5-2340-4137-9a5e-4b40641420eb">>}, + #{ + <<"Name">> => <<"email_verified">>, + <<"Value">> => <<"true">>}, + #{ + <<"Name">> => <<"email">>, + <<"Value">> => <<"test3@fake.email">>} + ], + <<"Enabled">> => true, + <<"UserCreateDate">> => 1631791632.572, + <<"UserLastModifiedDate">> => 1631791778.698, + <<"UserStatus">> => <<"CONFIRMED">>, + <<"Username">> => <<"test.user3">>} + ]}). + +-define(ADMIN_LIST_GROUPS_FOR_USERS, + #{<<"Groups">> => + [ + #{ + <<"CreationDate">> => 1632236861.937, + <<"GroupName">> => <<"test">>, + <<"LastModifiedDate">> => 1632236861.937, + <<"UserPoolId">> => ?USER_POOL_ID}, + #{ + <<"CreationDate">> => 1632231111.404, + <<"Description">> => <<"test desc">>, + <<"GroupName">> => <<"test2">>, + <<"LastModifiedDate">> => 1632231111.404, + <<"UserPoolId">> => ?USER_POOL_ID} + ]}). + +-define(ADMIN_GET_USER, + #{<<"Enabled">> => true, + <<"UserAttributes">> => + [ + #{ + <<"Name">> => <<"sub">>, + <<"Value">> => <<"ec64aac9-78da-4c0f-870c-6389a1ef38bf">>}, + #{ + <<"Name">> => <<"email_verified">>, + <<"Value">> => <<"true">>}, + #{ + <<"Name">> => <<"custom:test1">>, + <<"Value">> => <<"test">>}, + #{ + <<"Name">> => <<"email">>, + <<"Value">> => <<"test@fake.email">>}, + #{ + <<"Name">> => <<"custom:test2">>, + <<"Value">> => <<"test">>} + ], + <<"UserCreateDate">> => 1632222392.623, + <<"UserLastModifiedDate">> => 1633509297.315, + <<"UserStatus">> => <<"CONFIRMED">>, + <<"Username">> => ?USERNAME} +). + +-define(ADMIN_CREATE_USER, + #{<<"User">> => + #{<<"Attributes">> => + [ + #{ + <<"Name">> => <<"sub">>, + <<"Value">> => <<"1e172333-dd47-45ee-9da2-7b53d7ff3932">>} + ], + <<"Enabled">> => true, + <<"UserCreateDate">> => 1633946606.409, + <<"UserLastModifiedDate">> => 1633946606.409, + <<"UserStatus">> => <<"FORCE_CHANGE_PASSWORD">>, + <<"Username">> => ?USERNAME}} +). + +-define(CREATE_GROUP, + #{ + <<"Group">> => + #{ + <<"CreationDate">> => 1633949776.261, + <<"GroupName">> => <<"test">>, + <<"LastModifiedDate">> => 1633949776.261, + <<"UserPoolId">> => ?USER_POOL_ID} + }). + +-define(LIST_USER_POOLS, + #{ + <<"NextToken">> => <<"nextpage">>, + <<"UserPools">> => + [ + #{ + <<"CreationDate">> => 1634210679.535, + <<"Id">> => <<"eu-west-testpool1">>, + <<"LambdaConfig">> => #{}, + <<"LastModifiedDate">> => 1634210679.535, + <<"Name">> => ?USER_POOL_ID + }, + #{ + <<"CreationDate">> => 1632131813.194, + <<"Id">> => <<"eu-west-testpool2">>, + <<"LambdaConfig">> => #{}, + <<"LastModifiedDate">> => 1634209858.344, + <<"Name">> => <<"TestPool2">> + } + ] + }). + +-define(LIST_ALL_USER_POOLS, + #{ + <<"UserPools">> => + [ + #{ + <<"CreationDate">> => 1634210679.535, + <<"Id">> => <<"eu-west-testpool1">>, + <<"LambdaConfig">> => #{}, + <<"LastModifiedDate">> => 1634210679.535, + <<"Name">> => ?USER_POOL_ID + }, + #{ + <<"CreationDate">> => 1632131813.194, + <<"Id">> => <<"eu-west-testpool2">>, + <<"LambdaConfig">> => #{}, + <<"LastModifiedDate">> => 1634209858.344, + <<"Name">> => <<"TestPool2">> + }, + #{ + <<"CreationDate">> => 1634210679.535, + <<"Id">> => <<"eu-west-testpool3">>, + <<"LambdaConfig">> => #{}, + <<"LastModifiedDate">> => 1634210679.535, + <<"Name">> => <<"TestPool3">> + } + ] + }). + +-define(DESCRIBE_POOL, #{ + <<"UserPool">> => + #{<<"AccountRecoverySetting">> => + #{<<"RecoveryMechanisms">> => + [#{<<"Name">> => <<"verified_email">>,<<"Priority">> => 1}, + #{<<"Name">> => <<"verified_phone_number">>, + <<"Priority">> => 2}]}, + <<"AdminCreateUserConfig">> => + #{<<"AllowAdminCreateUserOnly">> => true, + <<"InviteMessageTemplate">> => + #{<<"EmailMessage">> => + <<"Your username is {username} and temporary password is {####}. ">>, + <<"EmailSubject">> => <<"Your temporary password">>, + <<"SMSMessage">> => + <<"Your username is {username} and temporary password is {####}. ">>}, + <<"UnusedAccountValidityDays">> => 7}, + <<"Arn">> => + <<"arn:aws:cognito-idp:eu-west-1:87806250403242:userpool/testpool">>, + <<"AutoVerifiedAttributes">> => [<<"email">>], + <<"CreationDate">> => 1636620065.355, + <<"Domain">> => <<"nxdomain3323">>, + <<"EmailConfiguration">> => + #{<<"EmailSendingAccount">> => <<"COGNITO_DEFAULT">>}, + <<"EmailVerificationMessage">> => + <<"Your verification code is {####}. ">>, + <<"EmailVerificationSubject">> => + <<"Your verification code">>, + <<"EstimatedNumberOfUsers">> => 3, + <<"Id">> => <<"eu-west-1_XEBAHPImu">>, + <<"LambdaConfig">> => #{}, + <<"LastModifiedDate">> => 1636620065.355, + <<"MfaConfiguration">> => <<"OFF">>, + <<"Name">> => <<"test-pool">>, + <<"Policies">> => + #{<<"PasswordPolicy">> => + #{<<"MinimumLength">> => 8,<<"RequireLowercase">> => true, + <<"RequireNumbers">> => true,<<"RequireSymbols">> => false, + <<"RequireUppercase">> => false, + <<"TemporaryPasswordValidityDays">> => 7}}, + <<"SchemaAttributes">> => [ + #{<<"AttributeDataType">> => <<"String">>, + <<"DeveloperOnlyAttribute">> => false,<<"Mutable">> => true, + <<"Name">> => <<"custom:partner">>,<<"Required">> => false, + <<"StringAttributeConstraints">> => + #{<<"MaxLength">> => <<"256">>,<<"MinLength">> => <<"1">>}}, + #{<<"AttributeDataType">> => <<"String">>, + <<"DeveloperOnlyAttribute">> => false,<<"Mutable">> => true, + <<"Name">> => <<"custom:organisation">>, + <<"Required">> => false, + <<"StringAttributeConstraints">> => + #{<<"MaxLength">> => <<"256">>,<<"MinLength">> => <<"1">>}}, + #{<<"AttributeDataType">> => <<"String">>, + <<"DeveloperOnlyAttribute">> => false,<<"Mutable">> => true, + <<"Name">> => <<"custom:role">>,<<"Required">> => false, + <<"StringAttributeConstraints">> => + #{<<"MaxLength">> => <<"256">>,<<"MinLength">> => <<"1">>}}, + #{<<"AttributeDataType">> => <<"String">>, + <<"DeveloperOnlyAttribute">> => false,<<"Mutable">> => true, + <<"Name">> => <<"custom:viewMode">>,<<"Required">> => false, + <<"StringAttributeConstraints">> => + #{<<"MaxLength">> => <<"256">>,<<"MinLength">> => <<"1">>}} + ], + <<"SmsAuthenticationMessage">> => <<"Your authentication code is {####}. ">>, + <<"SmsVerificationMessage">> => <<"Your verification code is {####}. ">>, + <<"UserPoolTags">> => #{}, + <<"UsernameAttributes">> => [<<"email">>], + <<"UsernameConfiguration">> => #{<<"CaseSensitive">> => false}, + <<"VerificationMessageTemplate">> => #{ + <<"DefaultEmailOption">> => <<"CONFIRM_WITH_CODE">>, + <<"EmailMessage">> => + <<"Your verification code is {####}. ">>, + <<"EmailSubject">> => <<"Your verification code">>, + <<"SmsMessage">> => + <<"Your verification code is {####}. ">> + }} +}). + +-define(MFA_CONFIG, + #{ + <<"MfaConfiguration">> => <<"ON">>, + <<"SoftwareTokenMfaConfiguration">> => #{<<"Enabled">> => true} + }). + +-define(LIST_IDENTITY_PROVIDERS, + #{<<"Providers">> => [#{<<"CreationDate">> => 1636620575.033, + <<"LastModifiedDate">> => 1636620730.0, + <<"ProviderName">> => <<"test">>, + <<"ProviderType">> => <<"SAML">>}], + <<"NextToken">> => <<"nextpage">>}). + +-define(LIST_ALL_IDENTITY_PROVIDERS, + #{<<"Providers">> => + [ + #{ + <<"CreationDate">> => 1636620575.033, + <<"LastModifiedDate">> => 1636620730.0, + <<"ProviderName">> => <<"test">>, + <<"ProviderType">> => <<"SAML">> + }, + #{ + <<"CreationDate">> => 1636620577.033, + <<"LastModifiedDate">> => 1636620730.0, + <<"ProviderName">> => <<"NewOauth">>, + <<"ProviderType">> => <<"SAML">> + } + ]} + ). + +-define(DESCRIBE_PROVIDER, + #{ + <<"IdentityProvider">> => + #{ + <<"AttributeMapping">> => + #{ + <<"email">> => + <<"http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress">>}, + <<"CreationDate">> => 1636620575.033, + <<"IdpIdentifiers">> => [], + <<"LastModifiedDate">> => 1636620730.0, + <<"ProviderDetails">> => + #{<<"IDPSignout">> => <<"false">>, + <<"MetadataFile">> => <<"meta xml">>, + <<"SLORedirectBindingURI">> => + <<"https://login.microsoftonline.com/test/saml2">>, + <<"SSORedirectBindingURI">> => + <<"https://login.microsoftonline.com/test/saml2">>}, + <<"ProviderName">> => <<"test">>, + <<"ProviderType">> => <<"SAML">>, + <<"UserPoolId">> => <<"us-east-2_test">>}}). + +-define(DESCRIBE_USER_POOL_CLIENT, + #{<<"UserPoolClient">> => + #{<<"AccessTokenValidity">> => 60, + <<"AllowedOAuthFlows">> => [<<"code">>,<<"implicit">>], + <<"AllowedOAuthFlowsUserPoolClient">> => true, + <<"AllowedOAuthScopes">> => + [<<"aws.cognito.signin.user.admin">>,<<"email">>, + <<"openid">>,<<"phone">>], + <<"CallbackURLs">> => [<<"https://test.dev:30102/login">>], + <<"ClientId">> => ?CLIENT_ID, + <<"ClientName">> => <<"test">>, + <<"CreationDate">> => 1635782978.411, + <<"EnableTokenRevocation">> => true, + <<"ExplicitAuthFlows">> => + [<<"ALLOW_ADMIN_USER_PASSWORD_AUTH">>, + <<"ALLOW_CUSTOM_AUTH">>,<<"ALLOW_REFRESH_TOKEN_AUTH">>, + <<"ALLOW_USER_SRP_AUTH">>], + <<"IdTokenValidity">> => 60, + <<"LastModifiedDate">> => 1636620585.842, + <<"LogoutURLs">> => [<<"https://test.dev:30102/logout">>], + <<"PreventUserExistenceErrors">> => <<"ENABLED">>, + <<"ReadAttributes">> => + [<<"address">>,<<"birthdate">>,<<"email">>, + <<"email_verified">>,<<"family_name">>,<<"gender">>, + <<"given_name">>,<<"locale">>,<<"middle_name">>,<<"name">>, + <<"nickname">>,<<"phone_number">>, + <<"phone_number_verified">>,<<"picture">>, + <<"preferred_username">>,<<"profile">>,<<"updated_at">>, + <<"website">>,<<"zoneinfo">>], + <<"RefreshTokenValidity">> => 30, + <<"SupportedIdentityProviders">> => + [<<"ActualExperience">>,<<"COGNITO">>], + <<"TokenValidityUnits">> => + #{<<"AccessToken">> => <<"minutes">>, + <<"IdToken">> => <<"minutes">>, + <<"RefreshToken">> => <<"days">>}, + <<"UserPoolId">> => <<"us-east-2_test">>, + <<"WriteAttributes">> => + [<<"address">>,<<"birthdate">>,<<"email">>, + <<"family_name">>,<<"gender">>,<<"given_name">>, + <<"locale">>,<<"middle_name">>,<<"name">>,<<"nickname">>, + <<"phone_number">>,<<"picture">>,<<"preferred_username">>, + <<"profile">>,<<"updated_at">>,<<"website">>, + <<"zoneinfo">>]}} +). + +-define(LIST_USER_POOL_CLIENTS, #{ + <<"UserPoolClients">> => [ + #{<<"ClientId">> => ?CLIENT_ID, + <<"ClientName">> => <<"name-1">>, + <<"UserPoolId">> => ?USER_POOL_ID} + ] +}). + +-define(LIST_ALL_USER_POOL_CLIENTS, #{ + <<"UserPoolClients">> => [ + #{<<"ClientId">> => ?CLIENT_ID, + <<"ClientName">> => <<"name-1">>, + <<"UserPoolId">> => ?USER_POOL_ID}, + #{<<"ClientId">> => ?CLIENT_ID, + <<"ClientName">> => <<"name-1">>, + <<"UserPoolId">> => ?USER_POOL_ID} + ] +}). + +-define(ADMIN_LIST_DEVICE, + #{ + <<"Devices">> => [ + #{ + <<"DeviceAttributes">> => [ + #{ + <<"Name">> => <<"test">>, + <<"Value">> => <<"testvalue">> + } + ], + <<"DeviceCreateDate">> => 1635782978.411, + <<"DeviceKey">> => <<"testKey">>, + <<"DeviceLastAuthenticatedDate">> => 1635782978.411, + <<"DeviceLastModifiedDate">> => 1635782978.411 + } + ] + } + ). + +-define(LIST_ALL_DEVICES, + #{ + <<"Devices">> => [ + #{ + <<"DeviceAttributes">> => [ + #{ + <<"Name">> => <<"test">>, + <<"Value">> => <<"testvalue">> + } + ], + <<"DeviceCreateDate">> => 1635782978.411, + <<"DeviceKey">> => <<"testKey">>, + <<"DeviceLastAuthenticatedDate">> => 1635782978.411, + <<"DeviceLastModifiedDate">> => 1635782978.411 + }, + #{ + <<"DeviceAttributes">> => [ + #{ + <<"Name">> => <<"test">>, + <<"Value">> => <<"testvalue">> + } + ], + <<"DeviceCreateDate">> => 1635782978.411, + <<"DeviceKey">> => <<"testKey2">>, + <<"DeviceLastAuthenticatedDate">> => 1635782978.411, + <<"DeviceLastModifiedDate">> => 1635782978.411 + } + ] + } +). + +-define(ADMIN_INITIATE_AUTH, #{ + <<"ChallengeName">> => <<"SOFTWARE_TOKEN_MFA">>, + <<"ChallengeParameters">> => #{<<"USER_ID_FOR_SRP">> => <<"id">>}, + <<"Session">> => <<"session-token">> +}). + +-define(RESPOND_TO_AUTH_CHALLENGE, #{ + <<"AuthenticationResult">> => #{ + <<"AccessToken">> => <<"access-token">>, + <<"ExpiresIn">> => 1800, + <<"IdToken">> => <<"id-token">>, + <<"RefreshToken">> => <<"refresh-token">>, + <<"TokenType">> => <<"Bearer">>, + <<"NewDeviceMetadata">> => #{ + <<"DeviceGroupKey">> => <<"-rowRNkJw">>, + <<"DeviceKey">> => <<"eu-west-1_b6657eba-2850-4a47-9357-fae69fd86b94">> + } + }, + <<"ChallengeParameters">> => #{} +}). + +-define(CREATE_UPDATE_IDP, #{ + <<"IdentityProvider">> => #{ + <<"AttributeMapping">> => #{ + <<"testAttr">> => <<"SAMLAttr">> + }, + <<"CreationDate">> => 1635782978.411, + <<"IdpIdentifiers">> => [ "Test IDP" ], + <<"LastModifiedDate">> => 1635782978.411, + <<"ProviderDetails">> => #{ + <<"MetadataURL">> => <<"http://test-idp.com/1234">> + }, + <<"ProviderName">> => <<"TestIdp">>, + <<"ProviderType">> => <<"SAML">>, + <<"UserPoolId">> => <<"test">> + } + }). + +config() -> + #aws_config{access_key_id = "id", + secret_access_key = "key", + retry = fun erlcloud_retry:default_retry/1, + retry_num = 3}. + +setup() -> + erlcloud_cognito_user_pools:configure("id", "key"), + MockedModules = [?EHTTPC, erlcloud_aws], + meck:new(MockedModules, [passthrough]), + meck:expect(erlcloud_aws, update_config, 1, {ok, config()}), + meck:expect(?EHTTPC, request, 6, fun do_erlcloud_httpc_request/6), + MockedModules. + +erlcloud_cognito_user_pools_test_() -> + { + foreach, + fun setup/0, + fun meck:unload/1, + [ + fun test_list_users/0, + fun test_list_all_users/0, + fun test_admin_list_groups_for_user/0, + fun test_admin_get_user/0, + fun test_admin_create_user/0, + fun test_admin_delete_user/0, + fun test_admin_add_user_to_group/0, + fun test_admin_remove_user_from_group/0, + fun test_create_group/0, + fun test_delete_group/0, + fun test_admin_reset_user_password/0, + fun test_admin_update_user_attributes/0, + fun test_change_password/0, + fun test_list_user_pools/0, + fun test_list_all_user_pools/0, + fun test_admin_set_user_password/0, + fun test_describe_user_pool/0, + fun test_get_user_pool_mfa_config/0, + fun test_list_identity_providers/0, + fun test_list_all_identity_provider/0, + fun test_describe_identity_provider/0, + fun test_describe_user_pool_client/0, + fun test_list_user_pool_clients/0, + fun test_list_all_user_pool_clients/0, + fun test_admin_list_devices/0, + fun test_list_all_devices/0, + fun test_admin_forget_device/0, + fun test_admin_confirm_signup/0, + fun test_admin_initiate_auth/0, + fun test_respond_to_auth_challenge/0, + fun test_create_identity_provider/0, + fun test_delete_identity_provider/0, + fun test_update_identity_provider/0, + fun test_error_no_retry/0, + fun test_error_retry/0 + ] + }. + +test_list_users() -> + Request = #{<<"UserPoolId">> => ?USER_POOL_ID}, + Expected = {ok, ?LIST_USERS_RESP}, + TestFun = fun() -> erlcloud_cognito_user_pools:list_users(?USER_POOL_ID) end, + do_test(Request, Expected, TestFun). + +test_list_all_users() -> + Mocked1 = ?LIST_USERS_RESP#{<<"PaginationToken">> => "1"}, + Mocked2 = #{<<"Users">> => [ + #{ + <<"Attributes">> => + [ + #{<<"Name">> => <<"sub">>, + <<"Value">> => <<"b23652f5-2340-4137-9a5e-4b40641420eb">>}, + #{<<"Name">> => <<"email_verified">>, + <<"Value">> => <<"true">>}, + #{<<"Name">> => <<"email">>, + <<"Value">> => <<"test3@fake.email">>} + ], + <<"Enabled">> => true, + <<"UserCreateDate">> => 1631791632.572, + <<"UserLastModifiedDate">> => 1631791778.698, + <<"UserStatus">> => <<"CONFIRMED">>, + <<"Username">> => <<"test.user3">>} + ]}, + meck:sequence(?EHTTPC, request, 6, [{ok, {{200, "OK"}, [], jsx:encode(Mocked1)}}, + {ok, {{200, "OK"}, [], jsx:encode(Mocked2)}}]), + Expected = {ok, ?LIST_ALL_USERS}, + Response = erlcloud_cognito_user_pools:list_all_users(?USER_POOL_ID), + ?assertEqual(Expected, Response). + +test_admin_list_groups_for_user() -> + Request = #{ + <<"Username">> => ?USERNAME, + <<"UserPoolId">> => ?USER_POOL_ID + }, + Expected = {ok, ?ADMIN_LIST_GROUPS_FOR_USERS}, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_list_groups_for_user(?USERNAME, ?USER_POOL_ID, config()) + end, + do_test(Request, Expected, TestFun). + +test_admin_get_user() -> + Request = #{ + <<"Username">> => ?USERNAME, + <<"UserPoolId">> => ?USER_POOL_ID + }, + Expected = {ok, ?ADMIN_GET_USER}, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_get_user(?USERNAME, ?USER_POOL_ID, config()) + end, + do_test(Request, Expected, TestFun). + +test_admin_create_user() -> + UserAttributes = #{ + <<"UserAttributes">> => [ + #{<<"Name">> => <<"custom:test">>, + <<"Value">> => <<"test">>}, + #{<<"Name">> => <<"custom:test2">>, + <<"Value">> => <<"test2">>} + ] + }, + Request = UserAttributes#{<<"Username">> => ?USERNAME, + <<"UserPoolId">> => ?USER_POOL_ID}, + Expected = {ok, ?ADMIN_CREATE_USER}, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_create_user(?USERNAME, ?USER_POOL_ID, UserAttributes) + end, + do_test(Request, Expected, TestFun). + +test_admin_delete_user() -> + Request = #{ + <<"Username">> => ?USERNAME, + <<"UserPoolId">> => ?USER_POOL_ID + }, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_delete_user(?USERNAME, ?USER_POOL_ID, config()) + end, + do_test(Request, ok, TestFun). + +test_admin_add_user_to_group() -> + GroupName = <<"test">>, + Request = #{ + <<"Username">> => ?USERNAME, + <<"UserPoolId">> => ?USER_POOL_ID, + <<"GroupName">> => GroupName}, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_add_user_to_group(GroupName, ?USERNAME, ?USER_POOL_ID, config()) + end, + do_test(Request, ok, TestFun). + +test_admin_remove_user_from_group() -> + GroupName = <<"test">>, + Request = #{ + <<"Username">> => ?USERNAME, + <<"UserPoolId">> => ?USER_POOL_ID, + <<"GroupName">> => GroupName}, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_remove_user_from_group(GroupName, ?USERNAME, ?USER_POOL_ID, config()) + end, + do_test(Request, ok, TestFun). + +test_create_group() -> + GroupName = <<"test">>, + Request = #{ + <<"UserPoolId">> => ?USER_POOL_ID, + <<"GroupName">> => GroupName}, + Expected = {ok, ?CREATE_GROUP}, + TestFun = fun() -> + erlcloud_cognito_user_pools:create_group(GroupName, ?USER_POOL_ID, config()) + end, + do_test(Request, Expected, TestFun). + +test_delete_group() -> + GroupName = <<"test">>, + Request = #{ + <<"UserPoolId">> => ?USER_POOL_ID, + <<"GroupName">> => GroupName}, + TestFun = fun() -> + erlcloud_cognito_user_pools:delete_group(GroupName, ?USER_POOL_ID, config()) + end, + do_test(Request, ok, TestFun). + +test_admin_reset_user_password() -> + Request = #{ + <<"Username">> => ?USERNAME, + <<"UserPoolId">> => ?USER_POOL_ID}, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_reset_user_password(?USERNAME, ?USER_POOL_ID) + end, + do_test(Request, ok, TestFun). + +test_admin_update_user_attributes() -> + Attributes = [ + #{ + <<"Name">> => <<"custom:partner">>, + <<"Value">> => <<"new value">> + } + ], + Request = #{ + <<"UserAttributes">> => Attributes, + <<"UserPoolId">> => ?USER_POOL_ID, + <<"Username">> => ?USERNAME}, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_update_user_attributes(?USERNAME, ?USER_POOL_ID, Attributes) + end, + do_test(Request, ok, TestFun). + +test_change_password() -> + AccessToken = <<"test token">>, + OldPass = <<"old p4ss!">>, + NewPass = <<"new p4ss!">>, + Request = #{ + <<"AccessToken">> => AccessToken, + <<"PreviousPassword">> => OldPass, + <<"ProposedPassword">> => NewPass + }, + TestFun = fun() -> + erlcloud_cognito_user_pools:change_password(OldPass, NewPass, AccessToken) + end, + do_test(Request, ok, TestFun). + +test_list_user_pools() -> + Expected = {ok, ?LIST_USER_POOLS}, + MaxResults = 60, + Request = #{<<"MaxResults">> => MaxResults}, + TestFun = fun() -> erlcloud_cognito_user_pools:list_user_pools(MaxResults) end, + do_test(Request, Expected, TestFun). + +test_error_no_retry() -> + erlcloud_cognito_user_pools:configure("test-access-key", "test-secret-key"), + ErrCode = 400, + Status = "Bad Request", + ErrMsg = <<"Message">>, + Operation = "ListUsers", + Config = config(), + Request = #{<<"UserPoolId">> => ?USER_POOL_ID}, + meck:expect(?EHTTPC, request, 6, {ok, {{ErrCode, Status}, [], ErrMsg}}), + ?assertEqual( + {error, {http_error, ErrCode, Status, ErrMsg, []}}, + erlcloud_cognito_user_pools:request(Config, Operation, Request) + ). + +test_list_all_user_pools() -> + Mocked1 = ?LIST_USER_POOLS, + Mocked2 = #{ + <<"UserPools">> => + [ + #{ + <<"CreationDate">> => 1634210679.535, + <<"Id">> => <<"eu-west-testpool3">>, + <<"LambdaConfig">> => #{}, + <<"LastModifiedDate">> => 1634210679.535, + <<"Name">> => <<"TestPool3">> + } + ]}, + meck:sequence(?EHTTPC, request, 6, [{ok, {{200, "OK"}, [], jsx:encode(Mocked1)}}, + {ok, {{200, "OK"}, [], jsx:encode(Mocked2)}}]), + Expected = {ok, ?LIST_ALL_USER_POOLS}, + Response = erlcloud_cognito_user_pools:list_all_user_pools(), + ?assertEqual(Expected, Response). + +test_admin_set_user_password() -> + Pass = <<"test pass">>, + Request = #{ + <<"Username">> => ?USERNAME, + <<"UserPoolId">> => ?USER_POOL_ID, + <<"Password">> => Pass, + <<"Permanent">> => false}, + TestFun = fun() -> + erlcloud_cognito_user_pools:admin_set_user_password(?USERNAME, ?USER_POOL_ID, Pass) + end, + do_test(Request, ok, TestFun). + +test_describe_user_pool() -> + Request = #{<<"UserPoolId">> => ?USER_POOL_ID}, + Expected = {ok, ?DESCRIBE_POOL}, + TestFun = fun() -> erlcloud_cognito_user_pools:describe_user_pool(?USER_POOL_ID) end, + do_test(Request, Expected, TestFun). + +test_get_user_pool_mfa_config() -> + Request = #{<<"UserPoolId">> => ?USER_POOL_ID}, + Expected = {ok, ?MFA_CONFIG}, + TestFun = fun() -> erlcloud_cognito_user_pools:get_user_pool_mfa_config(?USER_POOL_ID) end, + do_test(Request, Expected, TestFun). + +test_list_identity_providers() -> + Request = #{ + <<"UserPoolId">> => ?USER_POOL_ID, + <<"MaxResults">> => 60 + }, + Expected = {ok, ?LIST_IDENTITY_PROVIDERS}, + TestFun = fun() -> erlcloud_cognito_user_pools:list_identity_providers(?USER_POOL_ID) end, + do_test(Request, Expected, TestFun). + +test_list_all_identity_provider() -> + Mocked1 = ?LIST_IDENTITY_PROVIDERS, + Mocked2 = #{ + <<"Providers">> => [ + #{<<"CreationDate">> => 1636620577.033, + <<"LastModifiedDate">> => 1636620730.0, + <<"ProviderName">> => <<"NewOauth">>, + <<"ProviderType">> => <<"SAML">>} + ] + }, + meck:sequence(?EHTTPC, request, 6, [{ok, {{200, "OK"}, [], jsx:encode(Mocked1)}}, + {ok, {{200, "OK"}, [], jsx:encode(Mocked2)}}]), + Expected = {ok, ?LIST_ALL_IDENTITY_PROVIDERS}, + Response = erlcloud_cognito_user_pools:list_all_identity_providers(?USER_POOL_ID), + ?assertEqual(Expected, Response). + +test_describe_identity_provider() -> + ProviderName = <<"test">>, + Request = #{ + <<"UserPoolId">> => ?USER_POOL_ID, + <<"ProviderName">> => ProviderName + }, + Expected = {ok, ?DESCRIBE_PROVIDER}, + TestFun = fun() -> + erlcloud_cognito_user_pools:describe_identity_provider(?USER_POOL_ID, ProviderName) + end, + do_test(Request, Expected, TestFun). + +test_describe_user_pool_client() -> + Request = #{ + <<"UserPoolId">> => ?USER_POOL_ID, + <<"ClientId">> => ?CLIENT_ID + }, + Expected = {ok, ?DESCRIBE_USER_POOL_CLIENT}, + TestFun = fun() -> erlcloud_cognito_user_pools:describe_user_pool_client(?USER_POOL_ID, ?CLIENT_ID) end, + do_test(Request, Expected, TestFun). + +test_list_user_pool_clients() -> + Request = #{<<"UserPoolId">> => ?USER_POOL_ID, + <<"MaxResults">> => 60}, + TestFun = fun() -> erlcloud_cognito_user_pools:list_user_pool_clients(?USER_POOL_ID) end, + Expected = {ok, ?LIST_USER_POOL_CLIENTS}, + do_test(Request, Expected, TestFun). + +test_list_all_user_pool_clients() -> + Mocked1 = ?LIST_USER_POOL_CLIENTS#{<<"NextToken">> => <<"next">>}, + Mocked2 = ?LIST_USER_POOL_CLIENTS, + meck:sequence(?EHTTPC, request, 6, [{ok, {{200, "OK"}, [], jsx:encode(Mocked1)}}, + {ok, {{200, "OK"}, [], jsx:encode(Mocked2)}}]), + Expected = {ok, ?LIST_ALL_USER_POOL_CLIENTS}, + Response = erlcloud_cognito_user_pools:list_all_user_pool_clients(?USER_POOL_ID), + ?assertEqual(Expected, Response). + +test_admin_list_devices() -> + Request = #{ + <<"UserPoolId">> => ?USER_POOL_ID, + <<"Username">> => ?USERNAME, + <<"Limit">> => 60 + }, + Expected = {ok, ?ADMIN_LIST_DEVICE}, + TestFun = fun() -> erlcloud_cognito_user_pools:admin_list_devices(?USER_POOL_ID, ?USERNAME) end, + do_test(Request, Expected, TestFun). + +test_list_all_devices() -> + Mocked1 = ?ADMIN_LIST_DEVICE#{<<"PaginationToken">> => <<"next page">>}, + Devices = hd(maps:get(<<"Devices">>, ?ADMIN_LIST_DEVICE)), + Mocked2 = ?ADMIN_LIST_DEVICE#{<<"Devices">> => [Devices#{<<"DeviceKey">> => <<"testKey2">>}]}, + meck:sequence(?EHTTPC, request, 6, [{ok, {{200, "OK"}, [], jsx:encode(Mocked1)}}, + {ok, {{200, "OK"}, [], jsx:encode(Mocked2)}}]), + Expected = {ok, ?LIST_ALL_DEVICES}, + Response = erlcloud_cognito_user_pools:admin_list_all_devices(?USER_POOL_ID, ?USERNAME), + ?assertEqual(Expected, Response). + +test_admin_forget_device() -> + DeviceKey = <<"test key">>, + Request = #{ + <<"UserPoolId">> => ?USER_POOL_ID, + <<"Username">> => ?USERNAME, + <<"DeviceKey">> => DeviceKey + }, + TestFun = fun() -> erlcloud_cognito_user_pools:admin_forget_device(?USER_POOL_ID, ?USERNAME, DeviceKey) end, + do_test(Request, ok, TestFun). + +test_admin_confirm_signup() -> + Request = #{ + <<"UserPoolId">> => ?USER_POOL_ID, + <<"Username">> => ?USERNAME, + <<"ClientMetadata">> => #{} + }, + TestFun = fun() -> erlcloud_cognito_user_pools:admin_confirm_signup(?USER_POOL_ID, ?USERNAME) end, + do_test(Request, {ok, #{}}, TestFun). + +test_admin_initiate_auth() -> + AuthFlow = <<"ADMIN_USER_PASSWORD_AUTH">>, + AuthParams = #{<<"PASSWORD">> => <<"pass">>, + <<"USERNAME">> => ?USERNAME}, + Request = #{ + <<"AuthFlow">> => AuthFlow, + <<"AuthParameters">> => AuthParams, + <<"ClientId">> => ?CLIENT_ID, + <<"UserPoolId">> => ?USER_POOL_ID + }, + TestFun = fun() -> erlcloud_cognito_user_pools:admin_initiate_auth(?USER_POOL_ID, ?CLIENT_ID, + AuthFlow, AuthParams) end, + do_test(Request, {ok, ?ADMIN_INITIATE_AUTH}, TestFun). + +test_respond_to_auth_challenge() -> + ChallengeName = <<"SOFTWARE_TOKEN_MFA">>, + ChallengeResp = #{<<"USERNAME">> => ?USERNAME, + <<"SOFTWARE_TOKEN_MFA_CODE">> => <<"123456">>}, + Session = <<"session-token">>, + Request = #{ + <<"ChallengeName">> => ChallengeName, + <<"ChallengeResponses">> => ChallengeResp, + <<"ClientId">> => ?CLIENT_ID, + <<"Session">> => Session + }, + TestFun = fun() -> erlcloud_cognito_user_pools:respond_to_auth_challenge(?CLIENT_ID, ChallengeName, + ChallengeResp, Session) end, + do_test(Request, {ok, ?RESPOND_TO_AUTH_CHALLENGE}, TestFun). + +test_create_identity_provider() -> + UserPoolId = <<"test">>, + ProviderName = <<"testIdp">>, + ProviderType = <<"SAML">>, + ProviderDetails = #{ + <<"MetadataURL">> => <<"http://test-idp.com/1234">> + }, + AttributeMapping = #{ + <<"testAttr">> => <<"SAMLAttr">> + }, + IdpIdentifiers = [ "Test IDP" ], + + Request = #{ + <<"UserPoolId">> => UserPoolId, + <<"ProviderName">> => ProviderName, + <<"ProviderType">> => ProviderType, + <<"ProviderDetails">> => ProviderDetails, + <<"AttributeMapping">> => AttributeMapping, + <<"IdpIdentifiers">> => IdpIdentifiers + }, + + TestFun = fun() -> erlcloud_cognito_user_pools:create_identity_provider(UserPoolId, ProviderName, + ProviderType, ProviderDetails, + AttributeMapping, IdpIdentifiers) end, + do_test(Request, {ok, ?CREATE_UPDATE_IDP}, TestFun). + +test_delete_identity_provider() -> + UserPoolId = <<"test">>, + ProviderName = <<"testIdp">>, + + Request = #{ + <<"UserPoolId">> => UserPoolId, + <<"ProviderName">> => ProviderName + }, + + TestFun = fun() -> erlcloud_cognito_user_pools:delete_identity_provider(UserPoolId, ProviderName) end, + do_test(Request, ok, TestFun). + +test_update_identity_provider() -> + UserPoolId = <<"test">>, + ProviderName = <<"testIdp">>, + ProviderDetails = #{ + <<"MetadataURL">> => <<"http://test-idp.com/1234">> + }, + AttributeMapping = #{ + <<"testAttr">> => <<"SAMLAttr">> + }, + IdpIdentifiers = [ "Test IDP" ], + + Request = #{ + <<"UserPoolId">> => UserPoolId, + <<"ProviderName">> => ProviderName, + <<"ProviderDetails">> => ProviderDetails, + <<"AttributeMapping">> => AttributeMapping, + <<"IdpIdentifiers">> => IdpIdentifiers + }, + + TestFun = fun() -> erlcloud_cognito_user_pools:update_identity_provider(UserPoolId, ProviderName, + ProviderDetails, AttributeMapping, + IdpIdentifiers) end, + do_test(Request, {ok, ?CREATE_UPDATE_IDP}, TestFun). + +test_error_retry() -> + erlcloud_cognito_user_pools:configure("test-access-key", "test-secret-key"), + ErrCode1 = 500, + ErrCode2 = 400, + Status1 = "Internal Server Error", + Status2 = "Bad Request", + ErrMsg1 = <<"Message-1">>, + ErrMsg2 = <<"Message-2">>, + Operation = "ListUsers", + Config = config(), + Request = #{<<"UserPoolId">> => ?USER_POOL_ID}, + meck:sequence(?EHTTPC, request, 6, + [{ok, {{ErrCode1, Status1}, [], ErrMsg1}}, + {ok, {{ErrCode2, Status2}, [], ErrMsg2}}]), + ?assertEqual( + {error, {http_error, ErrCode2, Status2, ErrMsg2, []}}, + erlcloud_cognito_user_pools:request(Config, Operation, Request) + ). + +do_test(Request, ExpectedResult, TestedFun) -> + erlcloud_cognito_user_pools:configure("test-access-key", "test-secret-key"), + ?assertEqual(ExpectedResult, TestedFun()), + Encoded = jsx:encode(Request), + ?assertMatch([{_, {?EHTTPC, request, [_, post, _, Encoded, _, _]}, _}], + meck:history(?EHTTPC)). + +do_erlcloud_httpc_request(_, post, Headers, _, _, _) -> + Target = proplists:get_value("x-amz-target", Headers), + ["AWSCognitoIdentityProviderService", Operation] = string:tokens(Target, "."), + RespBody = + case Operation of + "ListUsers" -> ?LIST_USERS_RESP; + "AdminListGroupsForUser" -> ?ADMIN_LIST_GROUPS_FOR_USERS; + "AdminGetUser" -> ?ADMIN_GET_USER; + "AdminCreateUser" -> ?ADMIN_CREATE_USER; + "AdminDeleteUser" -> #{}; + "AdminAddUserToGroup" -> #{}; + "AdminRemoveUserFromGroup" -> #{}; + "CreateGroup" -> ?CREATE_GROUP; + "DeleteGroup" -> #{}; + "AdminResetUserPassword" -> #{}; + "AdminUpdateUserAttributes" -> #{}; + "ChangePassword" -> #{}; + "ListUserPools" -> ?LIST_USER_POOLS; + "AdminSetUserPassword" -> #{}; + "DescribeUserPool" -> ?DESCRIBE_POOL; + "ListUserPoolClients" -> ?LIST_USER_POOL_CLIENTS; + "GetUserPoolMfaConfig" -> ?MFA_CONFIG; + "ListIdentityProviders" -> ?LIST_IDENTITY_PROVIDERS; + "DescribeIdentityProvider" -> ?DESCRIBE_PROVIDER; + "DescribeUserPoolClient" -> ?DESCRIBE_USER_POOL_CLIENT; + "AdminListDevices" -> ?ADMIN_LIST_DEVICE; + "AdminForgetDevice" -> #{}; + "AdminConfirmSignUp" -> #{}; + "AdminInitiateAuth" -> ?ADMIN_INITIATE_AUTH; + "RespondToAuthChallenge" -> ?RESPOND_TO_AUTH_CHALLENGE; + "CreateIdentityProvider" -> ?CREATE_UPDATE_IDP; + "DeleteIdentityProvider" -> #{}; + "UpdateIdentityProvider" -> ?CREATE_UPDATE_IDP + end, + {ok, {{200, "OK"}, [], jsx:encode(RespBody)}}. From 139396337f353cd36e8bcf7c8e9bf6782c4f2e21 Mon Sep 17 00:00:00 2001 From: Aleksei Osin Date: Fri, 24 Jun 2022 21:55:42 +0100 Subject: [PATCH 247/310] Add functions to read server certificates stored in IAM --- src/erlcloud_iam.erl | 85 +++++++++++++++- test/erlcloud_iam_tests.erl | 189 ++++++++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index 82023889f..e376a194e 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -49,6 +49,9 @@ list_entities_for_policy_all/1, list_entities_for_policy_all/2, list_entities_for_policy_all/3, list_entities_for_policy_all/4, get_policy/1, get_policy/2, get_policy_version/2, get_policy_version/3, + get_server_certificate/1, get_server_certificate/2, + list_server_certificates/0, list_server_certificates/1, list_server_certificates/2, + list_server_certificates_all/0, list_server_certificates_all/1, list_server_certificates_all/2, list_instance_profiles/0, list_instance_profiles/1, list_instance_profiles/2, list_instance_profiles_all/0, list_instance_profiles_all/1, list_instance_profiles_all/2, get_instance_profile/1, get_instance_profile/2, @@ -614,6 +617,79 @@ get_policy_version(PolicyArn, VersionId, #aws_config{} = Config) ItemPath = "/GetPolicyVersionResponse/GetPolicyVersionResult/PolicyVersion", iam_query(Config, "GetPolicyVersion", [{"PolicyArn", PolicyArn}, {"VersionId", VersionId}], ItemPath, data_type("PolicyVersion")). +% +% ServerCertificate +% + +-spec get_server_certificate(string()) -> {ok, proplist()} | {error, any()}. +get_server_certificate(ServerCertificateName) -> + get_server_certificate(ServerCertificateName, default_config()). + +-spec get_server_certificate(string(), aws_config()) -> {ok, proplist()} | {error, any()}. +get_server_certificate(ServerCertificateName, #aws_config{} = Config) -> + Action = "GetServerCertificate", + Params = [{"ServerCertificateName", ServerCertificateName}], + case iam_query(Config, Action, Params) of + {ok, Doc} -> + ItemPath = "/GetServerCertificateResponse/GetServerCertificateResult/ServerCertificate", + [Item] = erlcloud_util:get_items(ItemPath, Doc), + Schema = [ + {server_certificate_metadata, "ServerCertificateMetadata", + {single, [ + {server_certificate_name, "ServerCertificateName", text}, + {server_certificate_id, "ServerCertificateId", text}, + {path, "Path", text}, + {arn, "Arn", text}, + {upload_date, "UploadDate", time}, + {expiration, "Expiration", time} + ]} + }, + {certificate_body, "CertificateBody", text}, + {certificate_chain, "CertificateChain", optional_text} + ], + {ok, erlcloud_xml:decode(Schema, Item)}; + {error, _} = Error -> + Error + end. + +-spec list_server_certificates() -> {ok, [proplist()]} | {ok, [proplist()], string()} | {error, any()}. +list_server_certificates() -> + list_server_certificates(default_config()). + +-spec list_server_certificates(string() | aws_config()) -> {ok, [proplist()]} | {ok, [proplist()], string()} | {error, any()}. +list_server_certificates(#aws_config{} = Config) -> + list_server_certificates("/", Config); +list_server_certificates(PathPrefix) -> + list_server_certificates(PathPrefix, default_config()). + +-spec list_server_certificates(string(), aws_config()) -> {ok, [proplist()]} | {ok, [proplist()], string()} | {error, any()}. +list_server_certificates(PathPrefix, #aws_config{} = Config) + when is_list(PathPrefix) -> + Action = "ListServerCertificates", + Params = [{"PathPrefix", PathPrefix}], + ItemPath = "/ListServerCertificatesResponse/ListServerCertificatesResult/ServerCertificateMetadataList/member", + DataTypeDef = data_type("ServerCertificateMetadataList"), + iam_query(Config, Action, Params, ItemPath, DataTypeDef). + +-spec list_server_certificates_all() -> {ok, [proplist()]} | {error, any()}. +list_server_certificates_all() -> + list_server_certificates_all(default_config()). + +-spec list_server_certificates_all(string() | aws_config()) -> {ok, [proplist()]} | {error, any()}. +list_server_certificates_all(#aws_config{} = Config) -> + list_server_certificates_all("/", Config); +list_server_certificates_all(PathPrefix) -> + list_server_certificates_all(PathPrefix, default_config()). + +-spec list_server_certificates_all(string(), aws_config()) -> {ok, [proplist()]} | {error, any()}. +list_server_certificates_all(PathPrefix, #aws_config{} = Config) + when is_list(PathPrefix) -> + Action = "ListServerCertificates", + Params = [{"PathPrefix", PathPrefix}], + ItemPath = "/ListServerCertificatesResponse/ListServerCertificatesResult/ServerCertificateMetadataList/member", + DataTypeDef = data_type("ServerCertificateMetadataList"), + iam_query_all(Config, Action, Params, ItemPath, DataTypeDef). + % % InstanceProfile % @@ -1065,7 +1141,14 @@ data_type("GetAccessKeyLastUsedResult") -> [{"UserName", user_name, "String"}, {"AccessKeyLastUsed/Region", access_key_last_used_region, "String"}, {"AccessKeyLastUsed/ServiceName", access_key_last_used_service_name, "String"}, - {"AccessKeyLastUsed/LastUsedDate", access_key_last_used_date, "DateTime"}]. + {"AccessKeyLastUsed/LastUsedDate", access_key_last_used_date, "DateTime"}]; +data_type("ServerCertificateMetadataList") -> + [{"Expiration", expiration, "DateTime"}, + {"UploadDate", upload_date, "DateTime"}, + {"Arn", arn, "String"}, + {"Path", path, "String"}, + {"ServerCertificateId", server_certificate_id, "String"}, + {"ServerCertificateName", server_certificate_name, "String"}]. data_fun("String") -> {erlcloud_xml, get_text}; data_fun("DateTime") -> {erlcloud_xml, get_time}; diff --git a/test/erlcloud_iam_tests.erl b/test/erlcloud_iam_tests.erl index 84f2ecaed..182b6b76d 100644 --- a/test/erlcloud_iam_tests.erl +++ b/test/erlcloud_iam_tests.erl @@ -72,6 +72,11 @@ iam_api_test_() -> fun list_role_policies_input_tests/1, fun list_role_policies_output_tests/1, fun list_role_policies_all_output_tests/1, + fun get_server_certificate_input_tests/1, + fun get_server_certificate_output_tests/1, + fun list_server_certificates_input_tests/1, + fun list_server_certificates_output_tests/1, + fun list_server_certificates_all_output_tests/1, fun list_instance_profiles_input_tests/1, fun list_instance_profiles_output_tests/1, fun list_instance_profiles_all_output_tests/1, @@ -1718,6 +1723,190 @@ list_role_policies_all_output_tests(_) -> ], output_tests_seq(?_f(erlcloud_iam:list_role_policies_all("S3Access")), Tests). +-define(SERVER_CERTIFICATE_BODY, +"-----BEGIN CERTIFICATE----- +MIICdzCCAeCgAwIBAgIGANc+Ha2wMA0GCSqGSIb3DQEBBQUAMFMxCzAJBgNVBAYT +AlVTMRMwEQYDVQQKEwpBbWF6b24uY29tMQwwCgYDVQQLEwNBV1MxITAfBgNVBAMT +GEFXUyBMaW1pdGVkLUFzc3VyYW5jZSBDQTAeFw0wOTAyMDQxNzE5MjdaFw0xMDAy +MDQxNzE5MjdaMFIxCzAJBgNVBAYTAlVTMRMwEQYDVQQKEwpBbWF6b24uY29tMRcw +FQYDVQQLEw5BV1MtRGV2ZWxvcGVyczEVMBMGA1UEAxMMNTdxNDl0c3ZwYjRtMIGf +MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCpB/vsOwmT/O0td1RqzKjttSBaPjbr +dqwNe9BrOyB08fw2+Ch5oonZYXfGUrT6mkYXH5fQot9HvASrzAKHO596FdJA6DmL +ywdWe1Oggk7zFSXO1Xv+3vPrJtaYxYo3eRIp7w80PMkiOv6M0XK8ubcTouODeJbf +suDqcLnLDxwsvwIDAQABo1cwVTAOBgNVHQ8BAf8EBAMCBaAwFgYDVR0lAQH/BAww +CgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQULGNaBphBumaKbDRK +CAi0mH8B3mowDQYJKoZIhvcNAQEFBQADgYEAuKxhkXaCLGcqDuweKtO/AEw9ZePH +wr0XqsaIK2HZboqruebXEGsojK4Ks0WzwgrEynuHJwTn760xe39rSqXWIOGrOBaX +wFpWHVjTFMKk+tSDG1lssLHyYWWdFFU4AnejRGORJYNaRHgVTKjHphc5jEhHm0BX +AEaHzTpmEXAMPLE= +-----END CERTIFICATE-----"). + +-define(GET_SERVER_CERTIFICATE_RESP, +" + + + + ProdServerCert + /company/servercerts/ + arn:aws:iam::123456789012:server-certificate/company/servercerts/ProdServerCert + 2010-05-08T01:02:03.004Z + ASCACKCEVSQ6C2EXAMPLE + 2012-05-08T01:02:03.004Z + + " + ++ ?SERVER_CERTIFICATE_BODY ++ + " + + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +"). + +get_server_certificate_input_tests(_) -> + Tests = [ + ?_iam_test({ + "Test returning a server certificate.", + ?_f(erlcloud_iam:get_server_certificate("test")), + [{"Action", "GetServerCertificate"}, {"ServerCertificateName", "test"}] + }) + ], + input_tests(?GET_SERVER_CERTIFICATE_RESP, Tests). + +get_server_certificate_output_tests(_) -> + Tests = [ + ?_iam_test({ + "This returns the server certificate", + ?GET_SERVER_CERTIFICATE_RESP, + {ok, [ + {server_certificate_metadata, [ + {server_certificate_name, "ProdServerCert"}, + {server_certificate_id, "ASCACKCEVSQ6C2EXAMPLE"}, + {path, "/company/servercerts/"}, + {arn, "arn:aws:iam::123456789012:server-certificate/company/servercerts/ProdServerCert"}, + {upload_date, {{2010,5,8}, {1,2,3}}}, + {expiration, {{2012,5,8}, {1,2,3}}} + ]}, + {certificate_body, ?SERVER_CERTIFICATE_BODY} + ]} + }) + ], + output_tests(?_f(erlcloud_iam:get_server_certificate("test")), Tests). + +-define(LIST_SERVER_CERTIFICATES_RESP, +" + + false + + + ProdServerCert + /company/servercerts/ + arn:aws:iam::123456789012:server-certificate/company/servercerts/ProdServerCert + 2010-05-08T01:02:03.004Z + ASCACKCEVSQ6C2EXAMPLE + 2012-05-08T01:02:03.004Z + + + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +"). + +list_server_certificates_input_tests(_) -> + Tests = [ + ?_iam_test({ + "Test returning a list of server certificates.", + ?_f(erlcloud_iam:list_server_certificates("test")), + [{"Action", "ListServerCertificates"}, {"PathPrefix", "test"}] + }) + ], + input_tests(?LIST_SERVER_CERTIFICATES_RESP, Tests). + +list_server_certificates_output_tests(_) -> + Tests = [ + ?_iam_test({ + "Test returning a list of server certificates.", + ?LIST_SERVER_CERTIFICATES_RESP, + {ok, [ + [ + {server_certificate_name, "ProdServerCert"}, + {server_certificate_id, "ASCACKCEVSQ6C2EXAMPLE"}, + {path, "/company/servercerts/"}, + {arn, "arn:aws:iam::123456789012:server-certificate/company/servercerts/ProdServerCert"}, + {upload_date, {{2010,5,8}, {1,2,3}}}, + {expiration, {{2012,5,8}, {1,2,3}}} + ] + ]} + }) + ], + output_tests(?_f(erlcloud_iam:list_server_certificates("test")), Tests). + +list_server_certificates_all_output_tests(_) -> + Tests = [ + ?_iam_test({ + "Test returning all server certificates", + [ + " + + true + marker + + + ProdServerCert + /company/servercerts/ + arn:aws:iam::123456789012:server-certificate/company/servercerts/ProdServerCert + 2010-05-08T01:02:03.004Z + ASCACKCEVSQ6CEXAMPLE1 + 2012-05-08T01:02:03.004Z + + + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + + ", + " + + false + + + TestServerCert + /company/servercerts/ + arn:aws:iam::123456789012:server-certificate/company/servercerts/TestServerCert + 2010-05-08T03:01:02.004Z + ASCACKCEVSQ6CEXAMPLE2 + 2012-05-08T03:01:02.004Z + + + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + + " + ], + {ok, [ + [ + {server_certificate_name, "ProdServerCert"}, + {server_certificate_id, "ASCACKCEVSQ6CEXAMPLE1"}, + {path, "/company/servercerts/"}, + {arn, "arn:aws:iam::123456789012:server-certificate/company/servercerts/ProdServerCert"}, + {upload_date, {{2010,5,8}, {1,2,3}}}, + {expiration, {{2012,5,8}, {1,2,3}}} + ], + [ + {server_certificate_name, "TestServerCert"}, + {server_certificate_id, "ASCACKCEVSQ6CEXAMPLE2"}, + {path, "/company/servercerts/"}, + {arn, "arn:aws:iam::123456789012:server-certificate/company/servercerts/TestServerCert"}, + {upload_date, {{2010,5,8}, {3,1,2}}}, + {expiration, {{2012,5,8}, {3,1,2}}} + ] + ]} + }) + ], + output_tests_seq(?_f(erlcloud_iam:list_server_certificates_all("test")), Tests). + -define(LIST_INSTANCE_PROFILES_RESP, " From ccd73b70e9a12109a5e18641e647cf70e0e7e636 Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Fri, 8 Jul 2022 14:51:46 +0100 Subject: [PATCH 248/310] adding mfa_delete attribute and public_access_block functions --- src/erlcloud_s3.erl | 102 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 99 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index d6eb8cc33..b278ad386 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -50,7 +50,9 @@ get_object_tagging/2, get_object_tagging/3, put_bucket_tagging/2, put_bucket_tagging/3, delete_bucket_tagging/1, delete_bucket_tagging/2, - get_bucket_tagging/1, get_bucket_tagging/2 + get_bucket_tagging/1, get_bucket_tagging/2, + get_public_access_block/1, get_public_access_block/2, + put_public_access_block/2, put_public_access_block/3 ]). -ifdef(TEST). @@ -126,6 +128,7 @@ configure(AccessKeyID, SecretAccessKey, Host, Port, Scheme) -> -type s3_bucket_attribute_name() :: acl | location | logging + | mfa_delete | request_payment | versioning | notification. @@ -160,6 +163,14 @@ configure(AccessKeyID, SecretAccessKey, Host, Port, Scheme) -> | 'me-south-1' | 'sa-east-1'. +-type public_access_block_attrib() :: block_public_policy + | ignore_public_acls + | block_public_policy + | restrict_public_buckets. + +-type public_access_block_param() :: {public_access_block_attrib(), boolean()}. + +-type public_access_block_configuration() :: [public_access_block_param()]. -define(XMLNS_S3, "http://s3.amazonaws.com/doc/2006-03-01/"). -define(XMLNS_SCHEMA_INSTANCE, "http://www.w3.org/2001/XMLSchema-instance"). @@ -579,6 +590,7 @@ get_bucket_attribute(BucketName, AttributeName, Config) acl -> "acl"; location -> "location"; logging -> "logging"; + mfa_delete -> "versioning"; %% https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketVersioning.html#API_GetBucketVersioning_ResponseElements request_payment -> "requestPayment"; versioning -> "versioning"; notification -> "notification" @@ -608,6 +620,11 @@ get_bucket_attribute(BucketName, AttributeName, Config) {target_trants, "TargetGrants/Grant", fun extract_acl/1}], [{enabled, true}|erlcloud_xml:decode(Attributes, LoggingEnabled)] end; + mfa_delete -> + case erlcloud_xml:get_text("/VersioningConfiguration/MFADelete", Doc) of + "Enabled" -> enabled; + _ -> disabled + end; request_payment -> case erlcloud_xml:get_text("/RequestPaymentConfiguration/Payer", Doc) of "Requester" -> requester; @@ -1237,7 +1254,6 @@ list_multipart_uploads(BucketName, Options, HTTPHeaders, Config) Error end. - -spec set_bucket_attribute(string(), atom(), term()) -> ok. set_bucket_attribute(BucketName, AttributeName, Value) -> @@ -1283,7 +1299,8 @@ set_bucket_attribute(BucketName, AttributeName, Value, Config) ] }, {"requestPayment", RPXML}; - versioning -> + Versioning when Versioning =:= versioning; + Versioning =:= mfa_delete -> Status = case proplists:get_value(status, Value) of suspended -> "Suspended"; enabled -> "Enabled" @@ -1682,6 +1699,69 @@ get_bucket_encryption(BucketName, Config) -> Error end. +%% @doc +%% S3 API: +%% https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetPublicAccessBlock.html +%% +%% ` +%% {ok, [{block_public_acls, true}, +%% {ignore_public_acls, true}, +%% {block_public_policy, true}, +%% {restrict_public_buckets, true}]} = erlcloud_s3:get_public_access_block("bucket-name"). +%% ' +%% +-spec get_public_access_block(string()) -> + {ok, public_access_block_configuration()} | {error, any()}. +get_public_access_block(BucketName) -> + get_public_access_block(BucketName, default_config()). + +-spec get_public_access_block(string(), aws_config()) -> + {ok, public_access_block_configuration()} | {error, any()}. +get_public_access_block(BucketName, Config) -> + case s3_xml_request2(Config, get, BucketName, "/", "publicAccessBlock", [], <<>>, []) of + {ok, XML} -> + XPath = "/PublicAccessBlockConfiguration", + BlockPublicAcls = XPath ++ "/BlockPublicAcls", + IgnorePublicAcls = XPath ++ "/IgnorePublicAcls", + BlockPublicPolicy = XPath ++ "/BlockPublicPolicy", + RestrictPublicBuckets = XPath ++ "/RestrictPublicBuckets", + {ok, [ + {block_public_acls, erlcloud_xml:get_bool(BlockPublicAcls, XML)}, + {ignore_public_acls, erlcloud_xml:get_bool(IgnorePublicAcls, XML)}, + {block_public_policy, erlcloud_xml:get_bool(BlockPublicPolicy, XML)}, + {restrict_public_buckets, erlcloud_xml:get_bool(RestrictPublicBuckets, XML)} + ]}; + Error -> + Error + end. + +%% @doc +%% S3 API: +%% https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutPublicAccessBlock.html +%% +%% ` +%% PublicAccessBlock = [{block_public_acls, true}, +%% {ignore_public_acls, true}, +%% {block_public_policy, true}, +%% {restrict_public_buckets, true}], +%% ok = erlcloud_s3:put_public_access_block("bucket-name", PublicAccessBlock). +%% ' +%% +-spec put_public_access_block(string(), public_access_block_configuration()) -> + ok | {error, any()}. +put_public_access_block(BucketName, PublicAccessBlock) -> + put_public_access_block(BucketName, PublicAccessBlock, default_config()). + +-spec put_public_access_block(string(), public_access_block_configuration(), aws_config()) -> + ok | {error, any()}. +put_public_access_block(BucketName, PublicAccessBlock, Config) -> + PublicAccessBlockXML = {'PublicAccessBlockConfiguration', [{xmlns, ?XMLNS_S3}], + encode_public_access_block(PublicAccessBlock)}, + POSTData = list_to_binary(xmerl:export_simple([PublicAccessBlockXML], xmerl_xml)), + Md5 = base64:encode(crypto:hash(md5, POSTData)), + Headers = [{"content-md5", Md5}, {"content-type", "application/xml"}], + s3_simple_request(Config, put, BucketName, "/", "publicAccessBlock", [], POSTData, Headers). + %% @doc %% S3 API: %% https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEencryption.html @@ -2084,3 +2164,19 @@ s3_endpoint_for_region(RegionName) -> _ -> lists:flatten(["s3-", RegionName, ".amazonaws.com"]) end. + +encode_public_access_block(PublicAccessBlockConfiguration) -> + [encode_public_access_block_param(Param) + || {_Attrib, Value} = Param <- PublicAccessBlockConfiguration, + Value =:= true orelse Value =:= false]. + +encode_public_access_block_param({block_public_acls, Value}) -> + {'BlockPublicAcls', [atom_to_list(Value)]}; +encode_public_access_block_param({ignore_public_acls, Value}) -> + {'IgnorePublicAcls', [atom_to_list(Value)]}; +encode_public_access_block_param({block_public_policy, Value}) -> + {'BlockPublicPolicy', [atom_to_list(Value)]}; +encode_public_access_block_param({restrict_public_buckets, Value}) -> + {'RestrictPublicBuckets', [atom_to_list(Value)]}; +encode_public_access_block_param(_) -> + throw(unknown_public_access_block_param). From 211fc1e2bf00e0d92e6309b0050bab05fb7a8913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Tholsg=C3=A5rd?= Date: Mon, 1 Aug 2022 22:53:52 +0700 Subject: [PATCH 249/310] Fix reading error message from failed DDB requests --- src/erlcloud_ddb_impl.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_ddb_impl.erl b/src/erlcloud_ddb_impl.erl index 60d076c39..a1044204e 100644 --- a/src/erlcloud_ddb_impl.erl +++ b/src/erlcloud_ddb_impl.erl @@ -248,7 +248,7 @@ client_error(Body, DDBError) -> undefined -> DDBError#ddb2_error{error_type = http, should_retry = false}; FullType -> - Message = proplists:get_value(<<"Message">>, Json, <<>>), + Message = proplists:get_value(<<"message">>, Json, <<>>), case binary:split(FullType, <<"#">>) of [_, Type] when Type =:= <<"ProvisionedThroughputExceededException">> orelse From 3ea195fc7eceec4ae86128cedf2ab78490901899 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Tue, 2 Aug 2022 21:39:39 +0800 Subject: [PATCH 250/310] Fix markdown issues raised by markdownlint (#730) See detailed description of the rules is available at https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md --- README.md | 197 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 127 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 81481e588..7e968df4e 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,14 @@ -# erlcloud: AWS APIs library for Erlang # +# erlcloud: AWS APIs library for Erlang [![Build Status](https://github.com/erlcloud/erlcloud/workflows/build/badge.svg)](https://github.com/erlcloud/erlcloud) -This library is not developed or maintained by AWS thus lots of functionality is still missing comparing to [aws-cli](https://aws.amazon.com/cli/) or [boto](https://github.com/boto/boto). -Required functionality is being added upon request. +This library is not developed or maintained by AWS thus lots of functionality +is still missing comparing to [aws-cli](https://aws.amazon.com/cli/) or +[boto](https://github.com/boto/boto). Required functionality is being added +upon request. Service APIs implemented: + - Amazon Elastic Compute Cloud (EC2) - Amazon EC2 Container Service (ECS) - Amazon Simple Storage Service (S3) @@ -40,15 +43,18 @@ Service APIs implemented: - AWS Systems Manager (SSM) - and more to come -The majority of API functions have been implemented. -Not all functions have been thoroughly tested, so exercise care when integrating this library into production code. -Please send issues and patches. +The majority of API functions have been implemented. Not all functions have +been thoroughly tested, so exercise care when integrating this library into +production code. Please send issues and patches. + +The libraries can be used two ways. You can: -The libraries can be used two ways. You can -- specify configuration parameters in the process dictionary. Useful for simple tasks, or -- create a configuration object and pass that to each request as the final parameter. Useful for Cross AWS Account access +- specify configuration parameters in the process dictionary. Useful for simple + tasks, or +- create a configuration object and pass that to each request as the final + parameter. Useful for Cross AWS Account access -## Roadmap ## +## Roadmap Below is the library roadmap update along with regular features and fixes. @@ -59,94 +65,129 @@ Below is the library roadmap update along with regular features and fixes. - 3.X.X - Fix dialyzer findings and make it mandatory for the library - Only SigV4 signing and generalised in one module. Keep SigV2 in SBD section only - - No more `erlang:error()` use and use of regular tuples as error API. Breaking change. + - No more `erlang:error()` use and use of regular tuples as error API. + Breaking change. ### Major API compatibility changes between 0.13.X and 2.0.x - - ELB APIs - - ... list to be filled shortly + +- ELB APIs +- ... list to be filled shortly ### Supported Erlang versions + At the moment we support the following OTP releases: - - 19.3 - - 20.3 - - 21.3 - - 22.3 - -it might still work on 17+ (primarily due to Erlang maps) but we do not guarantee that. - -## Getting started ## -You need to clone the repository and download rebar/rebar3 (if it's not already available in your path). -``` + +- 19.3 +- 20.3 +- 21.3 +- 22.3 + +It might still work on 17+ (primarily due to Erlang maps) but we do not +guarantee that. + +## Getting started + +You need to clone the repository and download rebar/rebar3 (if it's not already +available in your path). + +```sh git clone https://github.com/erlcloud/erlcloud.git cd erlcloud wget https://s3.amazonaws.com/rebar3/rebar3 chmod a+x rebar3 ``` -To compile and run erlcloud -``` + +To compile and run erlcloud: + +```sh make make run ``` -If you're using erlcloud in your application, add it as a dependency in your application's configuration file. -To use erlcloud in the shell, you can start it by calling: +If you're using erlcloud in your application, add it as a dependency in your +application's configuration file. To use erlcloud in the shell, you can start +it by calling: -``` +```erlang application:ensure_all_started(erlcloud). ``` + ### Using Temporary Security Credentials -When access to AWS resources is managed through [third-party identity providers](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp.html) it is performed using [temporary security credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). +When access to AWS resources is managed through [third-party identity +providers](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-idp.html) +it is performed using [temporary security +credentials](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html). You can provide your AWS credentials in OS environment variables -``` +```sh export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= export AWS_SESSION_TOKEN= export AWS_DEFAULT_REGION= ``` -If you did not provide your AWS credentials in the OS environment variables, then you need to provide configuration read from your profile: -``` + +If you did not provide your AWS credentials in the OS environment variables, +then you need to provide configuration read from your profile: + +```erlang {ok, Conf} = erlcloud_aws:profile(). erlcloud_s3:list_buckets(Conf). ``` + Or you can provide them via `erlcloud` application environment variables. + ```erlang application:set_env(erlcloud, aws_access_key_id, "your key"), application:set_env(erlcloud, aws_secret_access_key, "your secret key"), application:set_env(erlcloud, aws_security_token, "your token"), application:set_env(erlcloud, aws_region, "your region"), ``` -### Using Access Key ### + +### Using Access Key + You can provide your AWS credentials in environmental variables. -``` + +```sh export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= ``` -If you did not provide your AWS credentials in the environment variables, then you need to provide the per-process configuration: -``` + +If you did not provide your AWS credentials in the environment variables, then +you need to provide the per-process configuration: + +```erlang erlcloud_ec2:configure(AccessKeyId, SecretAccessKey [, Hostname]). ``` -Hostname defaults to non-existing `"ec2.amazonaws.com"` intentionally to avoid mix with US-East-1 -Refer to [aws_config](https://github.com/erlcloud/erlcloud/blob/master/include/erlcloud_aws.hrl) for full description of all services configuration. + +Hostname defaults to non-existing `"ec2.amazonaws.com"` intentionally to avoid +mix with US-East-1 Refer to +[aws_config](https://github.com/erlcloud/erlcloud/blob/master/include/erlcloud_aws.hrl) +for full description of all services configuration. Configuration object usage: -``` + +```erlang EC2 = erlcloud_ec2:new(AccessKeyId, SecretAccessKey [, Hostname]) erlcloud_ec2:describe_images(EC2). ``` ### aws_config -The [aws_config](https://github.com/erlcloud/erlcloud/blob/master/include/erlcloud_aws.hrl) record contains many valuable defaults, -such as protocols and ports for AWS services. You can always redefine them by making new `#aws_config{}` record and -changing particular fields, then passing the result to any erlcloud function. -But if you want to change something in runtime this might be tedious and/or not flexible enough. -An alternative approach is to set default fields within the `app.config -> erlcloud -> aws_config` section and -rely on the config, used by all functions by default. +The [aws_config](https://github.com/erlcloud/erlcloud/blob/master/include/erlcloud_aws.hrl) +record contains many valuable defaults, such as protocols and ports for AWS +services. You can always redefine them by making new `#aws_config{}` record and +changing particular fields, then passing the result to any `erlcloud` function. + +But if you want to change something in runtime this might be tedious and/or not +flexible enough. -Example of such app.config: +An alternative approach is to set default fields within the `app.config -> +erlcloud -> aws_config` section and rely on the config, used by all functions +by default. + +Example of such `app.config`: ```erlang [ @@ -160,7 +201,10 @@ Example of such app.config: ``` ### VPC endpoints -If you want to utilise AZ affinity for VPC endpoints you can configure those in application config via: + +If you want to utilise AZ affinity for VPC endpoints you can configure those in +application config via: + ```erlang {erlcloud, [ {services_vpc_endpoints, [ @@ -169,17 +213,23 @@ If you want to utilise AZ affinity for VPC endpoints you can configure those in ]} ]} ``` + Two options are supported: - - explicit list of Route53 AZ endpoints - - OS environment variable (handy for ECS deployments). The value of the variable should be of comma-separated string like `"myAZ1.sqs-dns.amazonaws.com,myAZ2.sqs-dns.amazonaws.com"` - -Upon config generation, `erlcloud` will check the AZ of the deployment -and match it to one of the pre-configured DNS records. First match is used and if not match found default is used. +- explicit list of Route53 AZ endpoints +- OS environment variable (handy for ECS deployments). The value of the + variable should be of comma-separated string like + `"myAZ1.sqs-dns.amazonaws.com,myAZ2.sqs-dns.amazonaws.com"` + +Upon config generation, `erlcloud` will check the AZ of the deployment and +match it to one of the pre-configured DNS records. First match is used and if +not match found default is used. + +## Basic use -## Basic use ## Then you can start making API calls, like: -``` + +```erlang erlcloud_ec2:describe_images(). % list buckets of Account stored in config in process dict % of of the account you are running in. @@ -190,13 +240,13 @@ erlcloud_s3:list_buckets(Conf). ``` Creating an EC2 instance may look like this: + ```erlang start_instance(Ami, KeyPair, UserData, Type, Zone) -> Config = #aws_config{ access_key_id = application:get_env(aws_key), secret_access_key = application:get_env(aws_secret) }, - InstanceSpec = #ec2_instance_spec{image_id = Ami, key_name = KeyPair, instance_type = Type, @@ -205,24 +255,31 @@ start_instance(Ami, KeyPair, UserData, Type, Zone) -> erlcloud_ec2:run_instances(InstanceSpec, Config). ``` -For usage information, consult the source code and https://hexdocs.pm/erlcloud. -For detailed API description refer to the AWS references at: +For usage information, consult the source code and +[GitHub repo](https://hexdocs.pm/erlcloud). For detailed API description refer +to the AWS references at: -- http://docs.aws.amazon.com/AWSEC2/latest/APIReference/Welcome.html -- http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html -- and other services https://aws.amazon.com/documentation/ +- [AmazonEC2](http://docs.aws.amazon.com/AWSEC2/latest/APIReference/Welcome.html) +- [Amazon S3](http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html) +- and [other services](https://aws.amazon.com/documentation/) -## Notes ## +## Notes -Indentation in contributions should follow indentation style of surrounding text. -In general it follows default indentation rules of official erlang-mode as provided by OTP team. +Indentation in contributions should follow indentation style of surrounding +text. In general it follows default indentation rules of official erlang-mode +as provided by OTP team. -## Best Practices ## +## Best Practices - All interfaces should provide a method for working with non-default config. -- Public interfaces with paging logic should prefer `{ok, Results, Marker}` style to the `{{paged, Marker}, Results}` found in some modules. -In case of records output, tokens should be part of the record. -- Passing next page `NextToken`, `NextMarker` is preferred with `Opts` rather than a fun parameter like found in many modules. -- Public interfaces should normally expose proplists over records. All new modules are preferred to have both. -- Exposed records are to be used only for complex outputs. Examples to follow: ddb2, ecs. -- Library should not expose any long running or stateful processes - no gen_servers, no caches and etc. +- Public interfaces with paging logic should prefer `{ok, Results, Marker}` + style to the `{{paged, Marker}, Results}` found in some modules. + In case of records output, tokens should be part of the record. +- Passing next page `NextToken`, `NextMarker` is preferred with `Opts` rather + than a fun parameter like found in many modules. +- Public interfaces should normally expose proplists over records. All new + modules are preferred to have both. +- Exposed records are to be used only for complex outputs. Examples to follow: + ddb2, ecs. +- Library should not expose any long running or stateful processes - no + gen_servers, no caches and etc. From c4f4f5191b57078077c661b2ec8cf467baa0bc1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Tholsg=C3=A5rd?= Date: Tue, 2 Aug 2022 22:07:10 +0700 Subject: [PATCH 251/310] Fix eunit tests to handle reading error message --- test/erlcloud_ddb2_tests.erl | 8 ++++---- test/erlcloud_ddb_tests.erl | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index 9f468bc8f..c82047f99 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -273,7 +273,7 @@ error_handling_tests(_) -> {"Test retry after ProvisionedThroughputExceededException", [httpc_response(400, " {\"__type\":\"com.amazonaws.dynamodb.v20111205#ProvisionedThroughputExceededException\", -\"Message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\"}" +\"message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\"}" ), OkResponse], OkResult}), @@ -281,7 +281,7 @@ error_handling_tests(_) -> {"Test ConditionalCheckFailed error", [httpc_response(400, " {\"__type\":\"com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException\", -\"Message\":\"The expected value did not match what was stored in the system.\"}" +\"message\":\"The expected value did not match what was stored in the system.\"}" )], {error, {<<"ConditionalCheckFailedException">>, <<"The expected value did not match what was stored in the system.">>}}}), ?_ddb_test( @@ -315,7 +315,7 @@ error_handling_tests(_) -> \"Message\":\"The conditional request failed\", } ], - \"Message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]\" + \"message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ConditionalCheckFailed]\" }" )], TransactionErrorResult}), @@ -333,7 +333,7 @@ error_handling_tests(_) -> \"Message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\", } ], - \"Message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ProvisionedThroughputExceeded]\" + \"message\":\"Transaction cancelled, please refer cancellation reasons for specific reasons [None, ProvisionedThroughputExceeded]\" }" ), TransactOkResponse], diff --git a/test/erlcloud_ddb_tests.erl b/test/erlcloud_ddb_tests.erl index 4b57c29b2..b607c0fcc 100644 --- a/test/erlcloud_ddb_tests.erl +++ b/test/erlcloud_ddb_tests.erl @@ -211,7 +211,7 @@ error_handling_tests(_) -> {"Test retry after ProvisionedThroughputExceededException", [httpc_response(400, " {\"__type\":\"com.amazonaws.dynamodb.v20111205#ProvisionedThroughputExceededException\", -\"Message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\"}" +\"message\":\"The level of configured provisioned throughput for the table was exceeded. Consider increasing your provisioning level with the UpdateTable API\"}" ), OkResponse], OkResult}), @@ -219,7 +219,7 @@ error_handling_tests(_) -> {"Test ConditionalCheckFailed error", [httpc_response(400, " {\"__type\":\"com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException\", -\"Message\":\"The expected value did not match what was stored in the system.\"}" +\"message\":\"The expected value did not match what was stored in the system.\"}" )], {error, {<<"ConditionalCheckFailedException">>, <<"The expected value did not match what was stored in the system.">>}}}), ?_ddb_test( From ac58528ad9aa393d3132175f3a020278474f891e Mon Sep 17 00:00:00 2001 From: neil-bleasdale <116366258+neil-bleasdale@users.noreply.github.com> Date: Mon, 24 Oct 2022 15:35:14 +0100 Subject: [PATCH 252/310] Add Launch Template retrieval to EC2 (#734) Add functions to allow calling of API endpoints for describe launch templates and describe launch template versions. --- src/erlcloud_ec2.erl | 468 +++++++++++++++++++++++++++++++++++- test/erlcloud_ec2_tests.erl | 314 +++++++++++++++++++++++- 2 files changed, 779 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index c45a67ba5..d84c59658 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -193,8 +193,20 @@ % VPC peering connections describe_vpc_peering_connections/0, describe_vpc_peering_connections/1, - describe_vpc_peering_connections/2, describe_vpc_peering_connections/3 - + describe_vpc_peering_connections/2, describe_vpc_peering_connections/3, + + % Launch templates + describe_launch_templates/0, describe_launch_templates/1, + describe_launch_templates/2, describe_launch_templates/3, + describe_launch_templates/4, describe_launch_templates/5, + + describe_launch_templates_all/0, describe_launch_templates_all/1, + + describe_launch_template_versions/1, describe_launch_template_versions/3, + describe_launch_template_versions/4, describe_launch_template_versions/5, + describe_launch_template_versions/6, describe_launch_template_versions/7, + + describe_launch_template_versions_all/2, describe_launch_template_versions_all/3 ]). -import(erlcloud_xml, [get_text/1, get_text/2, get_text/3, get_bool/2, get_list/2, get_integer/2]). @@ -228,6 +240,8 @@ -define(NAT_GATEWAYS_MR_MAX, 1000). -define(FLOWS_MR_MIN, 1). -define(FLOWS_MR_MAX, 1000). +-define(LAUNCH_TEMPLATES_MR_MIN, 1). +-define(LAUNCH_TEMPLATES_MR_MAX, 200). -type filter_list() :: [{string() | atom(),[string()] | string()}] | none. -type ec2_param_list() :: [{string(),string()}]. @@ -253,6 +267,7 @@ -type vpc_peering_connection_ids() :: [string()]. -type flow_id() :: string(). -type ec2_flow_ids() :: [flow_id()]. +-type launch_template_ids() :: [string()]. -spec new(string(), string()) -> aws_config(). @@ -3896,3 +3911,452 @@ extract_unsuccesful_item(Node) -> {error, [ {code, get_text("error/code", Node)}, {message, get_text("error/message", Node)}]} ]. + +%%%------------------------------------------------------------------------------ +%% @doc +%% Launch Templates API - DescribeLaunchTemplates +%% [https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeLaunchTemplates.html] +%% +%% ===Example=== +%% +%% Describe launch templates with matching launch template IDs. +%% +%% ` +%% {ok, Results} = erlcloud_ec2:describe_launch_templates(["lt-0a20c965061f64abc", "lt-32415965061f007aa"]) +%% ' +%% @end +%%%------------------------------------------------------------------------------ +-spec describe_launch_templates() -> ok_error([proplist()]). +describe_launch_templates() -> + describe_launch_templates([]). + +-spec describe_launch_templates(launch_template_ids()) -> ok_error([proplist()]); + (aws_config()) -> ok_error([proplist()]). +describe_launch_templates(LaunchTemplateIds) + when is_list(LaunchTemplateIds) -> + describe_launch_templates(LaunchTemplateIds, []); +describe_launch_templates(Config) + when is_record(Config, aws_config) -> + describe_launch_templates([], Config). + +-spec describe_launch_templates(launch_template_ids(), filter_list()) -> ok_error([proplist()]); + (launch_template_ids(), aws_config()) -> ok_error([proplist()]). +describe_launch_templates(LaunchTemplateIds, Filter) + when is_list(LaunchTemplateIds), is_list(Filter) -> + describe_launch_templates(LaunchTemplateIds, Filter, default_config()); +describe_launch_templates(LaunchTemplateIds, Config) + when is_list(LaunchTemplateIds), is_record(Config, aws_config) -> + describe_launch_templates(LaunchTemplateIds, [], Config). + +-spec describe_launch_templates(launch_template_ids(), filter_list(), aws_config()) -> ok_error([proplist()]). +describe_launch_templates(LaunchTemplateIds, Filter, Config) + when is_list(LaunchTemplateIds), is_list(Filter), is_record(Config, aws_config) -> + Params = erlcloud_aws:param_list(LaunchTemplateIds, "LaunchTemplateId") ++ list_to_ec2_filter(Filter), + case ec2_query(Config, "DescribeLaunchTemplates", Params, ?NEW_API_VERSION) of + {ok, Doc} -> + LaunchTemplates = extract_results("DescribeLaunchTemplatesResponse", "launchTemplates", fun extract_launch_template/1, Doc), + {ok, LaunchTemplates}; + {error, _} = E -> E + end. + +-spec describe_launch_templates(launch_template_ids(), filter_list(), ec2_max_result(), ec2_token()) -> ok_error([proplist()]); + (filter_list(), ec2_max_result(), ec2_token(), aws_config()) -> ok_error([proplist()]). +describe_launch_templates(LaunchTemplateIds, Filter, MaxResults, NextToken) + when is_list(LaunchTemplateIds), is_list(Filter) orelse Filter =:= none, is_integer(MaxResults), + is_list(NextToken) orelse NextToken =:= undefined -> + describe_launch_templates(LaunchTemplateIds, Filter, MaxResults, NextToken, default_config()); +describe_launch_templates(Filter, MaxResults, NextToken, Config) + when is_list(Filter) orelse Filter =:= none, is_integer(MaxResults), + is_list(NextToken) orelse NextToken =:= undefined, is_record(Config, aws_config) -> + describe_launch_templates([], Filter, MaxResults, NextToken, Config). + +-spec describe_launch_templates(launch_template_ids(), filter_list(), ec2_max_result(), ec2_token(), aws_config()) -> ok_error([proplist()], ec2_token()). +describe_launch_templates(LaunchTemplateIds, Filter, MaxResults, NextToken, Config) + when is_list(LaunchTemplateIds), is_list(Filter) orelse Filter =:= none, + is_integer(MaxResults) andalso MaxResults >= ?LAUNCH_TEMPLATES_MR_MIN andalso MaxResults =< ?LAUNCH_TEMPLATES_MR_MAX, + is_list(NextToken) orelse NextToken =:= undefined, + is_record(Config, aws_config) -> + Params = erlcloud_aws:param_list(LaunchTemplateIds, "LaunchTemplateId") ++ list_to_ec2_filter(Filter) ++ + [{"MaxResults", MaxResults}, {"NextToken", NextToken}], + case ec2_query(Config, "DescribeLaunchTemplates", Params, ?NEW_API_VERSION) of + {ok, Doc} -> + LaunchTemplates = extract_results("DescribeLaunchTemplatesResponse", "launchTemplates", fun extract_launch_template/1, Doc), + NewNextToken = extract_next_token("DescribeLaunchTemplatesResponse", Doc), + {ok, LaunchTemplates, NewNextToken}; + {error, _} = E -> E + end. + +-spec describe_launch_templates_all() -> ok_error([proplist()]). +describe_launch_templates_all() -> + describe_launch_templates_all(default_config()). + +-spec describe_launch_templates_all(aws_config()) -> ok_error([proplist()]). +describe_launch_templates_all(Config) + when is_record(Config, aws_config) -> + describe_launch_templates_all(Config, undefined, []). + +describe_launch_templates_all(Config, Token, Acc) + when is_record(Config, aws_config) -> + case describe_launch_templates([], none, ?LAUNCH_TEMPLATES_MR_MAX, Token, Config) of + {ok, Results, undefined} -> {ok, Results ++ Acc}; + {ok, Results, Next} -> describe_launch_templates_all(Config, Next, Results ++ Acc); + {error, _} = Error -> Error + end. + +launch_template_version_opts() -> + [ + {launch_template_version, "LaunchTemplateVersion"}, + {max_version, "MaxVersion"}, + {min_version, "MinVersion"} + ]. + +set_launch_template_optional_selector({_, none}, Acc) -> Acc; +set_launch_template_optional_selector({_, []}, Acc) -> Acc; +set_launch_template_optional_selector({Key, Value}, Acc) -> + [{Key, Value} | Acc]. + +set_launch_template_selectors(LaunchTemplateId, LaunchTemplateName) -> + ArgList = [{"LaunchTemplateId", LaunchTemplateId}, {"LaunchTemplateName", LaunchTemplateName}], + lists:foldl(fun set_launch_template_optional_selector/2, [], ArgList). + +set_launch_template_version_opts(Opts) -> + OptTable = launch_template_version_opts(), + Sorted = lists:ukeysort(1, Opts), + FnFolder = fun({K, Val}, Acc) -> + case lists:keyfind(K, 1, OptTable) of + {launch_template_version, ApiArg} -> erlcloud_aws:param_list(Val, ApiArg) ++ Acc; + {_, ApiArg} -> [{ApiArg, Val} | Acc]; + false -> Acc + end + end, + lists:foldl(FnFolder, [], Sorted). + +%%%------------------------------------------------------------------------------ +%% @doc +%% Launch Templates API - DescribeLaunchTemplateVersions +%% [https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeLaunchTemplateVersions.html] +%% +%% ===Example=== +%% +%% Describe launch template versions for the template described by ID. +%% +%% ` +%% {ok, Results} = erlcloud_ec2:describe_launch_template_versions("lt-0a20c965061f64abc") +%% ' +%% +%% Describe launch template versions for the template with the given name. +%% +%% ` +%% {ok, Results} = erlcloud_ec2:describe_launch_template_versions(none, "MyTemplateName") +%% ' +%% +%% Describe most recent launch template version for the template with the given ID. +%% +%% ` +%% {ok, Results} = erlcloud_ec2:describe_launch_template_versions("lt-0a20c965061f64abc", none, [{launch_template_version, ["$Latest"]}]) +%% ' +%% +%% @end +%%%------------------------------------------------------------------------------ +-spec describe_launch_template_versions(string()) -> ok_error([proplist()]). +describe_launch_template_versions(LaunchTemplateId) -> + describe_launch_template_versions(LaunchTemplateId, none, []). + +-spec describe_launch_template_versions(string() | none, string() | none, list()) -> ok_error([proplist()]); + (string() | none, string() | none, aws_config()) -> ok_error([proplist()]). +describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none, + is_list(Opts) -> + describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, default_config()); +describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Config) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none, + is_record(Config, aws_config) -> + describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, [], Config). + +-spec describe_launch_template_versions(string() | none, string() | none, list(), filter_list()) -> ok_error([proplist()]); + (string() | none, string() | none, list(), aws_config()) -> ok_error([proplist()]). +describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, Filter) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none, + is_list(Opts), is_list(Filter) -> + describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, Filter, default_config()); +describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, Config) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none, + is_list(Opts), is_record(Config, aws_config) -> + describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, [], Config). + +-spec describe_launch_template_versions(string() | none, string() | none, list(), filter_list(), aws_config()) -> ok_error([proplist()]). +describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, Filter, Config) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none, + is_list(Opts), is_list(Filter), is_record(Config, aws_config) -> + Params = set_launch_template_selectors(LaunchTemplateId, LaunchTemplateName) ++ + set_launch_template_version_opts(Opts) ++ list_to_ec2_filter(Filter), + case ec2_query(Config, "DescribeLaunchTemplateVersions", Params, ?NEW_API_VERSION) of + {ok, Doc} -> + LaunchTemplateVersions = extract_results("DescribeLaunchTemplateVersionsResponse", "launchTemplateVersionSet", fun extract_launch_template_version/1, Doc), + {ok, LaunchTemplateVersions}; + {error, _} = E -> E + end. + +-spec describe_launch_template_versions(string() | none, string() | none, list(), filter_list(), ec2_max_result(), ec2_token()) -> ok_error([proplist()]). +describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, Filter, MaxResults, NextToken) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none, is_list(Opts), is_list(Filter) orelse Filter =:= none, + is_integer(MaxResults) andalso MaxResults >= ?LAUNCH_TEMPLATES_MR_MIN andalso MaxResults =< ?LAUNCH_TEMPLATES_MR_MAX, + is_list(NextToken) orelse NextToken =:= undefined -> + describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, Filter, MaxResults, NextToken, default_config()). + +-spec describe_launch_template_versions(string() | none, string() | none, list(), filter_list(), ec2_max_result(), ec2_token(), aws_config()) + -> ok_error([proplist()], ec2_token()). +describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, Opts, Filter, MaxResults, NextToken, Config) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none, is_list(Opts), is_list(Filter) orelse Filter =:= none, + is_list(NextToken) orelse NextToken =:= undefined, + is_integer(MaxResults) andalso MaxResults >= ?LAUNCH_TEMPLATES_MR_MIN andalso MaxResults =< ?LAUNCH_TEMPLATES_MR_MAX, + is_record(Config, aws_config) -> + Params = set_launch_template_version_opts(Opts) ++ list_to_ec2_filter(Filter) ++ + set_launch_template_selectors(LaunchTemplateId, LaunchTemplateName) ++ + [{"NextToken", NextToken}, {"MaxResults", MaxResults}], + case ec2_query(Config, "DescribeLaunchTemplateVersions", Params, ?NEW_API_VERSION) of + {ok, Doc} -> + LaunchTemplateVersions = extract_results("DescribeLaunchTemplateVersionsResponse", "launchTemplateVersionSet", fun extract_launch_template_version/1, Doc), + NewNextToken = extract_next_token("DescribeLaunchTemplateVersionsResponse", Doc), + {ok, LaunchTemplateVersions, NewNextToken}; + {error, _} = E -> E + end. + +-spec describe_launch_template_versions_all(string() | none, string() | none) -> ok_error([proplist()]). +describe_launch_template_versions_all(LaunchTemplateId, LaunchTemplateName) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none -> + describe_launch_template_versions_all(LaunchTemplateId, LaunchTemplateName, default_config()). + +-spec describe_launch_template_versions_all(string() | none, string() | none, aws_config()) -> ok_error([proplist()]). +describe_launch_template_versions_all(LaunchTemplateId, LaunchTemplateName, Config) + when is_list(LaunchTemplateId) orelse LaunchTemplateId =:= none, + is_list(LaunchTemplateName) orelse LaunchTemplateName =:= none, + is_record(Config, aws_config) -> + describe_launch_template_versions_all(LaunchTemplateId, LaunchTemplateName, Config, undefined, []). + +describe_launch_template_versions_all(LaunchTemplateId, LaunchTemplateName, Config, Token, Acc) -> + case describe_launch_template_versions(LaunchTemplateId, LaunchTemplateName, [], none, ?LAUNCH_TEMPLATES_MR_MAX, Token, Config) of + {ok, Results, undefined} -> {ok, Results ++ Acc}; + {ok, Results, Next} -> + describe_launch_template_versions_all(LaunchTemplateId, LaunchTemplateName, Config, Next, Results ++ Acc); + {error, _} = Error -> Error + end. + +extract_launch_template(Node) -> + [ + {created_by, get_text("createdBy", Node)}, + {create_time, erlcloud_xml:get_time("createTime", Node)}, + {default_version_number, get_integer("defaultVersionNumber", Node)}, + {launch_template_id, get_text("launchTemplateId", Node)}, + {launch_template_name, get_text("launchTemplateName", Node)}, + {tag_set, [extract_tag_item(Item) || Item <- xmerl_xpath:string("tagSet/item", Node, [])]} + ]. + +extract_launch_template_version(Node) -> + [ + {created_by, get_text("createdBy", Node)}, + {create_time, erlcloud_xml:get_time("createTime", Node)}, + {default_version, get_bool("defaultVersion", Node)}, + {launch_template_data, transform_item_list(Node, "launchTemplateData", fun extract_launch_template_data/1)}, + {launch_template_id, get_text("launchTemplateId", Node)}, + {launch_template_name, get_text("launchTemplateName", Node)}, + {version_description, get_text("versionDescription", Node)}, + {version_number, get_integer("versionNumber", Node)} + ]. + +extract_capacity_reservation_target(Node) -> + [ + {capacity_reservation_id, get_text("capacityReservationId", Node)}, + {capacity_reservation_resource_group_arn, get_text("capacityReservationResourceGroupArn", Node)} + ]. + +extract_capacity_reservation_spec(Node) -> + [ + {capacity_reservation_preference, get_text("capacityReservationPreference", Node)}, + {capacity_reservation_target, transform_item_list(Node, "capacityReservationTarget", fun extract_capacity_reservation_target/1)} + ]. + +extract_elastic_gpu_specification(Node) -> + [{type, get_text("type", Node)}]. + +extract_elastic_inference_accelerator(Node) -> + [ + {count, get_integer("count", Node)}, + {type, get_text("type", Node)} + ]. + +extract_spot_market_options(Node) -> + [ + {block_duration_minutes, get_integer("blockDurationMinutes", Node)}, + {instance_interruption_behaviour, get_text("instanceInterruptionBehavior", Node)}, + {max_price, get_text("maxPrice", Node)}, + {spot_instance_type, get_text("spotInstanceType", Node)}, + {valid_until, erlcloud_xml:get_time("validUntil", Node)} + ]. + +extract_instance_market_options(Node) -> + [ + {market_type, get_text("marketType", Node)}, + {spot_options, transform_item_list(Node, "spotOptions", fun extract_spot_market_options/1)} + ]. + +extract_license_configuration(Node) -> + [{license_configuration_arn, get_text("licenseConfigurationArn", Node)}]. + +extract_maintenance_options(Node) -> + [{auto_recovery, get_text("autoRecovery", Node)}]. + +extract_metadata_options(Node) -> + [ + {http_endpoint, get_text("httpEndpoint", Node)}, + {http_protocol_ipv6, get_text("httpProtocolIpv6", Node)}, + {http_put_response_hop_limit, get_integer("httpPutResponseHopLimit", Node)}, + {http_tokens, get_text("httpTokens", Node)}, + {instance_metadata_tags, get_text("instanceMetadataTags", Node)}, + {state, get_text("state", Node)} + ]. + +extract_launch_template_placement(Node) -> + [ + {affinity, get_text("affinity", Node)}, + {availability_zone, get_text("availabilityZone", Node)}, + {group_name, get_text("groupName", Node)}, + {host_id, get_text("hostId", Node)}, + {host_resource_group_arn, get_text("hostResourceGroupArn", Node)}, + {partition_number, get_integer("partitionNumber", Node)}, + {spread_domain, get_text("spreadDomain", Node)}, + {tenancy, get_text("tenancy", Node)} + ]. + +extract_private_dns_name_options(Node) -> + [ + {enable_resource_name_dns_aaaa_record, get_bool("enableResourceNameDnsAAAARecord", Node)}, + {enable_resource_name_dns_a_record, get_bool("enableResourceNameDnsARecord", Node)}, + {hostname_type, get_text("hostnameType", Node)} + ]. + +extract_tag_specification(Node) -> + [ + {resource_type, get_text("resourceType", Node)}, + {tag_set, transform_item_list(Node, "tagSet/item", fun extract_tag_item/1)} + ]. + +extract_cpu_options(Node) -> + [ + {core_count, get_integer("coreCount", Node)}, + {threads_per_core, get_integer("threadsPerCore", Node)} + ]. + +extract_iam_instance_profile(Node) -> + [ + {arn, get_text("arn", Node)}, + {id, get_text("id", Node)} + ]. + +extract_credit_specification(Node) -> + [ + {cpu_credits, get_integer("cpuCredits", Node)} + ]. + +extract_range(Node, FnConvert) -> + [ + {max, FnConvert("max", Node)}, + {min, FnConvert("min", Node)} + ]. + +extract_int_range(Node) -> extract_range(Node, fun erlcloud_xml:get_integer/2). +extract_float_range(Node) -> extract_range(Node, fun erlcloud_xml:get_float/2). + +extract_instance_requirements(Node) -> + [ + {accelerator_count, transform_item_list(Node, "acceleratorCount", fun extract_int_range/1)}, + {accelerator_manufacturer_set, transform_item_list(Node, "acceleratorManufacturerSet/item", fun erlcloud_xml:get_text/1)}, + {accelerator_name_set, transform_item_list(Node, "acceleratorNameSet/item", fun erlcloud_xml:get_text/1)}, + {accelerator_total_memory_mib, transform_item_list(Node, "acceleratorTotalMemoryMiB", fun extract_int_range/1)}, + {accelerator_type_set, transform_item_list(Node, "acceleratorTypeSet/item", fun erlcloud_xml:get_text/1)}, + {bare_metal, get_text("bareMetal", Node)}, + {baseline_ebs_bandwidth_mbps, transform_item_list(Node, "baselineEbsBandwidthMbps", fun extract_int_range/1)}, + {burstable_performance, get_text("burstablePerformance", Node)}, + {cpu_manufacturer_set, transform_item_list(Node, "cpuManufacturerSet/item", fun erlcloud_xml:get_text/1)}, + {excluded_instance_type_set, transform_item_list(Node, "excludedInstanceTypeSet/item", fun erlcloud_xml:get_text/1)}, + {instance_generation_set, transform_item_list(Node, "instanceGenerationSet/item", fun erlcloud_xml:get_text/1)}, + {local_storage, get_text("localStorage", Node)}, + {local_storage_type_set, transform_item_list(Node, "localStorageTypeSet/item", fun erlcloud_xml:get_text/1)}, + {memory_gib_per_vcpu, transform_item_list(Node, "memoryGiBPerVCpu", fun extract_float_range/1)}, + {memory_mib, transform_item_list(Node, "memoryMiB", fun extract_int_range/1)}, + {network_interface_count, transform_item_list(Node, "networkInterfaceCount", fun extract_int_range/1)}, + {on_demand_max_price_percentage_over_lowest_price, get_integer("onDemandMaxPricePercentageOverLowestPrice", Node)}, + {require_hibernate_support, get_bool("requireHibernateSupport", Node)}, + {spot_max_price_percentage_over_lowest_price, get_integer("spotMaxPricePercentageOverLowestPrice", Node)}, + {total_local_storage_gb, transform_item_list(Node, "totalLocalStorageGB", fun extract_float_range/1)}, + {vcpu_count, transform_item_list(Node, "vCpuCount", fun extract_int_range/1)} + ]. + +extract_network_interface_specification(Node) -> + [ + {associate_carrier_ip_address, get_bool("associateCarrierIpAddress", Node)}, + {associate_public_ip_address, get_bool("associatePublicIpAddress", Node)}, + {delete_on_termination, get_bool("deleteOnTermination", Node)}, + {description, get_text("description", Node)}, + {device_index, get_integer("deviceIndex", Node)}, + {group_set, transform_item_list(Node, "groupSet", fun(N) -> {group_id, get_text("groupId", N)} end)}, + {interface_type, get_text("interfaceType", Node)}, + {ipv4_prefix_count, get_integer("ipv4PrefixCount", Node)}, + {ipv4_prefix_set, transform_item_list(Node, "ipv4PrefixSet/item", fun(N) -> {ipv4_prefix, get_text("ipv4Prefix", N)} end)}, + {ipv6_address_count, get_integer("ipv6AddressCount", Node)}, + {ipv6_addresses_set, transform_item_list(Node, "ipv6AddressesSet/item", fun(N) -> {ipv6_address, get_text("ipv6Address", N)} end)}, + {ipv6_prefix_count, get_integer("ipv6PrefixCount", Node)}, + {ipv6_prefix_set, transform_item_list(Node, "ipv6AddressesSet/item", fun(N) -> {ipv6_prefix, get_text("ipv6Prefix", N)} end)}, + {network_card_index, get_integer("networkCardIndex", Node)}, + {network_interface_id, get_text("networkInterfaceId", Node)}, + {private_ip_address, get_text("privateIpAddress", Node)}, + {private_ip_addresses_set, transform_item_list(Node, "privateIpAddressesSet/item", fun extract_private_ip_address/1)}, + {secondary_private_ip_address_count, get_integer("secondaryPrivateIpAddressCount", Node)}, + {subnet_id, get_text("subnetId", Node)} + ]. + +transform_item_list(ParentNode, Tag, FncTransform) -> + [FncTransform(Item) || Item <- xmerl_xpath:string(Tag, ParentNode)]. + +extract_launch_template_data(Node) -> + [ + {block_device_mapping_set, transform_item_list(Node, "blockDeviceMappingSet/item", fun extract_block_device_mapping/1)}, + {capacity_reservation_specification, transform_item_list(Node, "capacityReservationSpecification", fun extract_capacity_reservation_spec/1)}, + {cpu_options, transform_item_list(Node, "cpuOptions", fun extract_cpu_options/1)}, + {credit_specification, transform_item_list(Node, "creditSpecification", fun extract_credit_specification/1)}, + {disable_api_stop, get_bool("disableApiStop", Node)}, + {disable_api_termination, get_bool("disableApiTermination", Node)}, + {ebs_optimized, get_bool("ebsOptimized", Node)}, + {elastic_gpu_specification_set, transform_item_list(Node, "elasticGpuSpecificationSet/item", fun extract_elastic_gpu_specification/1)}, + {elastic_inference_accelerator_set, transform_item_list(Node, "elasticInferenceAcceleratorSet/item", fun extract_elastic_inference_accelerator/1)}, + {enclave_options, [{enabled, get_bool("enclaveOptions/enabled", Node)}]}, + {hibernation_options, [{configured, get_bool("hibernationOptions/configured", Node)}]}, + {iam_instance_profile, transform_item_list(Node, "iamInstanceProfile", fun extract_iam_instance_profile/1)}, + {image_id, get_text("imageId", Node)}, + {instance_initiated_shutdown_behavior, get_text("instanceInitiatedShutdownBehavior", Node)}, + {instance_market_options, transform_item_list(Node, "instanceMarketOptions", fun extract_instance_market_options/1)}, + {instance_requirements, transform_item_list(Node, "instanceRequirements", fun extract_instance_requirements/1)}, + {instance_type, get_text("instanceType", Node)}, + {kernel_id, get_text("kernelId", Node)}, + {key_name, get_text("keyName", Node)}, + {license_set, transform_item_list(Node, "licenseSet/item", fun extract_license_configuration/1)}, + {maintenance_options, transform_item_list(Node, "maintenanceOptions", fun extract_maintenance_options/1)}, + {metadata_options, transform_item_list(Node, "metadataOptions", fun extract_metadata_options/1)}, + {monitoring, [{enabled, get_bool("monitoring/enabled", Node)}]}, + {network_interface_set, transform_item_list(Node, "networkInterfaceSet/item", fun extract_network_interface_specification/1)}, + {placement, transform_item_list(Node, "placement", fun extract_launch_template_placement/1)}, + {private_dns_name_options, transform_item_list(Node, "privateDnsNameOptions", fun extract_private_dns_name_options/1)}, + {ram_disk_id, get_text("ramDiskId", Node)}, + {security_group_id_set, [get_text(Item) || Item <- xmerl_xpath:string("securityGroupIdSet/item", Node)]}, + {security_group_set, [get_text(Item) || Item <- xmerl_xpath:string("securityGroupSet/item", Node)]}, + {tag_specification_set, transform_item_list(Node, "tagSpecificationSet/item", fun extract_tag_specification/1)}, + {user_data, get_text("userData", Node)} + ]. \ No newline at end of file diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index a79d099f8..3a0e64e02 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -60,7 +60,9 @@ describe_test_() -> fun delete_flow_logs_input_tests/1, fun delete_flow_logs_output_tests/1, fun describe_flow_logs_input_tests/1, - fun describe_flow_logs_output_tests/1 + fun describe_flow_logs_output_tests/1, + fun describe_launch_template_versions_input_tests/1, + fun describe_launch_template_versions_output_tests/1 ]}. start() -> @@ -1316,6 +1318,316 @@ describe_reserved_instances_offerings_boundaries_test_() -> ?_assertException(error, function_clause, erlcloud_ec2:describe_reserved_instances_offerings([], 1001, undefined)) ]. +describe_launch_templates_test() -> + XML = " + 1afa6e44-eb38-4229-8db6-d5eaexample + + + 2017-10-31T11:38:52.000Z + arn:aws:iam::123456789012:root + 1 + 1 + lt-0a20c965061f64abc + MyLaunchTemplate + + + ", + XMERL = {ok, element(1, xmerl_scan:string(XML))}, + ExpectedResult = + {ok, + [[ + {created_by,"arn:aws:iam::123456789012:root"}, + {create_time,{{2017,10,31},{11,38,52}}}, + {default_version_number,1}, + {launch_template_id,"lt-0a20c965061f64abc"}, + {launch_template_name,"MyLaunchTemplate"}, + {tag_set,[]} + ]] + }, + meck:new(erlcloud_aws, [passthrough]), + meck:expect(erlcloud_aws, aws_request_xml4, + fun(_,_,_,_,_,_,_,_) -> + XMERL + end), + Result = erlcloud_ec2:describe_launch_templates(), + meck:unload(erlcloud_aws), + ?assertEqual(ExpectedResult, Result). + +describe_launch_template_versions_input_tests(_) -> + Tests = + [?_ec2_test({ + "This example describes a set of launch template versions for template with matching ID", + ?_f(erlcloud_ec2:describe_launch_template_versions("lt-0a20c965061f64abc")), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateId", "lt-0a20c965061f64abc"}]}), + ?_ec2_test({ + "This example describes a set of launch template versions for template with matching name", + ?_f(erlcloud_ec2:describe_launch_template_versions(none, "Test-LT-foo", [])), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateName", "Test-LT-foo"}]}), + ?_ec2_test({ + "This example describes launch template versions with matching version(s)", + ?_f(erlcloud_ec2:describe_launch_template_versions("lt-0f1b5e0ff43b7f2ad", none, [{launch_template_version, ["1"]}])), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateId", "lt-0f1b5e0ff43b7f2ad"}, + {"LaunchTemplateVersion.1", "1"}]}), + ?_ec2_test({ + "This example describes launch template versions with minimum version number", + ?_f(erlcloud_ec2:describe_launch_template_versions("lt-0a20c965061f64abc", none, [{min_version, 1}])), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateId", "lt-0a20c965061f64abc"}, + {"MinVersion", "1"}]}), + ?_ec2_test({ + "This example describes launch template versions with maximum version number", + ?_f(erlcloud_ec2:describe_launch_template_versions("lt-0a20c965061f64abc", none, [{max_version, 10}])), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateId", "lt-0a20c965061f64abc"}, + {"MaxVersion", "10"}]}), + ?_ec2_test({ + "This example describes launch template versions with max results count configured", + ?_f(erlcloud_ec2:describe_launch_template_versions("lt-0a20c965061f64abc", none, [], none, 10, undefined)), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateId", "lt-0a20c965061f64abc"}, + {"MaxResults", "10"}]}), + ?_ec2_test({ + "This example describes retrieval of subsequent launch template version result page", + ?_f(erlcloud_ec2:describe_launch_template_versions("lt-0a20c965061f64abc", none, [], none, 10, "next-token")), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateId", "lt-0a20c965061f64abc"}, + {"NextToken", "next-token"}, + {"MaxResults", "10"}]}), + ?_ec2_test({ + "This example describes launch template versions with max results count configured selected by template name", + ?_f(erlcloud_ec2:describe_launch_template_versions(none, "SomeNameHere", [], none, 10, undefined)), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateName", "SomeNameHere"}, + {"MaxResults", "10"}]}), + ?_ec2_test({ + "This example describes retrieval of subsequent launch template version result page selected by template name", + ?_f(erlcloud_ec2:describe_launch_template_versions(none, "SomeNameHere", [], none, 10, "next-token")), + [{"Action", "DescribeLaunchTemplateVersions"}, + {"LaunchTemplateName", "SomeNameHere"}, + {"NextToken", "next-token"}, + {"MaxResults", "10"}]}) + ], + Response = " + + 65cadec1-b364-4354-8ca8-4176dexample + + + 2017-10-31T11:38:52.000Z + arn:aws:iam::123456789012:root + true + + ami-8c1be5f6 + t2.micro + + lt-0a20c965061f64abc + MyLaunchTemplate + FirstVersion + 1 + + + 2017-10-31T11:52:03.000Z + arn:aws:iam::123456789012:root + false + + ami-12345678 + + lt-0a20c965061f64abc + MyLaunchTemplate + AMIOnlyv1 + 2 + + + 2017-10-31T11:55:15.000Z + arn:aws:iam::123456789012:root + false + + ami-aabbccdd + + lt-0a20c965061f64abc + MyLaunchTemplate + AMIOnlyv2 + 3 + + + + ", + input_tests(Response, Tests). + +describe_launch_template_versions_output_tests(_) -> + Tests = + [?_ec2_test( + {"This example describes all launch template versions for given template", + " + + c68f69e1-48fd-42c9-83cf-6b36cb07475b + + + 2022-10-21T08:51:54.000Z + arn:aws:sts::123483894321:assumed-role/centralized-users/john.doe + true + + + 1 + 2 + + false + false + true + + true + + + arn:aws:iam::123483894321:instance-profile/ecsInstanceRole + + ami-5934df24 + stop + c5.large + + default + + + enabled + optional + + + true + + + + 0 + eni-060ceeb735813d39d + + + + default + + + true + ip-name + + + + instance + + + tag1 + 134274125 + + + + + volume + + + tag1 + 134274125 + + + + + + lt-08ccaba0746110123 + nb-ec-test-1 + Test template foo + 1 + + + ", + {ok, [[{created_by, "arn:aws:sts::123483894321:assumed-role/centralized-users/john.doe"}, + {create_time,{{2022,10,21},{8,51,54}}}, + {default_version,true}, + {launch_template_data, + [[{block_device_mapping_set,[]}, + {capacity_reservation_specification,[]}, + {cpu_options, + [[{core_count,1},{threads_per_core,2}]]}, + {credit_specification,[]}, + {disable_api_stop,false}, + {disable_api_termination,false}, + {ebs_optimized,true}, + {elastic_gpu_specification_set,[]}, + {elastic_inference_accelerator_set,[]}, + {enclave_options,[{enabled,false}]}, + {hibernation_options,[{configured,true}]}, + {iam_instance_profile, + [[{arn, + "arn:aws:iam::123483894321:instance-profile/ecsInstanceRole"}, + {id,[]}]]}, + {image_id,"ami-5934df24"}, + {instance_initiated_shutdown_behavior,"stop"}, + {instance_market_options,[]}, + {instance_requirements,[]}, + {instance_type,"c5.large"}, + {kernel_id,[]}, + {key_name,[]}, + {license_set,[]}, + {maintenance_options, + [[{auto_recovery,"default"}]]}, + {metadata_options, + [[{http_endpoint,"enabled"}, + {http_protocol_ipv6,[]}, + {http_put_response_hop_limit,0}, + {http_tokens,"optional"}, + {instance_metadata_tags,[]}, + {state,[]}]]}, + {monitoring,[{enabled,true}]}, + {network_interface_set, + [[{associate_carrier_ip_address,false}, + {associate_public_ip_address,false}, + {delete_on_termination,false}, + {description,[]}, + {device_index,0}, + {group_set,[]}, + {interface_type,[]}, + {ipv4_prefix_count,0}, + {ipv4_prefix_set,[]}, + {ipv6_address_count,0}, + {ipv6_addresses_set,[]}, + {ipv6_prefix_count,0}, + {ipv6_prefix_set,[]}, + {network_card_index,0}, + {network_interface_id, "eni-060ceeb735813d39d"}, + {private_ip_address,[]}, + {private_ip_addresses_set,[]}, + {secondary_private_ip_address_count,0}, + {subnet_id,[]}]]}, + {placement, + [[{affinity,[]}, + {availability_zone,[]}, + {group_name,[]}, + {host_id,[]}, + {host_resource_group_arn,[]}, + {partition_number,0}, + {spread_domain,[]}, + {tenancy,"default"}]]}, + {private_dns_name_options, + [[{enable_resource_name_dns_aaaa_record, false}, + {enable_resource_name_dns_a_record,true}, + {hostname_type,"ip-name"}]]}, + {ram_disk_id,[]}, + {security_group_id_set,[]}, + {security_group_set,[]}, + {tag_specification_set, + [[{resource_type,"instance"}, + {tag_set, + [[{key,"tag1"},{value,"134274125"}]]}], + [{resource_type,"volume"}, + {tag_set, + [[{key,"tag1"}, + {value,"134274125"}]]}]]}, + {user_data,[]}]]}, + {launch_template_id,"lt-08ccaba0746110123"}, + {launch_template_name,"nb-ec-test-1"}, + {version_description, + "Test template foo"}, + {version_number,1}]]}} + ) + ], + output_tests(?_f(erlcloud_ec2:describe_launch_template_versions("lt-08ccaba0746110123")), Tests). + generate_one_instance(N) -> " r-69 From 42543592aadba1667fec46ff7b5350a89ea02ecc Mon Sep 17 00:00:00 2001 From: hardi171 <116284460+hardi171@users.noreply.github.com> Date: Tue, 25 Oct 2022 11:08:18 +0100 Subject: [PATCH 253/310] Nat gateway id not reflect in route table issue solved. (#733) --- src/erlcloud_ec2.erl | 9 +-- test/erlcloud_ec2_tests.erl | 106 ++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index d84c59658..8dbf6833b 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -1962,21 +1962,21 @@ extract_reserved_instances_offering(Node) -> {product_description, get_text("productDescription", Node)} ]. --spec describe_route_tables() -> ok_error(proplist()). +-spec describe_route_tables() -> ok_error([proplist()]). describe_route_tables() -> describe_route_tables([], none, default_config()). --spec describe_route_tables(filter_list() | none | aws_config()) -> ok_error(proplist()). +-spec describe_route_tables(filter_list() | none | aws_config()) -> ok_error([proplist()]). describe_route_tables(Config) when is_record(Config, aws_config) -> describe_route_tables([], none, Config); describe_route_tables(Filter) -> describe_route_tables([], Filter, default_config()). --spec describe_route_tables(filter_list() | none, aws_config()) -> ok_error(proplist()). +-spec describe_route_tables(filter_list() | none, aws_config()) -> ok_error([proplist()]). describe_route_tables(Filter, Config) -> describe_route_tables([], Filter, Config). --spec describe_route_tables([string()], filter_list() | none, aws_config()) -> ok_error(proplist()). +-spec describe_route_tables([string()], filter_list() | none, aws_config()) -> ok_error([proplist()]). describe_route_tables(RouteTableIds, Filter, Config) -> Params = erlcloud_aws:param_list(RouteTableIds, "RouteTableId") ++ list_to_ec2_filter(Filter), case ec2_query(Config, "DescribeRouteTables", Params, ?NEW_API_VERSION) of @@ -2005,6 +2005,7 @@ extract_route_set(Node) -> [ {destination_cidr_block, get_text("destinationCidrBlock", Node)}, {gateway_id, get_text("gatewayId", Node)}, + {nat_gateway_id, get_text("natGatewayId", Node)}, {instance_id, get_text("instanceId", Node)}, {vpc_peering_conn_id, get_text("vpcPeeringConnectionId", Node)}, {network_interface_id, get_text("networkInterfaceId", Node)}, diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 3a0e64e02..20997d014 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -38,6 +38,7 @@ describe_test_() -> fun start/0, fun stop/1, [ + fun describe_route_tables_tests/0, fun describe_vpcs_tests/1, fun describe_regions_input_tests/1, fun describe_regions_output_tests/1, @@ -925,6 +926,111 @@ describe_account_attributes_test() -> Result = erlcloud_ec2:describe_account_attributes(), meck:unload(erlcloud_aws), ?assertEqual(ExpectedResult, Result). + +describe_route_tables_tests() -> + XML = " + 03c84dd4-bc36-48aa-8ac3-1d1fbaec96b5 + + + rtb-0b70ded185be2a2e4 + vpc-012ff464df6bbf762 + 352283894008 + + + 10.0.0.0/26 + local + active + CreateRouteTable + + + 0.0.0.0/8 + nat-06e45944bb42d3ba2 + active + CreateRoute + + + 0.0.0.0/0 + vgw-09fea3ed190b9ea1e + active + CreateRoute + + + + + rtbassoc-123 + rtb-567 + subnet-0bdf348a667f86262 +
    false
    + + associated + +     + + rtbassoc-789 + rtb-345 +
    true
    + + associated + +     +     + + + + Name + PublicRT + + +     +     +    ", + XMERL = {ok, element(1, xmerl_scan:string(XML))}, + ExpectedResult = + {ok,[[{route_table_id,"rtb-0b70ded185be2a2e4"}, + {vpc_id,"vpc-012ff464df6bbf762"}, + {route_set, + [[{destination_cidr_block,"10.0.0.0/26"}, + {gateway_id,"local"}, + {nat_gateway_id,[]}, + {instance_id,[]}, + {vpc_peering_conn_id,[]}, + {network_interface_id,[]}, + {state,"active"}, + {origin,"CreateRouteTable"}], + [{destination_cidr_block,"0.0.0.0/8"}, + {gateway_id,[]}, + {nat_gateway_id,"nat-06e45944bb42d3ba2"}, + {instance_id,[]}, + {vpc_peering_conn_id,[]}, + {network_interface_id,[]}, + {state,"active"}, + {origin,"CreateRoute"}], + [{destination_cidr_block,"0.0.0.0/0"}, + {gateway_id,"vgw-09fea3ed190b9ea1e"}, + {nat_gateway_id,[]}, + {instance_id,[]}, + {vpc_peering_conn_id,[]}, + {network_interface_id,[]}, + {state,"active"}, + {origin,"CreateRoute"}]]}, + {association_set, + [[{route_table_association_id,"rtbassoc-123"}, + {route_table_id,"rtb-567"}, + {main,"false"}, + {subnet_id,"subnet-0bdf348a667f86262"}], + [{route_table_association_id,"rtbassoc-789"}, + {route_table_id,"rtb-345"}, + {main,"true"}, + {subnet_id,[]}]]}, + {tag_set,[[{key,"Name"},{value,"PublicRT"}]]}]]}, + meck:new(erlcloud_aws, [passthrough]), + meck:expect(erlcloud_aws, aws_request_xml4, + fun(_,_,_,_,_,_,_,_) -> + XMERL + end), + Result = erlcloud_ec2:describe_route_tables(), + meck:unload(erlcloud_aws), + ?assertEqual(ExpectedResult, Result). describe_nat_gateways_test() -> XML = " From d0d888d04646eddae38f876e1394f1ace24f5e56 Mon Sep 17 00:00:00 2001 From: Neil Bleasdale Date: Tue, 1 Nov 2022 09:14:25 +0000 Subject: [PATCH 254/310] Add funcs to describe all launch templates with additional args Allow retrieval of all pages of launch templates described by additional arguments; launch template ID list and filter options. --- src/erlcloud_ec2.erl | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index 8dbf6833b..6403e1b65 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -201,6 +201,7 @@ describe_launch_templates/4, describe_launch_templates/5, describe_launch_templates_all/0, describe_launch_templates_all/1, + describe_launch_templates_all/2, describe_launch_templates_all/3, describe_launch_template_versions/1, describe_launch_template_versions/3, describe_launch_template_versions/4, describe_launch_template_versions/5, @@ -3991,16 +3992,31 @@ describe_launch_templates(LaunchTemplateIds, Filter, MaxResults, NextToken, Conf describe_launch_templates_all() -> describe_launch_templates_all(default_config()). --spec describe_launch_templates_all(aws_config()) -> ok_error([proplist()]). +-spec describe_launch_templates_all(aws_config() | launch_template_ids()) -> ok_error([proplist()]). +describe_launch_templates_all(Ids) + when is_list(Ids) -> + describe_launch_templates_all(Ids, none, default_config(), undefined, []); describe_launch_templates_all(Config) when is_record(Config, aws_config) -> - describe_launch_templates_all(Config, undefined, []). + describe_launch_templates_all([], none, Config, undefined, []). -describe_launch_templates_all(Config, Token, Acc) - when is_record(Config, aws_config) -> - case describe_launch_templates([], none, ?LAUNCH_TEMPLATES_MR_MAX, Token, Config) of +-spec describe_launch_templates_all(launch_template_ids(), filter_list()) -> ok_error([proplist()]). +describe_launch_templates_all(LaunchTemplateIds, FilterOpts) + when is_list(LaunchTemplateIds), is_list(FilterOpts) orelse FilterOpts =:= none -> + describe_launch_templates_all(LaunchTemplateIds, FilterOpts, default_config()). + +-spec describe_launch_templates_all(launch_template_ids(), filter_list(), aws_config()) -> ok_error([proplist()]). +describe_launch_templates_all(LaunchTemplateIds, FilterOpts, Config) + when is_list(LaunchTemplateIds), is_list(FilterOpts) orelse FilterOpts =:= none, + is_record(Config, aws_config) -> + describe_launch_templates_all(LaunchTemplateIds, FilterOpts, Config, undefined, []). + +describe_launch_templates_all(LaunchTemplateIds, FilterOpts, Config, Token, Acc) + when is_list(LaunchTemplateIds), is_list(FilterOpts) orelse FilterOpts =:= none, + is_list(Token) orelse Token =:= undefined, is_record(Config, aws_config) -> + case describe_launch_templates(LaunchTemplateIds, FilterOpts, ?LAUNCH_TEMPLATES_MR_MAX, Token, Config) of {ok, Results, undefined} -> {ok, Results ++ Acc}; - {ok, Results, Next} -> describe_launch_templates_all(Config, Next, Results ++ Acc); + {ok, Results, Next} -> describe_launch_templates_all(LaunchTemplateIds, FilterOpts, Config, Next, Results ++ Acc); {error, _} = Error -> Error end. From 661e8d0a90e25864913d4fd2b089af65460eaabe Mon Sep 17 00:00:00 2001 From: Neil Bleasdale Date: Tue, 8 Nov 2022 18:01:42 +0000 Subject: [PATCH 255/310] Add associated Launch Template to ASG description --- include/erlcloud_as.hrl | 8 ++++ src/erlcloud_as.erl | 8 ++++ src/erlcloud_autoscaling.erl | 9 ++++ test/erlcloud_as_tests.erl | 49 +++++++++++++++++++- test/erlcloud_autoscaling_tests.erl | 69 ++++++++++++++++++++++++++++- 5 files changed, 141 insertions(+), 2 deletions(-) diff --git a/include/erlcloud_as.hrl b/include/erlcloud_as.hrl index 47a78a50c..e2c0f83e9 100644 --- a/include/erlcloud_as.hrl +++ b/include/erlcloud_as.hrl @@ -32,12 +32,20 @@ min_size :: undefined | integer(), max_size :: undefined | integer(), launch_configuration_name :: undefined | string(), + launch_template :: undefined | aws_launch_template_spec(), vpc_zone_id :: undefined | list(string()), instances :: undefined | list(aws_autoscaling_instance()), status :: undefined | string() }). -type(aws_autoscaling_group() :: #aws_autoscaling_group{}). +-record(aws_launch_template_spec, { + id :: string(), + name :: string(), + version :: string() +}). +-type(aws_launch_template_spec() :: #aws_launch_template_spec{}). + -record(aws_launch_config, { name :: string(), image_id :: string(), diff --git a/src/erlcloud_as.erl b/src/erlcloud_as.erl index 3ac8886ef..846e0aa1d 100644 --- a/src/erlcloud_as.erl +++ b/src/erlcloud_as.erl @@ -170,6 +170,7 @@ extract_group(G) -> min_size = erlcloud_xml:get_integer("MinSize", G), max_size = erlcloud_xml:get_integer("MaxSize", G), launch_configuration_name = get_text("LaunchConfigurationName", G), + launch_template = extract_launch_template_spec(xmerl_xpath:string("LaunchTemplate", G)), vpc_zone_id = [ erlcloud_xml:get_text(Zid) || Zid <- xmerl_xpath:string("VPCZoneIdentifier", G)], status = get_text("Status", G) }. @@ -177,6 +178,13 @@ extract_tags_from_group(G) -> [{erlcloud_xml:get_text("Key", T), erlcloud_xml:get_text("Value", T)} || T <- xmerl_xpath:string("Tags/member", G)]. +extract_launch_template_spec([]) -> undefined; +extract_launch_template_spec([L]) -> + #aws_launch_template_spec{ + id = get_text("LaunchTemplateId", L), + name = get_text("LaunchTemplateName", L), + version = get_text("Version", L) + }. %% -------------------------------------------------------------------- %% @doc set_desired_capacity(GroupName, Capacity, false, default_config()) %% @end diff --git a/src/erlcloud_autoscaling.erl b/src/erlcloud_autoscaling.erl index 6c1b17101..425fed8db 100644 --- a/src/erlcloud_autoscaling.erl +++ b/src/erlcloud_autoscaling.erl @@ -131,6 +131,7 @@ extract_autoscaling_group(Item) -> {autoscaling_group_name, get_text("AutoScalingGroupName", Item)}, {autoscaling_group_arn, get_text("AutoScalingGroupARN", Item)}, {launch_configuration_name, get_text("LaunchConfigurationName", Item)}, + {launch_template, extract_launch_template(xmerl_xpath:string("LaunchTemplate", Item))}, {min_size, get_integer("MinSize", Item)}, {max_size, get_integer("MaxSize", Item)}, {create_time, erlcloud_xml:get_time("CreatedTime", Item)}, @@ -180,6 +181,14 @@ extract_launch_configuration(Item) -> {security_groups, [get_text(L) || L <- xmerl_xpath:string("SecurityGroups/member", Item)]} ]. +extract_launch_template([]) -> []; +extract_launch_template([Item]) -> + [ + {launch_template_id, get_text("LaunchTemplateId", Item)}, + {launch_template_name, get_text("LaunchTemplateName", Item)}, + {launch_template_version, get_text("Version", Item)} + ]. + autoscaling_query(Config, Action, Params) -> autoscaling_query(Config, Action, Params, ?API_VERSION). diff --git a/test/erlcloud_as_tests.erl b/test/erlcloud_as_tests.erl index f6580d4c1..eeb202f5a 100644 --- a/test/erlcloud_as_tests.erl +++ b/test/erlcloud_as_tests.erl @@ -151,6 +151,37 @@ mocked_groups() -> 10 + + + + my-test-asg-lbs + ELB + 2013-05-06T17:47:15.107Z + + + lt-036fea5ec210c3294 + asg-test-launch-template-spec-1 + 2 + + + 2 + + us-east-1b + us-east-1a + + + my-test-asg-loadbalancer + + 2 + + 120 + 300 + arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/my-test-asg-lbs + + Default + + 10 + @@ -171,7 +202,23 @@ expected_groups() -> vpc_zone_id = [""], instances = [], status = [] - + }, #aws_autoscaling_group{ + group_name = "my-test-asg-lbs", + availability_zones = ["us-east-1b", "us-east-1a"], + load_balancer_names = ["my-test-asg-loadbalancer"], + tags = [], + desired_capacity = 2, + min_size = 2, + max_size = 10, + launch_configuration_name = [], + launch_template = #aws_launch_template_spec{ + id = "lt-036fea5ec210c3294", + name = "asg-test-launch-template-spec-1", + version = "2" + }, + vpc_zone_id = [""], + instances = [], + status = [] }]. mocked_instances() -> diff --git a/test/erlcloud_autoscaling_tests.erl b/test/erlcloud_autoscaling_tests.erl index 1e5846110..6179e5e0a 100644 --- a/test/erlcloud_autoscaling_tests.erl +++ b/test/erlcloud_autoscaling_tests.erl @@ -306,6 +306,7 @@ describe_autoscaling_groups_all_output_tests(_) -> ", " + blah @@ -339,11 +340,52 @@ describe_autoscaling_groups_all_output_tests(_) -> 0f02a07d-b677-11e2-9eb0-dd50EXAMPLE +", " + + + + + + + my-test-asg-lbs + ELB + 2013-05-06T17:47:15.107Z + + + lt-036fea5ec210c3294 + asg-test-launch-template-spec-1 + 2 + + + 2 + + us-east-1b + us-east-1a + + + my-test-asg-loadbalancer + + 2 + + 120 + 300 + arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/my-test-asg-lbs + + Default + + 10 + + + + + 0f02a07d-b677-11e2-9eb0-dd50EXAMPLE + "], {ok, [[ {autoscaling_group_name, "my-test-asg-lbs"}, {autoscaling_group_arn, "arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/my-test-asg-lbs"}, {launch_configuration_name, "my-test-lc1"}, + {launch_template, []}, {min_size, 2}, {max_size, 10}, {create_time,{{2013,5,6},{17,47,15}}}, @@ -360,6 +402,7 @@ describe_autoscaling_groups_all_output_tests(_) -> {autoscaling_group_name, "my-test-asg-lbs"}, {autoscaling_group_arn, "arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/my-test-asg-lbs"}, {launch_configuration_name, "my-test-lc2"}, + {launch_template, []}, {min_size, 2}, {max_size, 10}, {create_time,{{2013,5,6},{17,47,15}}}, @@ -376,6 +419,7 @@ describe_autoscaling_groups_all_output_tests(_) -> {autoscaling_group_name, "my-test-asg-lbs"}, {autoscaling_group_arn, "arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/my-test-asg-lbs"}, {launch_configuration_name, "my-test-lc3"}, + {launch_template, []}, {min_size, 2}, {max_size, 10}, {create_time,{{2013,5,6},{17,47,15}}}, @@ -392,6 +436,7 @@ describe_autoscaling_groups_all_output_tests(_) -> {autoscaling_group_name, "my-test-asg-lbs"}, {autoscaling_group_arn, "arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/my-test-asg-lbs"}, {launch_configuration_name, "my-test-lc4"}, + {launch_template, []}, {min_size, 2}, {max_size, 10}, {create_time,{{2013,5,6},{17,47,15}}}, @@ -404,7 +449,28 @@ describe_autoscaling_groups_all_output_tests(_) -> {load_balancers,["my-test-asg-loadbalancer"]}, {instances,[]}, {tag_set,[]} - ]]}})], + ], [ + {autoscaling_group_name, "my-test-asg-lbs"}, + {autoscaling_group_arn, "arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/my-test-asg-lbs"}, + {launch_configuration_name, []}, + {launch_template, [ + {launch_template_id, "lt-036fea5ec210c3294"}, + {launch_template_name, "asg-test-launch-template-spec-1"}, + {launch_template_version, "2"} + ]}, + {min_size, 2}, + {max_size, 10}, + {create_time,{{2013,5,6},{17,47,15}}}, + {health_check_type, "ELB"}, + {desired_capacity,2}, + {placement_group,[]}, + {status,[]}, + {subnets, []}, + {availability_zones,["us-east-1b","us-east-1a"]}, + {load_balancers,["my-test-asg-loadbalancer"]}, + {instances,[]}, + {tag_set,[]} + ]]}})], %% Remaining AWS API examples return subsets of the same data output_tests_seq(?_f(erlcloud_autoscaling:describe_autoscaling_groups_all()), Tests). @@ -477,6 +543,7 @@ describe_autoscaling_groups_output_tests(_) -> {autoscaling_group_name, "my-test-asg-lbs"}, {autoscaling_group_arn, "arn:aws:autoscaling:us-east-1:803981987763:autoScalingGroup:ca861182-c8f9-4ca7-b1eb-cd35505f5ebb:autoScalingGroupName/my-test-asg-lbs"}, {launch_configuration_name, "my-test-lc"}, + {launch_template, []}, {min_size, 2}, {max_size, 10}, {create_time,{{2013,5,6},{17,47,15}}}, From 35522c888cd679be9491520cc7fe870dd8315780 Mon Sep 17 00:00:00 2001 From: neil-bleasdale <116366258+neil-bleasdale@users.noreply.github.com> Date: Mon, 14 Nov 2022 16:58:06 +0000 Subject: [PATCH 256/310] Fix: badarg when parsing cpu credits (Launch Template Vsn) (#738) --- src/erlcloud_ec2.erl | 2 +- test/erlcloud_ec2_tests.erl | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index 6403e1b65..a21ba75c5 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -4280,7 +4280,7 @@ extract_iam_instance_profile(Node) -> extract_credit_specification(Node) -> [ - {cpu_credits, get_integer("cpuCredits", Node)} + {cpu_credits, get_text("cpuCredits", Node)} ]. extract_range(Node, FnConvert) -> diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 20997d014..7b037f89b 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -1575,6 +1575,9 @@ describe_launch_template_versions_output_tests(_) -> arn:aws:sts::123483894321:assumed-role/centralized-users/john.doe true + + standard + 1 2 @@ -1650,7 +1653,8 @@ describe_launch_template_versions_output_tests(_) -> {capacity_reservation_specification,[]}, {cpu_options, [[{core_count,1},{threads_per_core,2}]]}, - {credit_specification,[]}, + {credit_specification,[ + [{cpu_credits, "standard"}]]}, {disable_api_stop,false}, {disable_api_termination,false}, {ebs_optimized,true}, From 854b175f462f740fdb00680ebd94fbb0238dd5e8 Mon Sep 17 00:00:00 2001 From: Aleksei Osin Date: Wed, 16 Nov 2022 16:47:19 +0000 Subject: [PATCH 257/310] Support access-analyzer service (#736) --- include/erlcloud_aws.hrl | 1 + src/erlcloud_access_analyzer.erl | 240 +++++++++++++++++++ src/erlcloud_aws.erl | 30 ++- test/erlcloud_access_analyzer_tests.erl | 299 ++++++++++++++++++++++++ 4 files changed, 563 insertions(+), 7 deletions(-) create mode 100644 src/erlcloud_access_analyzer.erl create mode 100644 test/erlcloud_access_analyzer_tests.erl diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 39cbe8072..572071abd 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -19,6 +19,7 @@ -type(hackney_client_options() :: #hackney_client_options{}). -record(aws_config, { + access_analyzer_host="access-analyzer.us-east-1.amazonaws.com"::string(), as_host="autoscaling.amazonaws.com"::string(), ec2_protocol="https"::string(), ec2_host="ec2.amazonaws.com"::string(), diff --git a/src/erlcloud_access_analyzer.erl b/src/erlcloud_access_analyzer.erl new file mode 100644 index 000000000..97094d003 --- /dev/null +++ b/src/erlcloud_access_analyzer.erl @@ -0,0 +1,240 @@ +-module(erlcloud_access_analyzer). + +-include("erlcloud.hrl"). +-include("erlcloud_aws.hrl"). + +%% API +-export([ + list_analyzers/1, + list_analyzers/2, + list_analyzers_all/1, + list_analyzers_all/2, + get_analyzer/2, + create_analyzer/2, + delete_analyzer/2 +]). + +-type param_name() :: binary() | string() | atom(). +-type param_value() :: binary() | string() | atom() | integer(). +-type params() :: [param_name() | {param_name(), param_value()}]. + +-type analyzer() :: proplist(). +-type analyzers() :: [analyzer()]. +-type token() :: binary(). + +-type create_analyzer_spec() :: proplist(). + +%% ----------------------------------------------------------------------------- +%% Exported functions +%% ----------------------------------------------------------------------------- + +-spec list_analyzers(AwsConfig) -> Result +when AwsConfig :: aws_config(), + Result :: {ok, analyzers()} | {ok, analyzers(), token()} | {error, term()}. +list_analyzers(AwsConfig) + when is_record(AwsConfig, aws_config) -> + list_analyzers(AwsConfig, _Params = []); +list_analyzers(Params) -> + AwsConfig = erlcloud_aws:default_config(), + list_analyzers(AwsConfig, Params). + +-spec list_analyzers(AwsConfig, Params) -> Result +when AwsConfig :: aws_config(), + Params :: params(), + Result :: {ok, analyzers()} | {ok, analyzers(), token()} | {error, term()}. +list_analyzers(AwsConfig, Params) -> + Path = ["analyzer"], + case request(AwsConfig, _Method = get, Path, Params) of + {ok, Response} -> + Analyzers = proplists:get_value(<<"analyzers">>, Response), + case proplists:get_value(<<"nextToken">>, Response) of + undefined -> + {ok, Analyzers}; + Token -> + {ok, Analyzers, Token} + end; + {error, Reason} -> + {error, Reason} + end. + +-spec list_analyzers_all(AwsConfig) -> Result +when AwsConfig :: aws_config(), + Result :: {ok, analyzers()} | {ok, analyzers(), token()} | {error, term()}. +list_analyzers_all(AwsConfig) + when is_record(AwsConfig, aws_config) -> + list_analyzers_all(AwsConfig, _Params = []); +list_analyzers_all(Params) -> + AwsConfig = erlcloud_aws:default_config(), + list_analyzers(AwsConfig, Params). + +-spec list_analyzers_all(AwsConfig, Params) -> Result +when AwsConfig :: aws_config(), + Params :: params(), + Result :: {ok, analyzers()} | {ok, analyzers(), token()} | {error, term()}. +list_analyzers_all(AwsConfig, Params) -> + case list_analyzers(AwsConfig, Params) of + {ok, Analyzers} -> + {ok, Analyzers}; + {ok, Analyzers, Token} -> + list_analyzers_next(AwsConfig, Params, Token, Analyzers); + {error, Reason} -> + {error, Reason} + end. + +-spec get_analyzer(AwsConfig, AnalyzerName) -> Result +when AwsConfig :: aws_config(), + AnalyzerName :: binary() | string(), + Result :: {ok, analyzer()} | {error, not_found} | {error, term()}. +get_analyzer(AwsConfig, AnalyzerName) -> + Path = ["analyzer", AnalyzerName], + case request(AwsConfig, _Method = get, Path) of + {ok, Response} -> + Analyzer = proplists:get_value(<<"analyzer">>, Response), + {ok, Analyzer}; + {error, {<<"ResourceNotFoundException">>, _Message}} -> + {error, not_found}; + {error, Reason} -> + {error, Reason} + end. + +-spec create_analyzer(AwsConfig, Spec) -> Result +when AwsConfig :: aws_config(), + Spec :: create_analyzer_spec(), + Result :: {ok, Arn :: binary()} | {error, term()}. +create_analyzer(AwsConfig, Spec) -> + Path = ["analyzer"], + RequestBody = jsx:encode(Spec), + case request(AwsConfig, _Method = put, Path, _Params = [], RequestBody) of + {ok, Response} -> + Arn = proplists:get_value(<<"arn">>, Response), + {ok, Arn}; + {error, Reason} -> + {error, Reason} + end. + +-spec delete_analyzer(AwsConfig, AnalyzerName) -> Result +when AwsConfig :: aws_config(), + AnalyzerName :: binary() | string(), + Result :: ok | {error, not_found} | {error, term()}. +delete_analyzer(AwsConfig, AnalyzerName) -> + delete_analyser(AwsConfig, AnalyzerName, _Params = []). + +-spec delete_analyser(AwsConfig, AnalyzerName, Params) -> Result +when AwsConfig :: aws_config(), + AnalyzerName :: binary() | string(), + Params :: params(), + Result :: ok | {error, not_found} | {error, term()}. +delete_analyser(AwsConfig, AnalyzerName, Params) -> + Path = ["analyzer", AnalyzerName], + case request(AwsConfig, _Method = delete, Path, Params) of + ok -> + ok; + {error, {<<"ResourceNotFoundException">>, _Message}} -> + {error, not_found}; + {error, Reason} -> + {error, Reason} + end. + +%% ----------------------------------------------------------------------------- +%% Local functions +%% ----------------------------------------------------------------------------- + +list_analyzers_next(AwsConfig, Params, Token0, Analyzers0) -> + case list_analyzers(AwsConfig, [{<<"nextToken">>, Token0} | Params]) of + {ok, Analyzers} -> + Analyzers1 = lists:append(Analyzers0, Analyzers), + {ok, Analyzers1}; + {ok, Analyzers, Token1} -> + Analyzers1 = lists:append(Analyzers0, Analyzers), + list_analyzers_next(AwsConfig, Params, Token1, Analyzers1); + {error, Reason} -> + {error, Reason} + end. + +request(AwsConfig, Method, Path) -> + request(AwsConfig, Method, Path, _Params = []). + +request(AwsConfig, Method, Path, Params) -> + request(AwsConfig, Method, Path, Params, _RequestBody = <<>>). + +request(AwsConfig0, Method, Path, Params, RequestBody) -> + case erlcloud_aws:update_config(AwsConfig0) of + {ok, AwsConfig1} -> + AwsRequest0 = init_request(AwsConfig1, Method, Path, Params, RequestBody), + AwsRequest1 = erlcloud_retry:request(AwsConfig1, AwsRequest0, fun should_retry/1), + case AwsRequest1#aws_request.response_type of + ok -> + decode_response(AwsRequest1); + error -> + decode_error(AwsRequest1) + end; + {error, Reason} -> + {error, Reason} + end. + +init_request(AwsConfig, Method, Path, Params, Payload) -> + Host = AwsConfig#aws_config.access_analyzer_host, + Service = "access-analyzer", + NormPath = norm_path(Path), + NormParams = norm_params(Params), + Region = erlcloud_aws:aws_region_from_host(Host), + Headers = [{"host", Host}, {"content-type", "application/json"}], + SignedHeaders = erlcloud_aws:sign_v4(Method, NormPath, AwsConfig, Headers, Payload, Region, Service, Params), + #aws_request{ + service = access_analyzer, + method = Method, + uri = "https://" ++ Host ++ NormPath ++ NormParams, + request_headers = SignedHeaders, + request_body = Payload + }. + +norm_path(Path) -> + binary_to_list(iolist_to_binary(["/" | lists:join("/", Path)])). + +norm_params([] = _Params) -> + ""; +norm_params(Params) -> + "?" ++ erlcloud_aws:canonical_query_string(Params). + +should_retry(Request) + when Request#aws_request.response_type == ok -> + Request; +should_retry(Request) + when Request#aws_request.response_type == error, + Request#aws_request.response_status == 429 -> + Request#aws_request{should_retry = true}; +should_retry(Request) + when Request#aws_request.response_type == error, + Request#aws_request.response_status >= 500 -> + Request#aws_request{should_retry = true}; +should_retry(Request) -> + Request#aws_request{should_retry = false}. + +decode_response(AwsRequest) -> + case AwsRequest#aws_request.response_body of + <<>> -> + ok; + ResponseBody -> + Json = jsx:decode(ResponseBody, [{return_maps, false}]), + {ok, Json} + end. + +decode_error(AwsRequest) -> + case AwsRequest#aws_request.error_type of + aws -> + Type = extract_error_type(AwsRequest), + Message = extract_error_message(AwsRequest), + {error, {Type, Message}}; + _ -> + erlcloud_aws:request_to_return(AwsRequest) + end. + +extract_error_type(AwsRequest) -> + Headers = AwsRequest#aws_request.response_headers, + Value = proplists:get_value("x-amzn-errortype", Headers), + iolist_to_binary(Value). + +extract_error_message(AwsRequest) -> + ResponseBody = AwsRequest#aws_request.response_body, + Object = jsx:decode(ResponseBody, [{return_maps, false}]), + proplists:get_value(<<"message">>, Object, <<>>). diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 14a4086d4..2c51e2cb9 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -21,6 +21,7 @@ request_to_return/1, sign_v4_headers/5, sign_v4/8, + canonical_query_string/1, get_service_status/1, is_throttling_error_response/1, get_timeout/1, @@ -667,6 +668,11 @@ service_config( Service, Region, Config ) when is_atom(Region) -> service_config( Service, atom_to_binary(Region, latin1), Config ); service_config( Service, Region, Config ) when is_list(Region) -> service_config( Service, list_to_binary(Region), Config ); +service_config( <<"access_analyzer">>, Region, Config ) -> + service_config( <<"access-analyzer">>, Region, Config ); +service_config( <<"access-analyzer">> = Service, Region, Config ) -> + Host = service_host( Service, Region ), + Config#aws_config{access_analyzer_host = Host}; service_config( <<"as">>, Region, Config ) -> service_config( <<"autoscaling">>, Region, Config ); service_config( <<"autoscaling">> = Service, Region, Config ) -> @@ -1192,16 +1198,26 @@ canonical_headers(Headers) -> {Canonical, Signed}. %% @doc calculate canonical query string out of query params and according to v4 documentation +-spec canonical_query_string(Params) -> String +when Params :: [{Name, Value}], + Name :: atom() | string() | binary(), + Value :: atom() | string() | binary() | integer(), + String :: string(). canonical_query_string([]) -> ""; canonical_query_string(Params) -> - Normalized = [{erlcloud_http:url_encode(Name), erlcloud_http:url_encode(erlcloud_http:value_to_string(Value))} || {Name, Value} <- Params], - Sorted = lists:keysort(1, Normalized), - string:join([case Value of - [] -> [Key, "="]; - _ -> [Key, "=", Value] - end - || {Key, Value} <- Sorted, Value =/= none, Value =/= undefined], "&"). + Encoded = [encode_param(Name, Value) || {Name, Value} <- Params], + Sorted = lists:sort(Encoded), + string:join(Sorted, "&"). + +encode_param(Name, Value) -> + EncodedName = erlcloud_http:url_encode(erlcloud_http:value_to_string(Name)), + case erlcloud_http:url_encode(erlcloud_http:value_to_string(Value)) of + "" -> + EncodedName ++ "="; + EncodedValue -> + EncodedName ++ "=" ++ EncodedValue + end. trimall(Value) -> %% TODO - remove excess internal whitespace in header values diff --git a/test/erlcloud_access_analyzer_tests.erl b/test/erlcloud_access_analyzer_tests.erl new file mode 100644 index 000000000..154bcad03 --- /dev/null +++ b/test/erlcloud_access_analyzer_tests.erl @@ -0,0 +1,299 @@ +-module(erlcloud_access_analyzer_tests). + +-include_lib("eunit/include/eunit.hrl"). +-include("erlcloud_aws.hrl"). + +-define(TEST_AWS_CONFIG, + #aws_config{ + access_key_id = "TEST_ACCESS_KEY_ID", + secret_access_key = "TEST_ACCESS_KEY", + security_token = "TEST_SECURITY_TOKEN" + } +). + +api_test_() -> + { + foreach, + fun() -> meck:new(erlcloud_httpc) end, + fun(_) -> meck:unload() end, + [ + fun list_analyzers_tests/1, + fun get_analyzer_tests/1, + fun create_analyzer_tests/1, + fun delete_analyzer_tests/1 + ] + }. + +list_analyzers_tests(_) -> + [ + { + "ListAnalyzers", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://access-analyzer.us-east-1.amazonaws.com/analyzer?maxResults=10", + Method = get, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseContent = << + "{" + "\"analyzers\":[" + "{" + "\"arn\":\"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-1\"," + "\"name\":\"test-1\"," + "\"type\":\"ACCOUNT\"," + "\"createdAt\":\"2022-01-01T01:02:03Z\"," + "\"status\":\"ACTIVE\"" + "}," + "{" + "\"arn\":\"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-2\"," + "\"name\":\"test-2\"," + "\"type\":\"ACCOUNT\"," + "\"createdAt\":\"2022-02-02T01:02:03Z\"," + "\"status\":\"ACTIVE\"" + "}" + "]" + "}" + >>, + {ok, {{200, "OK"}, [], ResponseContent}} + end + end + ), + Params = [{"maxResults", 10}], + Result = erlcloud_access_analyzer:list_analyzers(AwsConfig, Params), + Analyzers = [ + [ + {<<"arn">>, <<"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-1">>}, + {<<"name">>, <<"test-1">>}, + {<<"type">>, <<"ACCOUNT">>}, + {<<"createdAt">>, <<"2022-01-01T01:02:03Z">>}, + {<<"status">>, <<"ACTIVE">>} + ], + [ + {<<"arn">>, <<"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-2">>}, + {<<"name">>, <<"test-2">>}, + {<<"type">>, <<"ACCOUNT">>}, + {<<"createdAt">>, <<"2022-02-02T01:02:03Z">>}, + {<<"status">>, <<"ACTIVE">>} + ] + ], + ?assertEqual({ok, Analyzers}, Result) + end + }, + { + "ListAnalyzers [all]", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url1 = "https://access-analyzer.us-east-1.amazonaws.com/analyzer?maxResults=1&type=ACCOUNT", + Url2 = "https://access-analyzer.us-east-1.amazonaws.com/analyzer?maxResults=1&nextToken=TEST_NEXT_TOKEN&type=ACCOUNT", + Method = get, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url1, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseContent = << + "{" + "\"analyzers\":[" + "{" + "\"arn\":\"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-1\"," + "\"name\":\"test-1\"," + "\"type\":\"ACCOUNT\"," + "\"createdAt\":\"2022-01-01T01:02:03Z\"," + "\"status\":\"ACTIVE\"" + "}" + "]," + "\"nextToken\":\"TEST_NEXT_TOKEN\"" + "}" + >>, + {ok, {{200, "OK"}, [], ResponseContent}}; + [Url2, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseContent = << + "{" + "\"analyzers\":[" + "{" + "\"arn\":\"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-2\"," + "\"name\":\"test-2\"," + "\"type\":\"ACCOUNT\"," + "\"createdAt\":\"2022-02-02T01:02:03Z\"," + "\"status\":\"ACTIVE\"" + "}" + "]" + "}" + >>, + {ok, {{200, "OK"}, [], ResponseContent}} + end + end + ), + Params = [{"type", "ACCOUNT"}, {"maxResults", 1}], + Result = erlcloud_access_analyzer:list_analyzers_all(AwsConfig, Params), + Analyzers = [ + [ + {<<"arn">>, <<"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-1">>}, + {<<"name">>, <<"test-1">>}, + {<<"type">>, <<"ACCOUNT">>}, + {<<"createdAt">>, <<"2022-01-01T01:02:03Z">>}, + {<<"status">>, <<"ACTIVE">>} + ], + [ + {<<"arn">>, <<"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-2">>}, + {<<"name">>, <<"test-2">>}, + {<<"type">>, <<"ACCOUNT">>}, + {<<"createdAt">>, <<"2022-02-02T01:02:03Z">>}, + {<<"status">>, <<"ACTIVE">>} + ] + ], + ?assertEqual({ok, Analyzers}, Result) + end + } + ]. + +get_analyzer_tests(_) -> + [ + { + "GetAnalyzer", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://access-analyzer.us-east-1.amazonaws.com/analyzer/test-1", + Method = get, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseContent = << + "{" + "\"analyzer\":{" + "\"arn\":\"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-1\"," + "\"name\":\"test-1\"," + "\"type\":\"ACCOUNT\"," + "\"createdAt\":\"2022-01-01T01:02:03Z\"," + "\"status\":\"ACTIVE\"" + "}" + "}" + >>, + {ok, {{200, "OK"}, [], ResponseContent}} + end + end + ), + Result = erlcloud_access_analyzer:get_analyzer(AwsConfig, _AnalyzerName = "test-1"), + Analyzer = [ + {<<"arn">>, <<"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-1">>}, + {<<"name">>, <<"test-1">>}, + {<<"type">>, <<"ACCOUNT">>}, + {<<"createdAt">>, <<"2022-01-01T01:02:03Z">>}, + {<<"status">>, <<"ACTIVE">>} + ], + ?assertEqual({ok, Analyzer}, Result) + end + }, + { + "GetAnalyzer -> not_found", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://access-analyzer.us-east-1.amazonaws.com/analyzer/test-2", + Method = get, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseHeaders = [{"x-amzn-errortype", "ResourceNotFoundException"}], + ResponseContent = <<"{\"message\": \"Analyzer not found\"}">>, + {ok, {{404, "NotFound"}, ResponseHeaders, ResponseContent}} + end + end + ), + Result = erlcloud_access_analyzer:get_analyzer(AwsConfig, _AnalyzerName = "test-2"), + ?assertEqual({error, not_found}, Result) + end + } + ]. + +create_analyzer_tests(_) -> + [ + { + "CreateAnalyzer", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://access-analyzer.us-east-1.amazonaws.com/analyzer", + Method = put, + RequestContent = << + "{" + "\"analyzerName\":\"test-1\"," + "\"type\":\"ACCOUNT\"" + "}" + >>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseContent = <<"{\"arn\":\"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-1\"}">>, + {ok, {{200, "OK"}, [], ResponseContent}} + end + end + ), + AnalyzerSpec = [ + {<<"analyzerName">>, <<"test-1">>}, + {<<"type">>, <<"ACCOUNT">>} + ], + Result = erlcloud_access_analyzer:create_analyzer(AwsConfig, AnalyzerSpec), + Arn = <<"arn:aws:access-analyzer:us-east-1:123456789012:analyzer/test-1">>, + ?assertEqual({ok, Arn}, Result) + end + } + ]. + +delete_analyzer_tests(_) -> + [ + { + "DeleteAnalyzer", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://access-analyzer.us-east-1.amazonaws.com/analyzer/test-1", + Method = delete, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseContent = <<>>, + {ok, {{200, "OK"}, [], ResponseContent}} + end + end + ), + Result = erlcloud_access_analyzer:delete_analyzer(AwsConfig, _AnalyzerName = "test-1"), + ?assertEqual(ok, Result) + end + }, + { + "DeleteAnalyzer -> not_found", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://access-analyzer.us-east-1.amazonaws.com/analyzer/test-1", + Method = delete, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseHeaders = [{"x-amzn-errortype", "ResourceNotFoundException"}], + ResponseContent = <<"{\"message\": \"Analyzer not found\"}">>, + {ok, {{404, "NotFound"}, ResponseHeaders, ResponseContent}} + end + end + ), + Result = erlcloud_access_analyzer:delete_analyzer(AwsConfig, _AnalyzerName = "test-1"), + ?assertEqual({error, not_found}, Result) + end + } + ]. From fb0e459febbae48b663788ea08681c36c06bd2f9 Mon Sep 17 00:00:00 2001 From: Pavel Puchkin Date: Thu, 17 Nov 2022 13:46:41 +0200 Subject: [PATCH 258/310] Add SetIdentityHeadersInNotificationsEnabled to SES --- src/erlcloud_ses.erl | 57 +++++++++++++++++++++++++++++++++++-- test/erlcloud_ses_tests.erl | 23 +++++++++++++++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_ses.erl b/src/erlcloud_ses.erl index cbd7ae222..c52549a57 100644 --- a/src/erlcloud_ses.erl +++ b/src/erlcloud_ses.erl @@ -48,6 +48,7 @@ -export([set_identity_dkim_enabled/2, set_identity_dkim_enabled/3]). -export([set_identity_feedback_forwarding_enabled/2, set_identity_feedback_forwarding_enabled/3]). -export([set_identity_notification_topic/3, set_identity_notification_topic/4]). +-export([set_identity_headers_in_notifications_enabled/3, set_identity_headers_in_notifications_enabled/4]). -export([update_custom_verification_email_template/2, update_custom_verification_email_template/3]). @@ -82,11 +83,14 @@ -type verification_status() :: pending | success | failed | temporary_failure | not_started. +-type notification_type() :: bounce | complaint | delivery. + -export_type([custom_template_attribute_names/0, custom_template_attributes/0, identity/0, identities/0, email/0, emails/0, domain/0, - verification_status/0]). + verification_status/0, + notification_type/0]). %%%------------------------------------------------------------------------------ %%% Library initialization @@ -874,8 +878,6 @@ set_identity_feedback_forwarding_enabled(Identity, ForwardingEnabled, Config) -> %%% SetIdentityNotificationTopic %%%------------------------------------------------------------------------------ --type notification_type() :: bounce | complaint | delivery. - -type sns_topic() :: string() | binary(). -type set_identity_notification_topic_result() :: ok | {error, term()}. @@ -915,6 +917,44 @@ set_identity_notification_topic(Identity, NotificationType, SnsTopic, Config) -> {error, Reason} -> {error, Reason} end. + +%%%------------------------------------------------------------------------------ +%%% SetIdentityHeadersInNotificationsEnabled +%%%------------------------------------------------------------------------------ + +set_identity_headers_in_notifications_enabled(Identity, NotificationType, Enabled) -> + set_identity_headers_in_notifications_enabled(Identity, NotificationType, Enabled, default_config()). + +%%------------------------------------------------------------------------------ +%% @doc +%% SES API: +%% [https://docs.aws.amazon.com/ses/latest/APIReference/API_SetIdentityHeadersInNotificationsEnabled.html] +%% +%% ===Example=== +%% +%% Enable headers in notifications for an identity. +%% +%% ` +%% ok = erlcloud_ses:set_identity_headers_in_notifications_enabled(<<"user@example.com">>, bounce, true). +%% ' +%% +%% @end +%%------------------------------------------------------------------------------ + +-spec set_identity_headers_in_notifications_enabled(identity(), + notification_type(), + boolean(), + aws_config()) -> + ok | {error, term()}. +set_identity_headers_in_notifications_enabled(Identity, NotificationType, Enabled, Config) -> + Params = encode_params([{identity, Identity}, + {notification_type, NotificationType}, + {enabled, Enabled}]), + case ses_request(Config, "SetIdentityHeadersInNotificationsEnabled", Params) of + {ok, _Doc} -> ok; + {error, Reason} -> {error, Reason} + end. + %%------------------------------------------------------------------------------ %% @doc %% SES API: @@ -1080,6 +1120,8 @@ encode_params([{destination, Destination} | T], Acc) -> encode_params(T, encode_destination(Destination, Acc)); encode_params([{email_address, EmailAddress} | T], Acc) when is_list(EmailAddress); is_binary(EmailAddress) -> encode_params(T, [{"EmailAddress", EmailAddress} | Acc]); +encode_params([{enabled, Enabled} | T], Acc) when is_boolean(Enabled) -> + encode_params(T, [{"Enabled", Enabled} | Acc]); encode_params([{dkim_enabled, DkimEnabled} | T], Acc) when is_boolean(DkimEnabled) -> encode_params(T, [{"DkimEnabled", DkimEnabled} | Acc]); encode_params([{domain, Domain} | T], Acc) when is_list(Domain); is_binary(Domain) -> @@ -1259,6 +1301,15 @@ decode_dkim_attributes(DkimAttributesDoc) -> decode_notification_attributes(NotificationAttributesDoc) -> [{erlcloud_xml:get_text("key", Entry), erlcloud_xml:decode([{forwarding_enabled, "value/ForwardingEnabled", boolean}, + {headers_in_bounce_notifications_enabled, + "value/HeadersInBounceNotificationsEnabled", + boolean}, + {headers_in_complaint_notifications_enabled, + "value/HeadersInComplaintNotificationsEnabled", + boolean}, + {headers_in_delivery_notifications_enabled, + "value/HeadersInDeliveryNotificationsEnabled", + boolean}, {bounce_topic, "value/BounceTopic", optional_text}, {complaint_topic, "value/ComplaintTopic", optional_text}, {delivery_topic, "value/DeliveryTopic", optional_text}], diff --git a/test/erlcloud_ses_tests.erl b/test/erlcloud_ses_tests.erl index 786dae07e..d483eafed 100644 --- a/test/erlcloud_ses_tests.erl +++ b/test/erlcloud_ses_tests.erl @@ -25,6 +25,7 @@ operation_test_() -> fun send_raw_email_tests/1, fun set_identity_dkim_enabled_tests/1, fun set_identity_feedback_forwarding_enabled_tests/1, + fun set_identity_headers_in_notifications_enabled_tests/1, fun set_identity_notification_topic_tests/1, fun verify_domain_dkim_tests/1, fun verify_domain_identity_tests/1, @@ -135,6 +136,9 @@ get_identity_notification_attributes_tests(_) -> user@example.com true + true + true + true arn:aws:sns:us-east-1:123456789012:example arn:aws:sns:us-east-1:123456789012:example arn:aws:sns:us-east-1:123456789012:example @@ -149,6 +153,9 @@ get_identity_notification_attributes_tests(_) -> meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), ?assertEqual({ok, [{notification_attributes, [{"user@example.com", [{forwarding_enabled, true}, + {headers_in_bounce_notifications_enabled, true}, + {headers_in_complaint_notifications_enabled, true}, + {headers_in_delivery_notifications_enabled, true}, {bounce_topic, "arn:aws:sns:us-east-1:123456789012:example"}, {complaint_topic, "arn:aws:sns:us-east-1:123456789012:example"}, {delivery_topic, "arn:aws:sns:us-east-1:123456789012:example"}]}]}]}, @@ -422,6 +429,22 @@ set_identity_feedback_forwarding_enabled_tests(_) -> end ]. +set_identity_headers_in_notifications_enabled_tests(_) -> + [fun() -> + configure(), + Expected = "Action=SetIdentityHeadersInNotificationsEnabled&Version=2010-12-01&Identity=user%40example.com&NotificationType=Bounce&Enabled=true", + Response = + " + + + 299f4af4-b72a-11e1-901f-1fbd90e8104f + + ", + meck:expect(erlcloud_httpc, request, input_expect(Response, Expected)), + ?assertEqual(ok, erlcloud_ses:set_identity_headers_in_notifications_enabled("user@example.com", bounce, true)) + end + ]. + set_identity_notification_topic_tests(_) -> [fun() -> configure(), From 642b1da7819a10908ba9e916342f9c5efebc1b92 Mon Sep 17 00:00:00 2001 From: neil-bleasdale <116366258+neil-bleasdale@users.noreply.github.com> Date: Mon, 28 Nov 2022 13:51:46 +0000 Subject: [PATCH 259/310] Add functionality for retrieval of IAM server certificate tags (#740) --- src/erlcloud_iam.erl | 35 +++++++++++- test/erlcloud_iam_tests.erl | 108 ++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_iam.erl b/src/erlcloud_iam.erl index e376a194e..3a968871a 100644 --- a/src/erlcloud_iam.erl +++ b/src/erlcloud_iam.erl @@ -52,6 +52,8 @@ get_server_certificate/1, get_server_certificate/2, list_server_certificates/0, list_server_certificates/1, list_server_certificates/2, list_server_certificates_all/0, list_server_certificates_all/1, list_server_certificates_all/2, + list_server_certificate_tags/1, list_server_certificate_tags/2, + list_server_certificate_tags_all/1, list_server_certificate_tags_all/2, list_instance_profiles/0, list_instance_profiles/1, list_instance_profiles/2, list_instance_profiles_all/0, list_instance_profiles_all/1, list_instance_profiles_all/2, get_instance_profile/1, get_instance_profile/2, @@ -690,6 +692,34 @@ list_server_certificates_all(PathPrefix, #aws_config{} = Config) DataTypeDef = data_type("ServerCertificateMetadataList"), iam_query_all(Config, Action, Params, ItemPath, DataTypeDef). +-spec list_server_certificate_tags(string()) -> {ok, [proplist()]} | {error, any()}. +list_server_certificate_tags(ServerCertificateName) + when is_list(ServerCertificateName) -> + list_server_certificate_tags(ServerCertificateName, default_config()). + +-spec list_server_certificate_tags(string(), aws_config()) -> {ok, [proplist()]} | {error, any()}. +list_server_certificate_tags(ServerCertificateName, #aws_config{} = Config) + when is_list(ServerCertificateName) -> + Action = "ListServerCertificateTags", + Params = [{"ServerCertificateName", ServerCertificateName}], + ItemPath = "/ListServerCertificateTagsResponse/ListServerCertificateTagsResult/Tags/member", + DataTypeDef = data_type("ServerCertificateTags"), + iam_query(Config, Action, Params, ItemPath, DataTypeDef). + +-spec list_server_certificate_tags_all(string()) -> {ok, [proplist()]} | {error, any()}. +list_server_certificate_tags_all(ServerCertificateName) + when is_list(ServerCertificateName) -> + list_server_certificate_tags_all(ServerCertificateName, default_config()). + +-spec list_server_certificate_tags_all(string(), aws_config()) -> {ok, [proplist()]} | {error, any()}. +list_server_certificate_tags_all(ServerCertificateName, #aws_config{} = Config) + when is_list(ServerCertificateName) -> + Action = "ListServerCertificateTags", + Params = [{"ServerCertificateName", ServerCertificateName}], + ItemPath = "/ListServerCertificateTagsResponse/ListServerCertificateTagsResult/Tags/member", + DataTypeDef = data_type("ServerCertificateTags"), + iam_query_all(Config, Action, Params, ItemPath, DataTypeDef). + % % InstanceProfile % @@ -1148,7 +1178,10 @@ data_type("ServerCertificateMetadataList") -> {"Arn", arn, "String"}, {"Path", path, "String"}, {"ServerCertificateId", server_certificate_id, "String"}, - {"ServerCertificateName", server_certificate_name, "String"}]. + {"ServerCertificateName", server_certificate_name, "String"}]; +data_type("ServerCertificateTags") -> + [{"Value", value, "String"}, + {"Key", key, "String"}]. data_fun("String") -> {erlcloud_xml, get_text}; data_fun("DateTime") -> {erlcloud_xml, get_time}; diff --git a/test/erlcloud_iam_tests.erl b/test/erlcloud_iam_tests.erl index 182b6b76d..fa948c893 100644 --- a/test/erlcloud_iam_tests.erl +++ b/test/erlcloud_iam_tests.erl @@ -77,6 +77,9 @@ iam_api_test_() -> fun list_server_certificates_input_tests/1, fun list_server_certificates_output_tests/1, fun list_server_certificates_all_output_tests/1, + fun list_server_certificate_tags_input_tests/1, + fun list_server_certificate_tags_output_tests/1, + fun list_server_certificate_tags_all_output_tests/1, fun list_instance_profiles_input_tests/1, fun list_instance_profiles_output_tests/1, fun list_instance_profiles_all_output_tests/1, @@ -1907,6 +1910,111 @@ list_server_certificates_all_output_tests(_) -> ], output_tests_seq(?_f(erlcloud_iam:list_server_certificates_all("test")), Tests). +-define(SERVER_CERTIFICATE_TAGS_RESP, + " + + false + + + Dept + 12345 + + + Team + Accounting + + + + + EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE + + " +). + +list_server_certificate_tags_input_tests(_) -> + Tests = [ + ?_iam_test({ + "Test input for returning server certificate tags", + ?_f(erlcloud_iam:list_server_certificate_tags("test")), + [{"Action", "ListServerCertificateTags"}, {"ServerCertificateName", "test"}] + }) + ], + input_tests(?SERVER_CERTIFICATE_TAGS_RESP, Tests). + +list_server_certificate_tags_output_tests(_) -> + Tests = [ + ?_iam_test({ + "Test returning server certificate tags", + ?SERVER_CERTIFICATE_TAGS_RESP, + {ok, [ + [{key, "Dept"}, + {value, "12345"}], + [{key, "Team"}, + {value, "Accounting"}] + ]} + }) + ], + output_tests(?_f(erlcloud_iam:list_server_certificate_tags("test")), Tests). + +-define(SERVER_CERTIFICATE_TAGS_ALL_RESP, [ + " + + true + marker + + + Dept + 12345 + + + Team + Accounting + + + + + EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE + + ", + " + + false + + + Dept + 00001 + + + Team + Engineering + + + + + EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE + + " +]). + +list_server_certificate_tags_all_output_tests(_) -> + Tests = [ + ?_iam_test({ + "Test returning all pages of server certificate tags", + ?SERVER_CERTIFICATE_TAGS_ALL_RESP, + {ok, [ + [{key, "Dept"}, + {value, "12345"}], + [{key, "Team"}, + {value, "Accounting"}], + [{key, "Dept"}, + {value, "00001"}], + [{key, "Team"}, + {value, "Engineering"}] + ]} + }) + ], + output_tests_seq(?_f(erlcloud_iam:list_server_certificate_tags_all("test")), Tests). + -define(LIST_INSTANCE_PROFILES_RESP, " From 077c23873e05fd9b3982880c676746289a0df12d Mon Sep 17 00:00:00 2001 From: Carl Isom Date: Fri, 6 Jan 2023 10:18:21 -0600 Subject: [PATCH 260/310] Add athena prepared statement support --- src/erlcloud_athena.erl | 156 +++++++++++++++++++++++++++++++-- test/erlcloud_athena_tests.erl | 112 ++++++++++++++++++++--- 2 files changed, 248 insertions(+), 20 deletions(-) diff --git a/src/erlcloud_athena.erl b/src/erlcloud_athena.erl index c04e081be..a2d43dbbd 100644 --- a/src/erlcloud_athena.erl +++ b/src/erlcloud_athena.erl @@ -10,18 +10,32 @@ batch_get_named_query/1, batch_get_named_query/2, + batch_get_prepared_statement/2, + batch_get_prepared_statement/3, + batch_get_query_execution/1, batch_get_query_execution/2, create_named_query/4, create_named_query/5, create_named_query/6, + + create_prepared_statement/3, + create_prepared_statement/4, + create_prepared_statement/5, + delete_named_query/1, delete_named_query/2, + delete_prepared_statement/2, + delete_prepared_statement/3, + get_named_query/1, get_named_query/2, + get_prepared_statement/2, + get_prepared_statement/3, + get_query_execution/1, get_query_execution/2, @@ -33,6 +47,10 @@ list_named_queries/1, list_named_queries/2, + list_prepared_statements/1, + list_prepared_statements/2, + list_prepared_statements/3, + list_query_executions/0, list_query_executions/1, list_query_executions/2, @@ -43,7 +61,11 @@ start_query_execution/7, stop_query_execution/1, - stop_query_execution/2 + stop_query_execution/2, + + update_prepared_statement/3, + update_prepared_statement/4, + update_prepared_statement/5 ]). -spec new(string(), string()) -> aws_config(). @@ -111,6 +133,19 @@ batch_get_named_query(NamedQueryIds, Config) -> Request = #{<<"NamedQueryIds">> => NamedQueryIds}, request(Config, "BatchGetNamedQuery", Request). +%% @doc +%% Athena API: +%% https://docs.aws.amazon.com/athena/latest/APIReference/API_BatchGetPreparedStatement.html +%% +-spec batch_get_prepared_statement(binary(), list(binary())) -> {ok, map()} | {error, any()}. +batch_get_prepared_statement(WorkGroup, PreparedStatementNames) -> + batch_get_prepared_statement(WorkGroup, PreparedStatementNames, default_config()). + +-spec batch_get_prepared_statement(binary(), list(binary()), aws_config()) -> {ok, map()} | {error, any()}. +batch_get_prepared_statement(WorkGroup, PreparedStatementNames, Config) -> + Request = #{<<"WorkGroup">> => WorkGroup, <<"PreparedStatementNames">> => PreparedStatementNames}, + request(Config, "BatchGetPreparedStatement", Request). + %% @doc %% Athena API: %% http://docs.aws.amazon.com/athena/latest/APIReference/API_BatchGetQueryExecution.html @@ -162,18 +197,40 @@ create_named_query(ClientReqToken, Db, Name, Query, Description) {ok, binary()} | {error, any()}. create_named_query(ClientReqToken, Db, Name, Query, Description, Config) -> Request0 = #{<<"ClientRequestToken">> => ClientReqToken, - <<"Database">> => Db, - <<"Name">> => Name, - <<"QueryString">> => Query}, - Request1 = update_query_description(Request0, Description), + <<"Database">> => Db, + <<"Name">> => Name, + <<"QueryString">> => Query}, + Request1 = update_description(Request0, Description), case request(Config, "CreateNamedQuery", Request1) of {ok, Res} -> {ok, maps:get(<<"NamedQueryId">>, Res)}; Error -> Error end. -update_query_description(Request, undefined) -> Request; -update_query_description(Request, Description) -> - maps:put(<<"Description">>, Description, Request). +%% @doc +%% Athena API: +%% https://docs.aws.amazon.com/athena/latest/APIReference/API_CreatePreparedStatement.html +%% +-spec create_prepared_statement(binary(), binary(), binary()) -> ok | {error, any()}. +create_prepared_statement(WorkGroup, StatementName, QueryStatement) -> + create_prepared_statement(WorkGroup, StatementName, QueryStatement, undefined, default_config()). + +-spec create_prepared_statement(binary(), binary(), binary(), binary() | aws_config()) -> ok | {error, any()}. +create_prepared_statement(WorkGroup, StatementName, QueryStatement, Config) when is_record(Config, aws_config) -> + create_prepared_statement(WorkGroup, StatementName, QueryStatement, undefined, Config); +create_prepared_statement(WorkGroup, StatementName, QueryStatement, Description) when is_binary(Description) -> + create_prepared_statement(WorkGroup, StatementName, QueryStatement, Description, default_config()). + +-spec create_prepared_statement(binary(), binary(), binary(), binary() | undefined, aws_config()) -> ok | {error, any()}. +create_prepared_statement(WorkGroup, StatementName, QueryStatement, Description, Config) -> + Request0 = #{<<"WorkGroup">> => WorkGroup, + <<"StatementName">> => StatementName, + <<"QueryStatement">> => QueryStatement}, + Request = update_description(Request0, Description), + case request(Config, "CreatePreparedStatement", Request) of + {ok, _} -> ok; + Error -> Error + end. + %% @doc %% Athena API: @@ -190,6 +247,22 @@ delete_named_query(NamedQueryId, Config) -> {ok, _} -> ok; Error -> Error end. +%% @doc +%% Athena API: +%% https://docs.aws.amazon.com/athena/latest/APIReference/API_DeletePreparedStatement.html +%% + +-spec delete_prepared_statement(binary(), binary()) -> ok | {error | any}. +delete_prepared_statement(WorkGroup, StatementName) -> + delete_prepared_statement(WorkGroup, StatementName, default_config()). + +-spec delete_prepared_statement(binary(), binary(), aws_config()) -> ok | {error | any}. +delete_prepared_statement(WorkGroup, StatementName, Config) -> + Request = #{<<"WorkGroup">> => WorkGroup, <<"StatementName">> => StatementName}, + case request(Config, "DeletePreparedStatement", Request) of + {ok, _} -> ok; + Error -> Error + end. %% @doc %% Athena API: @@ -204,6 +277,19 @@ get_named_query(NamedQueryId, Config) -> Request = #{<<"NamedQueryId">> => NamedQueryId}, request(Config, "GetNamedQuery", Request). +%% @doc +%% Athena API: +%% https://docs.aws.amazon.com/athena/latest/APIReference/API_GetPreparedStatement.html +%% +-spec get_prepared_statement(binary(), binary()) -> {ok, map()} | {error, any()}. +get_prepared_statement(WorkGroup, StatementName) -> + get_prepared_statement(WorkGroup, StatementName, default_config()). + +-spec get_prepared_statement(binary(), binary(), aws_config()) -> {ok, map()} | {error, any()}. +get_prepared_statement(WorkGroup, StatementName, Config) -> + Request = #{<<"WorkGroup">> => WorkGroup, <<"StatementName">> => StatementName}, + request(Config, "GetPreparedStatement", Request). + %% @doc %% Athena API: %% http://docs.aws.amazon.com/athena/latest/APIReference/API_GetQueryExecution.html @@ -270,6 +356,31 @@ list_named_queries(PaginationMap) when is_map(PaginationMap) -> list_named_queries(PaginationMap, Config) -> request(Config, "ListNamedQueries", PaginationMap). +%% @doc +%% Athena API: +%% https://docs.aws.amazon.com/athena/latest/APIReference/API_ListPreparedStatements.html +%% +%% ` +%% erlcloud_athena:list_prepared_statements(<<"some-work-group">>, +%% #{<<"MaxResults">> => 1, +%% <<"NextToken">> => <<"some-token">>}). +%% ' +%% +- spec list_prepared_statements(binary()) -> {ok, map()} | {error, any()}. +list_prepared_statements(WorkGroup) -> + list_prepared_statements(WorkGroup, #{}, default_config()). + +- spec list_prepared_statements(binary(), map() | aws_config()) -> {ok, map()} | {error, any()}. +list_prepared_statements(WorkGroup, Config) when is_record(Config, aws_config) -> + list_prepared_statements(WorkGroup, #{}, Config); +list_prepared_statements(WorkGroup, PaginationMap) when is_map(PaginationMap) -> + list_prepared_statements(WorkGroup, PaginationMap, default_config()). + +-spec list_prepared_statements(binary(), map(), aws_config()) -> {ok, map()} | {error, any()}. +list_prepared_statements(WorkGroup, PaginationMap, Config) -> + Request = PaginationMap#{<<"WorkGroup">> => WorkGroup}, + request(Config, "ListPreparedStatements", Request). + %% @doc %% Athena API: %% http://docs.aws.amazon.com/athena/latest/APIReference/API_ListQueryExecutions.html @@ -362,6 +473,31 @@ get_encrypt_config(EncryptOption, KmsKey) -> #{<<"EncryptionOption">> => EncryptOption, <<"KmsKey">> => KmsKey}}. +%% @doc +%% Athena API: +%% https://docs.aws.amazon.com/athena/latest/APIReference/API_UpdatePreparedStatement.html +%% +-spec update_prepared_statement(binary(), binary(), binary()) -> ok | {error, any()}. +update_prepared_statement(WorkGroup, StatementName, QueryStatement) -> + update_prepared_statement(WorkGroup, StatementName, QueryStatement, undefined, default_config()). + +-spec update_prepared_statement(binary(), binary(), binary(), binary() | aws_config()) -> ok | {error, any()}. +update_prepared_statement(WorkGroup, StatementName, QueryStatement, Config) when is_record(Config, aws_config) -> + update_prepared_statement(WorkGroup, StatementName, QueryStatement, undefined, Config); +update_prepared_statement(WorkGroup, StatementName, QueryStatement, Description) when is_binary(Description) -> + update_prepared_statement(WorkGroup, StatementName, QueryStatement, Description, default_config()). + +-spec update_prepared_statement(binary(), binary(), binary(), binary() | undefined, aws_config()) -> ok | {error, any()}. +update_prepared_statement(WorkGroup, StatementName, QueryStatement, Description, Config) -> + Request0 = #{<<"WorkGroup">> => WorkGroup, + <<"StatementName">> => StatementName, + <<"QueryStatement">> => QueryStatement}, + Request = update_description(Request0, Description), + case request(Config, "UpdatePreparedStatement", Request) of + {ok, _} -> ok; + Error -> Error + end. + %% @doc %% Athena API: %% http://docs.aws.amazon.com/athena/latest/APIReference/API_StopQueryExecution.html @@ -429,3 +565,7 @@ get_url(#aws_config{athena_scheme = Scheme, athena_host = Host, athena_port = Port}) -> Scheme ++ Host ++ ":" ++ integer_to_list(Port). + +update_description(Request, undefined) -> Request; +update_description(Request, Description) -> + maps:put(<<"Description">>, Description, Request). diff --git a/test/erlcloud_athena_tests.erl b/test/erlcloud_athena_tests.erl index 36e3462ce..cafb8846d 100644 --- a/test/erlcloud_athena_tests.erl +++ b/test/erlcloud_athena_tests.erl @@ -21,6 +21,9 @@ -define(LOCATION_1, <<"s3://1/1.csv">>). -define(LOCATION_2, <<"s3://2/2.csv">>). -define(CLIENT_TOKEN, <<"some-token-uuid">>). +-define(STATEMENT_NAME_1, <<"some-statement-name">>). +-define(STATEMENT_NAME_2, <<"some-statement-name-2">>). +-define(WORKGROUP, <<"workgroup-name">>). -define(BATCH_GET_NAMED_QUERY_RESP, #{<<"NamedQueries">> => @@ -37,6 +40,21 @@ <<"UnprocessedNamedQueryIds">> => []} ). +-define(BATCH_GET_PREPARED_STATEMENT_RESP, + #{<<"QueryExecutions">> => + [#{<<"Description">> => ?QUERY_STR_1, + <<"LastModifiedTime">> => 1506093456.234, + <<"QueryStatement">> => ?QUERY_STR_1, + <<"StatementName">> => ?STATEMENT_NAME_1, + <<"WorkGroupName">> => ?WORKGROUP}, + #{<<"Description">> => ?QUERY_STR_2, + <<"LastModifiedTime">> => 1506094499.507, + <<"QueryStatement">> => ?QUERY_STR_2, + <<"StatementName">> => ?STATEMENT_NAME_2, + <<"WorkGroupName">> => ?WORKGROUP}], + <<"UnprocessedQueryExecutionIds">> => []} +). + -define(BATCH_GET_QUERY_EXECUTION_RESP, #{<<"QueryExecutions">> => [#{<<"Query">> => ?QUERY_STR_1, @@ -72,6 +90,15 @@ <<"QueryString">> => ?QUERY_STR_1}} ). +-define(GET_PREPARED_STATEMENT_RESP, + #{<<"PreparedStatement">> => + #{<<"Description">> => ?QUERY_STR_1, + <<"LastModifiedTime">> => 1506093456.234, + <<"QueryStatement">> => ?QUERY_STR_1, + <<"StatementName">> => ?STATEMENT_NAME_1, + <<"WorkGroupName">> => ?WORKGROUP}} +). + -define(GET_QUERY_EXECUTION_RESP, #{<<"QueryExecutions">> => #{<<"Query">> => ?QUERY_STR_1, @@ -121,6 +148,13 @@ #{<<"NamedQueryIds">> => [?QUERY_ID_1, ?QUERY_ID_2], <<"NextToken">> => ?CLIENT_TOKEN}). +-define(LIST_PREPARED_STATEMENTS_RESP, + #{<<"NextToken">> => ?CLIENT_TOKEN, + <<"PreparedStatements">> => + [#{<<"LastModifiedTime">> => 1506094499.507, + <<"StatementName">> => ?STATEMENT_NAME_1}] + }). + -define(LIST_QUERY_EXECUTIONS_RESP, #{<<"QueryExecutionIds">> => [?QUERY_ID_1, ?QUERY_ID_2], <<"NextToken">> => ?CLIENT_TOKEN}). @@ -142,13 +176,18 @@ erlcloud_athena_test_() -> fun meck:unload/1, [ fun test_batch_get_named_query/0, + fun test_batch_get_prepared_statement/0, fun test_batch_get_query_execution/0, fun test_create_named_query/0, + fun test_create_prepared_statement/0, fun test_delete_named_query/0, + fun test_delete_prepared_statements/0, fun test_get_named_query/0, + fun test_get_prepared_statement/0, fun test_get_query_execution/0, fun test_get_query_results/0, fun test_list_named_queries/0, + fun test_list_prepared_statements/0, fun test_list_query_executions/0, fun test_start_query_execution/0, fun test_start_query_execution_with_encryption/0, @@ -156,7 +195,8 @@ erlcloud_athena_test_() -> fun test_start_query_execution_without_encrypt_option/0, fun test_stop_query_execution/0, fun test_error_no_retry/0, - fun test_error_retry/0 + fun test_error_retry/0, + fun test_update_prepared_statement/0 ] }. @@ -168,6 +208,14 @@ test_batch_get_named_query() -> end, do_test(Request, Expected, TestFun). +test_batch_get_prepared_statement() -> + Request = #{<<"WorkGroup">> => ?WORKGROUP, <<"PreparedStatementNames">> => [?STATEMENT_NAME_1, ?STATEMENT_NAME_2]}, + Expected = {ok, ?BATCH_GET_PREPARED_STATEMENT_RESP}, + TestFun = fun() -> + erlcloud_athena:batch_get_prepared_statement(?WORKGROUP, [?STATEMENT_NAME_1, ?STATEMENT_NAME_2]) + end, + do_test(Request, Expected, TestFun). + test_batch_get_query_execution() -> Request = #{<<"QueryExecutionIds">> => [?QUERY_ID_1, ?QUERY_ID_2]}, Expected = {ok, ?BATCH_GET_QUERY_EXECUTION_RESP}, @@ -190,18 +238,38 @@ test_create_named_query() -> end, do_test(Request, Expected, TestFun). +test_create_prepared_statement() -> + Request = #{<<"WorkGroup">> => ?WORKGROUP, + <<"StatementName">> => ?STATEMENT_NAME_1, + <<"QueryStatement">> => ?QUERY_STR_1}, + Expected = ok, + TestFun = fun() -> erlcloud_athena:create_prepared_statement(?WORKGROUP, ?STATEMENT_NAME_1, ?QUERY_STR_1) end, + do_test(Request, Expected, TestFun). + test_delete_named_query() -> Request = #{<<"NamedQueryId">> => ?QUERY_ID_1}, Expected = ok, TestFun = fun() -> erlcloud_athena:delete_named_query(?QUERY_ID_1) end, do_test(Request, Expected, TestFun). +test_delete_prepared_statements() -> + Request = #{<<"WorkGroup">> => ?WORKGROUP, <<"StatementName">> => ?STATEMENT_NAME_1}, + Expected = ok, + TestFun = fun() -> erlcloud_athena:delete_prepared_statement(?WORKGROUP, ?STATEMENT_NAME_1) end, + do_test(Request, Expected, TestFun). + test_get_named_query() -> Request = #{<<"NamedQueryId">> => ?QUERY_ID_1}, Expected = {ok, ?GET_NAMED_QUERY_RESP}, TestFun = fun() -> erlcloud_athena:get_named_query(?QUERY_ID_1) end, do_test(Request, Expected, TestFun). +test_get_prepared_statement() -> + Request = #{<<"WorkGroup">> => ?WORKGROUP, <<"StatementName">> => ?STATEMENT_NAME_1}, + Expected = {ok, ?GET_PREPARED_STATEMENT_RESP}, + TestFun = fun() -> erlcloud_athena:get_prepared_statement(?WORKGROUP, ?STATEMENT_NAME_1) end, + do_test(Request, Expected, TestFun). + test_get_query_execution() -> Request = #{<<"QueryExecutionId">> => ?QUERY_ID_1}, Expected = {ok, ?GET_QUERY_EXECUTION_RESP}, @@ -223,6 +291,12 @@ test_list_named_queries() -> TestFun = fun() -> erlcloud_athena:list_named_queries() end, do_test(#{}, Expected, TestFun). +test_list_prepared_statements() -> + Request = #{<<"WorkGroup">> => ?WORKGROUP}, + Expected = {ok, ?LIST_PREPARED_STATEMENTS_RESP}, + TestFun = fun() -> erlcloud_athena:list_prepared_statements(?WORKGROUP) end, + do_test(Request, Expected, TestFun). + test_list_query_executions() -> Request = #{<<"MaxResults">> => 1, <<"NextToken">> => ?CLIENT_TOKEN}, @@ -307,6 +381,14 @@ test_error_retry() -> erlcloud_athena:stop_query_execution(?QUERY_ID_1) ). +test_update_prepared_statement() -> + Request = #{<<"WorkGroup">> => ?WORKGROUP, + <<"StatementName">> => ?STATEMENT_NAME_1, + <<"QueryStatement">> => ?QUERY_STR_2}, + Expected = ok, + TestFun = fun() -> erlcloud_athena:update_prepared_statement(?WORKGROUP, ?STATEMENT_NAME_1, ?QUERY_STR_2) end, + do_test(Request, Expected, TestFun). + do_test(Request, ExpectedResult, TestedFun) -> erlcloud_athena:configure("test-access-key", "test-secret-key"), ?assertEqual(ExpectedResult, TestedFun()), @@ -319,17 +401,23 @@ do_erlcloud_httpc_request(_, post, Headers, _, _, _) -> ["AmazonAthena", Operation] = string:tokens(Target, "."), RespBody = case Operation of - "BatchGetNamedQuery" -> ?BATCH_GET_NAMED_QUERY_RESP; - "BatchGetQueryExecution" -> ?BATCH_GET_QUERY_EXECUTION_RESP; - "CreateNamedQuery" -> ?CREATE_NAMED_QUERY_RESP; - "DeleteNamedQuery" -> #{}; - "GetNamedQuery" -> ?GET_NAMED_QUERY_RESP; - "GetQueryExecution" -> ?GET_QUERY_EXECUTION_RESP; - "GetQueryResults" -> ?GET_QUERY_RESULTS_RESP; - "ListNamedQueries" -> ?LIST_NAMED_QUERIES_RESP; - "ListQueryExecutions" -> ?LIST_QUERY_EXECUTIONS_RESP; - "StartQueryExecution" -> ?START_QUERY_EXECUTION_RESP; - "StopQueryExecution" -> #{} + "BatchGetNamedQuery" -> ?BATCH_GET_NAMED_QUERY_RESP; + "BatchGetPreparedStatement" -> ?BATCH_GET_PREPARED_STATEMENT_RESP; + "BatchGetQueryExecution" -> ?BATCH_GET_QUERY_EXECUTION_RESP; + "CreateNamedQuery" -> ?CREATE_NAMED_QUERY_RESP; + "CreatePreparedStatement" -> #{}; + "DeleteNamedQuery" -> #{}; + "DeletePreparedStatement" -> #{}; + "GetNamedQuery" -> ?GET_NAMED_QUERY_RESP; + "GetPreparedStatement" -> ?GET_PREPARED_STATEMENT_RESP; + "GetQueryExecution" -> ?GET_QUERY_EXECUTION_RESP; + "GetQueryResults" -> ?GET_QUERY_RESULTS_RESP; + "ListNamedQueries" -> ?LIST_NAMED_QUERIES_RESP; + "ListPreparedStatements" -> ?LIST_PREPARED_STATEMENTS_RESP; + "ListQueryExecutions" -> ?LIST_QUERY_EXECUTIONS_RESP; + "StartQueryExecution" -> ?START_QUERY_EXECUTION_RESP; + "StopQueryExecution" -> #{}; + "UpdatePreparedStatement" -> #{} end, {ok, {{200, "OK"}, [], jsx:encode(RespBody)}}. From b49df76d60c8c8e7e538470f8319b0fd205b714b Mon Sep 17 00:00:00 2001 From: asta-simaityte Date: Fri, 13 Jan 2023 15:28:08 +0000 Subject: [PATCH 261/310] add workgroup option to start, create, list query API calls (#743) * add workgroup option support to create, list athena API calls * update list_named_queries * fix type specifications * update list_query_executions specs --- src/erlcloud_athena.erl | 166 ++++++++++++++++++++++++--------- test/erlcloud_athena_tests.erl | 35 +++++-- 2 files changed, 146 insertions(+), 55 deletions(-) diff --git a/src/erlcloud_athena.erl b/src/erlcloud_athena.erl index a2d43dbbd..9bf4ae4ae 100644 --- a/src/erlcloud_athena.erl +++ b/src/erlcloud_athena.erl @@ -19,6 +19,7 @@ create_named_query/4, create_named_query/5, create_named_query/6, + create_named_query/7, create_prepared_statement/3, create_prepared_statement/4, @@ -46,6 +47,7 @@ list_named_queries/0, list_named_queries/1, list_named_queries/2, + list_named_queries/3, list_prepared_statements/1, list_prepared_statements/2, @@ -54,11 +56,13 @@ list_query_executions/0, list_query_executions/1, list_query_executions/2, + list_query_executions/3, start_query_execution/4, start_query_execution/5, start_query_execution/6, start_query_execution/7, + start_query_execution/8, stop_query_execution/1, stop_query_execution/2, @@ -170,38 +174,53 @@ batch_get_query_execution(QueryExecutionIds, Config) -> %% <<"db-name">>, %% <<"query-name">>, %% <<"select * from some-tbl">>, -%% <<"optional-query-description">> +%% <<"optional-query-description">>, +%% <<"optional-some-workgroup">> %% ). %% ' %% +-type create_named_query_opts() :: [create_named_query_opt()]. +-type create_named_query_opt() :: {workgroup, binary()}. + -spec create_named_query(binary(), binary(), binary(), binary()) -> {ok, binary()} | {error, any()}. create_named_query(ClientReqToken, Db, Name, Query) -> Config = default_config(), - create_named_query(ClientReqToken, Db, Name, Query, undefined, Config). + create_named_query(ClientReqToken, Db, Name, Query, undefined, [], Config). -spec create_named_query(binary(), binary(), binary(), binary(), aws_config() | binary()) -> {ok, binary()} | {error, any()}. create_named_query(ClientReqToken, Db, Name, Query, Config) when is_record(Config, aws_config) -> - create_named_query(ClientReqToken, Db, Name, Query, undefined, Config); + create_named_query(ClientReqToken, Db, Name, Query, undefined, [], Config); create_named_query(ClientReqToken, Db, Name, Query, Description) when is_binary(Description) -> Config = default_config(), - create_named_query(ClientReqToken, Db, Name, Query, Description, Config). + create_named_query(ClientReqToken, Db, Name, Query, Description, [], Config). + +-spec create_named_query(binary(), binary(), binary(), binary(), + binary(), aws_config() | create_named_query_opts()) -> + {ok, binary()} | {error, any()}. +create_named_query(ClientReqToken, Db, Name, Query, Description, Config) + when is_record(Config, aws_config) -> + create_named_query(ClientReqToken, Db, Name, Query, Description, [], Config); +create_named_query(ClientReqToken, Db, Name, Query, Description, Options) + when is_list(Options) -> + create_named_query(ClientReqToken, Db, Name, Query, Description, Options, default_config()). -spec create_named_query(binary(), binary(), binary(), binary(), binary() | undefined, + create_named_query_opts(), aws_config()) -> {ok, binary()} | {error, any()}. -create_named_query(ClientReqToken, Db, Name, Query, Description, Config) -> - Request0 = #{<<"ClientRequestToken">> => ClientReqToken, - <<"Database">> => Db, - <<"Name">> => Name, - <<"QueryString">> => Query}, - Request1 = update_description(Request0, Description), - case request(Config, "CreateNamedQuery", Request1) of +create_named_query(ClientReqToken, Db, Name, Query, Description, Options, Config) -> + Params = encode_params([{description, Description} | Options]), + Request = Params#{<<"ClientRequestToken">> => ClientReqToken, + <<"Database">> => Db, + <<"Name">> => Name, + <<"QueryString">> => Query}, + case request(Config, "CreateNamedQuery", Request) of {ok, Res} -> {ok, maps:get(<<"NamedQueryId">>, Res)}; Error -> Error end. @@ -222,10 +241,10 @@ create_prepared_statement(WorkGroup, StatementName, QueryStatement, Description) -spec create_prepared_statement(binary(), binary(), binary(), binary() | undefined, aws_config()) -> ok | {error, any()}. create_prepared_statement(WorkGroup, StatementName, QueryStatement, Description, Config) -> - Request0 = #{<<"WorkGroup">> => WorkGroup, - <<"StatementName">> => StatementName, - <<"QueryStatement">> => QueryStatement}, - Request = update_description(Request0, Description), + Params = encode_params([{description, Description}]), + Request = Params#{<<"WorkGroup">> => WorkGroup, + <<"StatementName">> => StatementName, + <<"QueryStatement">> => QueryStatement}, case request(Config, "CreatePreparedStatement", Request) of {ok, _} -> ok; Error -> Error @@ -339,22 +358,36 @@ get_query_results(QueryExecutionId, PaginationMap, Config) -> %% %% ` %% erlcloud_athena:list_named_queries(#{<<"MaxResults">> => 1, -%% <<"NextToken">> => <<"some-token">>}). +%% <<"NextToken">> => <<"some-token">>, +%% <<"WorkGroup">> => <<"some-workgroup">>}). %% ' %% +-type list_named_queries_opts() :: [list_named_queries_opt()]. +-type list_named_queries_opt() :: {workgroup, binary()}. + -spec list_named_queries() -> {ok, map()} | {error, any()}. list_named_queries() -> - list_named_queries(#{}, default_config()). + list_named_queries(#{}, [], default_config()). --spec list_named_queries(map() | aws_config()) -> {ok, map()} | {error, any()}. +-spec list_named_queries(map() | aws_config()) -> + {ok, map()} | {error, any()}. list_named_queries(Config) when is_record(Config, aws_config) -> - list_named_queries(#{}, Config); + list_named_queries(#{}, [], Config); list_named_queries(PaginationMap) when is_map(PaginationMap) -> - list_named_queries(PaginationMap, default_config()). + list_named_queries(PaginationMap, [], default_config()). + +-spec list_named_queries(map(), aws_config() | list_named_queries_opts()) -> + {ok, map} | {error, any()}. +list_named_queries(PaginationMap, Config) when is_record(Config, aws_config) -> + list_named_queries(PaginationMap, [], Config); +list_named_queries(PaginationMap, Options) when is_list(Options) -> + list_named_queries(PaginationMap, Options, default_config()). --spec list_named_queries(map(), aws_config()) -> {ok, map} | {error, any()}. -list_named_queries(PaginationMap, Config) -> - request(Config, "ListNamedQueries", PaginationMap). +-spec list_named_queries(map(), list_named_queries_opts(), aws_config()) -> + {ok, map} | {error, any()}. +list_named_queries(PaginationMap, Options, Config) -> + Params = encode_params(Options), + request(Config, "ListNamedQueries", maps:merge(PaginationMap, Params)). %% @doc %% Athena API: @@ -390,21 +423,32 @@ list_prepared_statements(WorkGroup, PaginationMap, Config) -> %% <<"NextToken">> => <<"some-token">>}). %% ' %% +-type list_query_executions_opts() :: [list_query_executions_opt()]. +-type list_query_executions_opt() :: {workgroup, binary()}. + -spec list_query_executions() -> {ok, map()} | {error, any()}. list_query_executions() -> - list_query_executions(#{}, default_config()). + list_query_executions(#{}, [], default_config()). -spec list_query_executions(map() | aws_config()) -> {ok, map()} | {error, any()}. list_query_executions(Config) when is_record(Config, aws_config) -> - list_query_executions(#{}, Config); + list_query_executions(#{}, [], Config); list_query_executions(PaginationMap) when is_map(PaginationMap) -> - list_query_executions(PaginationMap, default_config()). + list_query_executions(PaginationMap, [], default_config()). + +-spec list_query_executions(map(), aws_config() | list_query_executions_opts()) -> + {ok, map()} | {error, any()}. +list_query_executions(PaginationMap, Config) when is_record(Config, aws_config) -> + list_query_executions(PaginationMap, [], Config); +list_query_executions(PaginationMap, Options) when is_list(Options) -> + list_query_executions(PaginationMap, Options, default_config()). --spec list_query_executions(map(), aws_config()) -> +-spec list_query_executions(map(), list_query_executions_opts(), aws_config()) -> {ok, map()} | {error, any()}. -list_query_executions(PaginationMap, Config) -> - request(Config, "ListQueryExecutions", PaginationMap). +list_query_executions(PaginationMap, Options, Config) -> + Params = encode_params(Options), + request(Config, "ListQueryExecutions", maps:merge(PaginationMap, Params)). %% @doc %% Athena API: @@ -417,15 +461,19 @@ list_query_executions(PaginationMap, Config) -> %% <<"select * from some-tbl">>, %% <<"s3://some-bucket">>, %% <<"SSE_KMS">>, -%% <<"some-kms-key-id">>}] +%% <<"some-kms-key-id">>, +%% <<"optional-some-workgroup">>}] %% ). %% ' %% +-type start_query_execution_opts() :: [start_query_execution_opt()]. +-type start_query_execution_opt() :: {workgroup, binary()}. + -spec start_query_execution(binary(), binary(), binary(), binary()) -> {ok, binary()} | {error, any()}. start_query_execution(ClientReqToken, Db, Query, OutputLocation) -> start_query_execution(ClientReqToken, Db, Query, OutputLocation, undefined, - undefined, default_config()). + undefined, [], default_config()). -spec start_query_execution(binary(), binary(), binary(), binary(), aws_config()) -> @@ -433,7 +481,7 @@ start_query_execution(ClientReqToken, Db, Query, OutputLocation) -> start_query_execution(ClientReqToken, Db, Query, OutputLocation, Config) when is_record(Config, aws_config) -> start_query_execution(ClientReqToken, Db, Query, OutputLocation, undefined, - undefined, Config). + undefined, [], Config). -spec start_query_execution(binary(), binary(), binary(), binary(), binary() | undefined, @@ -442,22 +490,40 @@ start_query_execution(ClientReqToken, Db, Query, OutputLocation, Config) start_query_execution(ClientReqToken, Db, Query, OutputLocation, EncryptionOption, KmsKey) -> start_query_execution(ClientReqToken, Db, Query, OutputLocation, - EncryptionOption, KmsKey, default_config()). + EncryptionOption, KmsKey, [], default_config()). + +-spec start_query_execution(binary(), binary(), binary(), binary(), + binary() | undefined, + binary() | undefined, + aws_config() | start_query_execution_opts()) -> + {ok, binary()} | {error, any()}. +start_query_execution(ClientReqToken, Db, Query, OutputLocation, + EncryptionOption, KmsKey, Config) + when is_record(Config, aws_config) -> + start_query_execution(ClientReqToken, Db, Query, OutputLocation, + EncryptionOption, KmsKey, [], Config); +start_query_execution(ClientReqToken, Db, Query, OutputLocation, + EncryptionOption, KmsKey, Opts) + when is_list(Opts) -> + start_query_execution(ClientReqToken, Db, Query, OutputLocation, + EncryptionOption, KmsKey, Opts, default_config()). -spec start_query_execution(binary(), binary(), binary(), binary(), binary() | undefined, binary() | undefined, + start_query_execution_opts(), aws_config()) -> {ok, binary()} | {error, any()}. start_query_execution(ClientReqToken, Db, Query, OutputLocation, - EncryptionOption, KmsKey, Config) -> + EncryptionOption, KmsKey, Options, Config) -> + Params = encode_params(Options), EncryptConfig = get_encrypt_config(EncryptionOption, KmsKey), ResultConfig = EncryptConfig#{<<"OutputLocation">> => OutputLocation}, QueryExecCtxt = #{<<"Database">> => Db}, - Request = #{<<"ClientRequestToken">> => ClientReqToken, - <<"QueryExecutionContext">> => QueryExecCtxt, - <<"QueryString">> => Query, - <<"ResultConfiguration">> => ResultConfig}, + Request = Params#{<<"ClientRequestToken">> => ClientReqToken, + <<"QueryExecutionContext">> => QueryExecCtxt, + <<"QueryString">> => Query, + <<"ResultConfiguration">> => ResultConfig}, case request(Config, "StartQueryExecution", Request) of {ok, Res} -> {ok, maps:get(<<"QueryExecutionId">>, Res)}; Error -> Error @@ -489,10 +555,10 @@ update_prepared_statement(WorkGroup, StatementName, QueryStatement, Description) -spec update_prepared_statement(binary(), binary(), binary(), binary() | undefined, aws_config()) -> ok | {error, any()}. update_prepared_statement(WorkGroup, StatementName, QueryStatement, Description, Config) -> - Request0 = #{<<"WorkGroup">> => WorkGroup, - <<"StatementName">> => StatementName, - <<"QueryStatement">> => QueryStatement}, - Request = update_description(Request0, Description), + Params = encode_params([{description, Description}]), + Request = Params#{<<"WorkGroup">> => WorkGroup, + <<"StatementName">> => StatementName, + <<"QueryStatement">> => QueryStatement}, case request(Config, "UpdatePreparedStatement", Request) of {ok, _} -> ok; Error -> Error @@ -566,6 +632,16 @@ get_url(#aws_config{athena_scheme = Scheme, athena_port = Port}) -> Scheme ++ Host ++ ":" ++ integer_to_list(Port). -update_description(Request, undefined) -> Request; -update_description(Request, Description) -> - maps:put(<<"Description">>, Description, Request). +encode_params(Params) -> + encode_params(Params, []). + +encode_params([], Acc) -> + maps:from_list(Acc); +encode_params([{_, undefined} | T], Acc) -> + encode_params(T, Acc); +encode_params([{description, Description} | T], Acc) when is_binary(Description) -> + encode_params(T, [{<<"Description">>, Description} | Acc]); +encode_params([{workgroup, WorkGroup} | T], Acc) when is_binary(WorkGroup) -> + encode_params(T, [{<<"WorkGroup">>, WorkGroup} | Acc]); +encode_params([Option | _], _Acc) -> + error({erlcloud_athena, {invalid_parameter, Option}}). diff --git a/test/erlcloud_athena_tests.erl b/test/erlcloud_athena_tests.erl index cafb8846d..4d4bebe97 100644 --- a/test/erlcloud_athena_tests.erl +++ b/test/erlcloud_athena_tests.erl @@ -193,6 +193,7 @@ erlcloud_athena_test_() -> fun test_start_query_execution_with_encryption/0, fun test_start_query_execution_without_kms_key/0, fun test_start_query_execution_without_encrypt_option/0, + fun test_start_query_execution_with_workgroup_option/0, fun test_stop_query_execution/0, fun test_error_no_retry/0, fun test_error_retry/0, @@ -229,12 +230,13 @@ test_create_named_query() -> <<"Database">> => ?DB_NAME_1, <<"Name">> => ?QUERY_NAME_1, <<"QueryString">> => ?QUERY_STR_1, - <<"Description">> => ?QUERY_DESC_1}, + <<"Description">> => ?QUERY_DESC_1, + <<"WorkGroup">> => ?WORKGROUP}, Expected = {ok, ?QUERY_ID_1}, TestFun = fun() -> erlcloud_athena:create_named_query(?CLIENT_TOKEN, ?DB_NAME_1, ?QUERY_NAME_1, ?QUERY_STR_1, - ?QUERY_DESC_1) + ?QUERY_DESC_1, [{workgroup, ?WORKGROUP}]) end, do_test(Request, Expected, TestFun). @@ -288,8 +290,8 @@ test_get_query_results() -> test_list_named_queries() -> Expected = {ok, ?LIST_NAMED_QUERIES_RESP}, - TestFun = fun() -> erlcloud_athena:list_named_queries() end, - do_test(#{}, Expected, TestFun). + TestFun = fun() -> erlcloud_athena:list_named_queries(#{}, [{workgroup, ?WORKGROUP}]) end, + do_test(#{<<"WorkGroup">> => ?WORKGROUP}, Expected, TestFun). test_list_prepared_statements() -> Request = #{<<"WorkGroup">> => ?WORKGROUP}, @@ -301,8 +303,8 @@ test_list_query_executions() -> Request = #{<<"MaxResults">> => 1, <<"NextToken">> => ?CLIENT_TOKEN}, Expected = {ok, ?LIST_QUERY_EXECUTIONS_RESP}, - TestFun = fun() -> erlcloud_athena:list_query_executions(Request) end, - do_test(Request, Expected, TestFun). + TestFun = fun() -> erlcloud_athena:list_query_executions(Request, [{workgroup, ?WORKGROUP}]) end, + do_test(Request#{<<"WorkGroup">> => ?WORKGROUP}, Expected, TestFun). test_start_query_execution() -> Request = get_start_query_execution_req(#{}), @@ -348,6 +350,17 @@ test_start_query_execution_without_encrypt_option() -> end, do_test(Request, Expected, TestFun). +test_start_query_execution_with_workgroup_option() -> + Request = get_start_query_execution_req(#{<<"WorkGroup">> => ?WORKGROUP},#{}), + Expected = {ok, ?QUERY_ID_1}, + TestFun = fun() -> + erlcloud_athena:start_query_execution(?CLIENT_TOKEN, ?DB_NAME_1, + ?QUERY_STR_1, ?LOCATION_1, + undefined, undefined, + [{workgroup, ?WORKGROUP}]) + end, + do_test(Request, Expected, TestFun). + test_stop_query_execution() -> Request = #{<<"QueryExecutionId">> => ?QUERY_ID_1}, Expected = ok, @@ -422,8 +435,10 @@ do_erlcloud_httpc_request(_, post, Headers, _, _, _) -> {ok, {{200, "OK"}, [], jsx:encode(RespBody)}}. get_start_query_execution_req(EncryptConfig) -> + get_start_query_execution_req(#{}, EncryptConfig). +get_start_query_execution_req(ReqBody, EncryptConfig) -> ResultConfig = EncryptConfig#{<<"OutputLocation">> => ?LOCATION_1}, - #{<<"ClientRequestToken">> => ?CLIENT_TOKEN, - <<"QueryExecutionContext">> => #{<<"Database">> => ?DB_NAME_1}, - <<"QueryString">> => ?QUERY_STR_1, - <<"ResultConfiguration">> => ResultConfig}. + ReqBody#{<<"ClientRequestToken">> => ?CLIENT_TOKEN, + <<"QueryExecutionContext">> => #{<<"Database">> => ?DB_NAME_1}, + <<"QueryString">> => ?QUERY_STR_1, + <<"ResultConfiguration">> => ResultConfig}. From ab65f405db2fdc7b99b1c7880ae5a94cf59e266e Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Thu, 16 Feb 2023 16:31:04 +0000 Subject: [PATCH 262/310] =?UTF-8?q?Extend=20erlcloud=20with=20Security=20H?= =?UTF-8?q?ub=20module=20that=20implements=20=E2=80=9CDescibe=E2=80=9D=20o?= =?UTF-8?q?peration.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/erlcloud_aws.hrl | 1 + src/erlcloud_aws.erl | 5 ++ src/erlcloud_securityhub.erl | 127 +++++++++++++++++++++++++++++ test/erlcloud_securityhub_test.erl | 84 +++++++++++++++++++ 4 files changed, 217 insertions(+) create mode 100644 src/erlcloud_securityhub.erl create mode 100644 test/erlcloud_securityhub_test.erl diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 572071abd..3a0662ea9 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -20,6 +20,7 @@ -record(aws_config, { access_analyzer_host="access-analyzer.us-east-1.amazonaws.com"::string(), + security_hub_host="securityhub.us-east-1.amazonaws.com"::string(), as_host="autoscaling.amazonaws.com"::string(), ec2_protocol="https"::string(), ec2_host="ec2.amazonaws.com"::string(), diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 2c51e2cb9..4e6b4a33e 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -668,6 +668,11 @@ service_config( Service, Region, Config ) when is_atom(Region) -> service_config( Service, atom_to_binary(Region, latin1), Config ); service_config( Service, Region, Config ) when is_list(Region) -> service_config( Service, list_to_binary(Region), Config ); +service_config( <<"security_hub">>, Region, Config ) -> + service_config( <<"securityhub">>, Region, Config ); +service_config( <<"securityhub">> = Service, Region, Config ) -> + Host = service_host( Service, Region ), + Config#aws_config{security_hub_host = Host}; service_config( <<"access_analyzer">>, Region, Config ) -> service_config( <<"access-analyzer">>, Region, Config ); service_config( <<"access-analyzer">> = Service, Region, Config ) -> diff --git a/src/erlcloud_securityhub.erl b/src/erlcloud_securityhub.erl new file mode 100644 index 000000000..9d8642e2c --- /dev/null +++ b/src/erlcloud_securityhub.erl @@ -0,0 +1,127 @@ +-module(erlcloud_securityhub). + +-include("erlcloud.hrl"). +-include("erlcloud_aws.hrl"). + +%% API +-export([ + describe_hub/1, + describe_hub/2 +]). + +-type param_name() :: binary() | string() | atom(). +-type param_value() :: binary() | string() | atom() | integer(). +-type params() :: [param_name() | {param_name(), param_value()}]. + +-spec describe_hub(AwsConfig) -> Result + when AwsConfig :: aws_config(), + Result :: ok | {error, not_found} | {error, term()}. +describe_hub(AwsConfig) + when is_record(AwsConfig, aws_config) -> + describe_hub(AwsConfig, _Params = []); +describe_hub(Params) -> + AwsConfig = erlcloud_aws:default_config(), + describe_hub(AwsConfig, Params). + +-spec describe_hub(AwsConfig, Params) -> Result + when AwsConfig :: aws_config(), + Params :: params(), + Result :: ok | {error, not_found} | {error, term()}. +describe_hub(AwsConfig, Params) -> + Path = ["accounts"], + case request(AwsConfig, _Method = get, Path, Params) of + {ok, Response} -> + {ok, Response}; + {error, {<<"ResourceNotFoundException">>, _Message}} -> + {error, not_found}; + {error, {<<"InvalidAccessException">>, _Message}} -> + {error, permission_denied}; + {error, Reason} -> + {error, Reason} + end. + + +request(AwsConfig, Method, Path, Params) -> + request(AwsConfig, Method, Path, Params, _RequestBody = <<>>). + +request(AwsConfig0, Method, Path, Params, RequestBody) -> + case erlcloud_aws:update_config(AwsConfig0) of + {ok, AwsConfig1} -> + AwsRequest0 = init_request(AwsConfig1, Method, Path, Params, RequestBody), + AwsRequest1 = erlcloud_retry:request(AwsConfig1, AwsRequest0, fun should_retry/1), + case AwsRequest1#aws_request.response_type of + ok -> + decode_response(AwsRequest1); + error -> + decode_error(AwsRequest1) + end; + {error, Reason} -> + {error, Reason} + end. + +init_request(AwsConfig, Method, Path, Params, Payload) -> + Host = AwsConfig#aws_config.security_hub_host, + Service = "securityhub", + NormPath = norm_path(Path), + NormParams = norm_params(Params), + Region = erlcloud_aws:aws_region_from_host(Host), + Headers = [{"host", Host}, {"content-type", "application/json"}], + SignedHeaders = erlcloud_aws:sign_v4(Method, NormPath, AwsConfig, Headers, Payload, Region, Service, Params), + #aws_request{ + service = securityhub, + method = Method, + uri = "https://" ++ Host ++ NormPath ++ NormParams, + request_headers = SignedHeaders, + request_body = Payload + }. + +norm_path(Path) -> + binary_to_list(iolist_to_binary(["/" | lists:join("/", Path)])). + +norm_params([] = _Params) -> + ""; +norm_params(Params) -> + "?" ++ erlcloud_aws:canonical_query_string(Params). + +should_retry(Request) + when Request#aws_request.response_type == ok -> + Request; +should_retry(Request) + when Request#aws_request.response_type == error, + Request#aws_request.response_status == 429 -> + Request#aws_request{should_retry = true}; +should_retry(Request) + when Request#aws_request.response_type == error, + Request#aws_request.response_status >= 500 -> + Request#aws_request{should_retry = true}; +should_retry(Request) -> + Request#aws_request{should_retry = false}. + +decode_response(AwsRequest) -> + case AwsRequest#aws_request.response_body of + <<>> -> + ok; + ResponseBody -> + Json = jsx:decode(ResponseBody, [{return_maps, false}]), + {ok, Json} + end. + +decode_error(AwsRequest) -> + case AwsRequest#aws_request.error_type of + aws -> + Type = extract_error_type(AwsRequest), + Message = extract_error_message(AwsRequest), + {error, {Type, Message}}; + _ -> + erlcloud_aws:request_to_return(AwsRequest) + end. + +extract_error_type(AwsRequest) -> + Headers = AwsRequest#aws_request.response_headers, + Value = proplists:get_value("x-amzn-errortype", Headers), + iolist_to_binary(Value). + +extract_error_message(AwsRequest) -> + ResponseBody = AwsRequest#aws_request.response_body, + Object = jsx:decode(ResponseBody, [{return_maps, false}]), + proplists:get_value(<<"message">>, Object, <<>>). \ No newline at end of file diff --git a/test/erlcloud_securityhub_test.erl b/test/erlcloud_securityhub_test.erl new file mode 100644 index 000000000..eb6487cd0 --- /dev/null +++ b/test/erlcloud_securityhub_test.erl @@ -0,0 +1,84 @@ +-module(erlcloud_securityhub_test). + +-include_lib("eunit/include/eunit.hrl"). +-include("erlcloud_aws.hrl"). + +-define(TEST_AWS_CONFIG, + #aws_config{ + access_key_id = "TEST_ACCESS_KEY_ID", + secret_access_key = "TEST_ACCESS_KEY", + security_token = "TEST_SECURITY_TOKEN" + } +). + +api_test_() -> + { + foreach, + fun() -> meck:new(erlcloud_httpc) end, + fun(_) -> meck:unload() end, + [fun describe_hub_tests/1] + }. + +describe_hub_tests(_) -> + [ + { + "SecurityHub", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=arn:aws:securityhub:us-east-1:123456789012:hub/default", + Method = get, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseContent = << + "{" + "\"HubArn\":\"arn:aws:securityhub:us-east-1:123456789012:hub/default\"," + "\"AutoEnableControls\":\"true\"," + "\SubscribedAt\":\"2023-02-15T15:46:42.158Z\"" + "}" + >>, + {ok, {{200, "OK"}, [], ResponseContent}} + end + end + ), + + Params = [{"HubArn", "arn:aws:securityhub:us-east-1:123456789012:hub/default"}], + Result = erlcloud_securityhub:describe_hub(AwsConfig, Params), + DescribeHub = [ + [ + {<<"HubArn">>, <<"arn:aws:securityhub:us-east-1:123456789012:hub/default">>}, + {<<"AutoEnableControls">>, true}, + {<<"SubscribedAt">>, <<"2023-02-15T15:46:42.158Z">>} + ] + ], + ?assertEqual({ok, DescribeHub}, Result) + end + }, + { + "SecurityHub -> not_found", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=arn:aws:securityhub:us-east-1:123456789012:hub/", + Method = get, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseHeaders = [{"x-amzn-errortype", "ResourceNotFoundException"}], + ResponseContent = <<"{\"message\": \"not found\"}">>, + {ok, {{404, "NotFound"}, ResponseHeaders, ResponseContent}} + end + end + ), + Params = [{"HubArn", "arn:aws:securityhub:us-east-1:123456789012:hub/"}], + Result = erlcloud_securityhub:describe_hub(AwsConfig, Params), + ?assertEqual({error, not_found}, Result) + end + } + ]. + From 41e1ae4051ded8864493d154fcaa0dd983748900 Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Tue, 21 Feb 2023 09:58:42 +0000 Subject: [PATCH 263/310] =?UTF-8?q?Extend=20erlcloud=20with=20Security=20H?= =?UTF-8?q?ub=20module=20that=20implements=20=E2=80=9CDescibe=E2=80=9D=20o?= =?UTF-8?q?peration.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/erlcloud_securityhub.erl | 11 ++++-- test/erlcloud_securityhub_test.erl | 54 +++++++++++++++--------------- 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/src/erlcloud_securityhub.erl b/src/erlcloud_securityhub.erl index 9d8642e2c..7dfcdb6be 100644 --- a/src/erlcloud_securityhub.erl +++ b/src/erlcloud_securityhub.erl @@ -9,13 +9,15 @@ describe_hub/2 ]). +-type securityhub() :: proplist(). + -type param_name() :: binary() | string() | atom(). -type param_value() :: binary() | string() | atom() | integer(). -type params() :: [param_name() | {param_name(), param_value()}]. -spec describe_hub(AwsConfig) -> Result when AwsConfig :: aws_config(), - Result :: ok | {error, not_found} | {error, term()}. + Result :: {ok, securityhub()} | {error, not_found} | {error, term()}. describe_hub(AwsConfig) when is_record(AwsConfig, aws_config) -> describe_hub(AwsConfig, _Params = []); @@ -26,9 +28,10 @@ describe_hub(Params) -> -spec describe_hub(AwsConfig, Params) -> Result when AwsConfig :: aws_config(), Params :: params(), - Result :: ok | {error, not_found} | {error, term()}. + Result :: {ok, securityhub()}| {error, not_found} | {error, term()}. describe_hub(AwsConfig, Params) -> Path = ["accounts"], + ct:pal("Params ~p",[Params]), case request(AwsConfig, _Method = get, Path, Params) of {ok, Response} -> {ok, Response}; @@ -67,6 +70,8 @@ init_request(AwsConfig, Method, Path, Params, Payload) -> Region = erlcloud_aws:aws_region_from_host(Host), Headers = [{"host", Host}, {"content-type", "application/json"}], SignedHeaders = erlcloud_aws:sign_v4(Method, NormPath, AwsConfig, Headers, Payload, Region, Service, Params), + ct:pal("NormPath ~p",[NormPath]), + ct:pal("NormParams ~p",[NormParams]), #aws_request{ service = securityhub, method = Method, @@ -102,7 +107,9 @@ decode_response(AwsRequest) -> <<>> -> ok; ResponseBody -> + ct:pal("Response ~p",[ResponseBody]), Json = jsx:decode(ResponseBody, [{return_maps, false}]), + ct:pal("Json ~p",[Json]), {ok, Json} end. diff --git a/test/erlcloud_securityhub_test.erl b/test/erlcloud_securityhub_test.erl index eb6487cd0..819694b3d 100644 --- a/test/erlcloud_securityhub_test.erl +++ b/test/erlcloud_securityhub_test.erl @@ -28,7 +28,7 @@ describe_hub_tests(_) -> meck:expect( erlcloud_httpc, request, fun(A1, A2, A3, A4, A5, A6) -> - Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=arn:aws:securityhub:us-east-1:123456789012:hub/default", + Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=arn%3Aaws%3Asecurityhub%3Aus-east-1%3A123456789012%3Ahub%2Fdefault", Method = get, RequestContent = <<>>, case [A1, A2, A3, A4, A5, A6] of @@ -45,40 +45,40 @@ describe_hub_tests(_) -> end ), - Params = [{"HubArn", "arn:aws:securityhub:us-east-1:123456789012:hub/default"}], + Params = [{<<"HubArn">>, <<"arn:aws:securityhub:us-east-1:123456789012:hub/default">>}], Result = erlcloud_securityhub:describe_hub(AwsConfig, Params), DescribeHub = [ - [ {<<"HubArn">>, <<"arn:aws:securityhub:us-east-1:123456789012:hub/default">>}, {<<"AutoEnableControls">>, true}, {<<"SubscribedAt">>, <<"2023-02-15T15:46:42.158Z">>} - ] ], + + ct:pal("Response content ~p",[DescribeHub]), ?assertEqual({ok, DescribeHub}, Result) end - }, - { - "SecurityHub -> not_found", - fun() -> - AwsConfig = ?TEST_AWS_CONFIG, - meck:expect( - erlcloud_httpc, request, - fun(A1, A2, A3, A4, A5, A6) -> - Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=arn:aws:securityhub:us-east-1:123456789012:hub/", - Method = get, - RequestContent = <<>>, - case [A1, A2, A3, A4, A5, A6] of - [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> - ResponseHeaders = [{"x-amzn-errortype", "ResourceNotFoundException"}], - ResponseContent = <<"{\"message\": \"not found\"}">>, - {ok, {{404, "NotFound"}, ResponseHeaders, ResponseContent}} - end - end - ), - Params = [{"HubArn", "arn:aws:securityhub:us-east-1:123456789012:hub/"}], - Result = erlcloud_securityhub:describe_hub(AwsConfig, Params), - ?assertEqual({error, not_found}, Result) - end } +%% { +%% "SecurityHub -> not_found", +%% fun() -> +%% AwsConfig = ?TEST_AWS_CONFIG, +%% meck:expect( +%% erlcloud_httpc, request, +%% fun(A1, A2, A3, A4, A5, A6) -> +%% Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=arn%3Aaws%3Asecurityhub%3Aus-east-1%3A123456789012%3Ahub%2F", +%% Method = get, +%% RequestContent = <<>>, +%% case [A1, A2, A3, A4, A5, A6] of +%% [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> +%% ResponseHeaders = [{"x-amzn-errortype", "ResourceNotFoundException"}], +%% ResponseContent = <<"{\"message\": \"not found\"}">>, +%% {ok, {{404, "NotFound"}, ResponseHeaders, ResponseContent}} +%% end +%% end +%% ), +%% Params = [{<<"HubArn">>, <<"arn:aws:securityhub:us-east-1:352283894008:hub/">>}], +%% Result = erlcloud_securityhub:describe_hub(AwsConfig, Params), +%% ?assertEqual({error, not_found}, Result) +%% end +%% } ]. From a29699e5af63c7d2d45486bdbe97b2d810365337 Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Wed, 22 Feb 2023 14:40:14 +0000 Subject: [PATCH 264/310] =?UTF-8?q?Extend=20erlcloud=20with=20Security=20H?= =?UTF-8?q?ub=20module=20that=20implements=20=E2=80=9CDescibe=E2=80=9D=20o?= =?UTF-8?q?peration.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/erlcloud_securityhub.erl | 5 --- test/erlcloud_securityhub_test.erl | 56 ++++++++++++++---------------- 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/src/erlcloud_securityhub.erl b/src/erlcloud_securityhub.erl index 7dfcdb6be..4746fb6cc 100644 --- a/src/erlcloud_securityhub.erl +++ b/src/erlcloud_securityhub.erl @@ -31,7 +31,6 @@ describe_hub(Params) -> Result :: {ok, securityhub()}| {error, not_found} | {error, term()}. describe_hub(AwsConfig, Params) -> Path = ["accounts"], - ct:pal("Params ~p",[Params]), case request(AwsConfig, _Method = get, Path, Params) of {ok, Response} -> {ok, Response}; @@ -70,8 +69,6 @@ init_request(AwsConfig, Method, Path, Params, Payload) -> Region = erlcloud_aws:aws_region_from_host(Host), Headers = [{"host", Host}, {"content-type", "application/json"}], SignedHeaders = erlcloud_aws:sign_v4(Method, NormPath, AwsConfig, Headers, Payload, Region, Service, Params), - ct:pal("NormPath ~p",[NormPath]), - ct:pal("NormParams ~p",[NormParams]), #aws_request{ service = securityhub, method = Method, @@ -107,9 +104,7 @@ decode_response(AwsRequest) -> <<>> -> ok; ResponseBody -> - ct:pal("Response ~p",[ResponseBody]), Json = jsx:decode(ResponseBody, [{return_maps, false}]), - ct:pal("Json ~p",[Json]), {ok, Json} end. diff --git a/test/erlcloud_securityhub_test.erl b/test/erlcloud_securityhub_test.erl index 819694b3d..356522496 100644 --- a/test/erlcloud_securityhub_test.erl +++ b/test/erlcloud_securityhub_test.erl @@ -28,7 +28,7 @@ describe_hub_tests(_) -> meck:expect( erlcloud_httpc, request, fun(A1, A2, A3, A4, A5, A6) -> - Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=arn%3Aaws%3Asecurityhub%3Aus-east-1%3A123456789012%3Ahub%2Fdefault", + Url = "https://securityhub.us-east-1.amazonaws.com/accounts", Method = get, RequestContent = <<>>, case [A1, A2, A3, A4, A5, A6] of @@ -37,7 +37,7 @@ describe_hub_tests(_) -> "{" "\"HubArn\":\"arn:aws:securityhub:us-east-1:123456789012:hub/default\"," "\"AutoEnableControls\":\"true\"," - "\SubscribedAt\":\"2023-02-15T15:46:42.158Z\"" + "\"SubscribedAt\":\"2023-02-15T15:46:42.158Z\"" "}" >>, {ok, {{200, "OK"}, [], ResponseContent}} @@ -45,40 +45,38 @@ describe_hub_tests(_) -> end ), - Params = [{<<"HubArn">>, <<"arn:aws:securityhub:us-east-1:123456789012:hub/default">>}], - Result = erlcloud_securityhub:describe_hub(AwsConfig, Params), + Result = erlcloud_securityhub:describe_hub(AwsConfig, []), DescribeHub = [ {<<"HubArn">>, <<"arn:aws:securityhub:us-east-1:123456789012:hub/default">>}, - {<<"AutoEnableControls">>, true}, + {<<"AutoEnableControls">>, <<"true">>}, {<<"SubscribedAt">>, <<"2023-02-15T15:46:42.158Z">>} ], - ct:pal("Response content ~p",[DescribeHub]), ?assertEqual({ok, DescribeHub}, Result) end + }, + { + "SecurityHub -> not_found", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, request, + fun(A1, A2, A3, A4, A5, A6) -> + Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=test", + Method = get, + RequestContent = <<>>, + case [A1, A2, A3, A4, A5, A6] of + [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> + ResponseHeaders = [{"x-amzn-errortype", "ResourceNotFoundException"}], + ResponseContent = <<"{\"message\": \"not found\"}">>, + {ok, {{404, "NotFound"}, ResponseHeaders, ResponseContent}} + end + end + ), + Params = [{<<"HubArn">>, <<"test">>}], + Result = erlcloud_securityhub:describe_hub(AwsConfig, Params), + ?assertEqual({error, not_found}, Result) + end } -%% { -%% "SecurityHub -> not_found", -%% fun() -> -%% AwsConfig = ?TEST_AWS_CONFIG, -%% meck:expect( -%% erlcloud_httpc, request, -%% fun(A1, A2, A3, A4, A5, A6) -> -%% Url = "https://securityhub.us-east-1.amazonaws.com/accounts?HubArn=arn%3Aaws%3Asecurityhub%3Aus-east-1%3A123456789012%3Ahub%2F", -%% Method = get, -%% RequestContent = <<>>, -%% case [A1, A2, A3, A4, A5, A6] of -%% [Url, Method, _Headers, RequestContent, _Timeout, AwsConfig] -> -%% ResponseHeaders = [{"x-amzn-errortype", "ResourceNotFoundException"}], -%% ResponseContent = <<"{\"message\": \"not found\"}">>, -%% {ok, {{404, "NotFound"}, ResponseHeaders, ResponseContent}} -%% end -%% end -%% ), -%% Params = [{<<"HubArn">>, <<"arn:aws:securityhub:us-east-1:352283894008:hub/">>}], -%% Result = erlcloud_securityhub:describe_hub(AwsConfig, Params), -%% ?assertEqual({error, not_found}, Result) -%% end -%% } ]. From b4da928932ecb67b3d3c6084b2f15d6c22df703c Mon Sep 17 00:00:00 2001 From: Neil Bleasdale Date: Tue, 21 Feb 2023 15:34:52 +0000 Subject: [PATCH 265/310] Return ipv6 range(s) for security group rules Add test coverage for the describe_security_groups functionality. --- src/erlcloud_ec2.erl | 3 +- test/erlcloud_ec2_tests.erl | 115 +++++++++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index a21ba75c5..dfa554687 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -2088,7 +2088,8 @@ extract_ip_permissions(Node) -> {users, get_list("groups/item/userId", Node)}, {groups, [extract_user_id_group_pair(Item) || Item <- xmerl_xpath:string("groups/item", Node)]}, - {ip_ranges, get_list("ipRanges/item/cidrIp", Node)} + {ip_ranges, get_list("ipRanges/item/cidrIp", Node)}, + {ipv6_ranges, get_list("ipv6Ranges/item/cidrIpv6", Node)} ]. extract_user_id_group_pair(Node) -> diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 7b037f89b..702b11123 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -63,7 +63,9 @@ describe_test_() -> fun describe_flow_logs_input_tests/1, fun describe_flow_logs_output_tests/1, fun describe_launch_template_versions_input_tests/1, - fun describe_launch_template_versions_output_tests/1 + fun describe_launch_template_versions_output_tests/1, + fun describe_security_groups_input_tests/1, + fun describe_security_groups_output_tests/1 ]}. start() -> @@ -1738,6 +1740,117 @@ describe_launch_template_versions_output_tests(_) -> ], output_tests(?_f(erlcloud_ec2:describe_launch_template_versions("lt-08ccaba0746110123")), Tests). +generate_security_group_response() -> + " + 1d62eae0-acdd-481d-88c9-example + + + 123456789012 + sg-9bf6ceff + SSHAccess + Security group for SSH access + vpc-31896b55 + + + tcp + 22 + 22 + + + + 0.0.0.0/0 + + + + + ::/0 + + + + + + + + -1 + + + + 0.0.0.0/0 + + + + + ::/0 + + + + + + + + ". + +describe_security_groups_input_tests(_) -> + Tests = + [?_ec2_test({ + "Describe security group(s), default call with no additional parameters", + ?_f(erlcloud_ec2:describe_security_groups()), + [{"Action", "DescribeSecurityGroups"}]}), + ?_ec2_test({ + "Describe security group(s) matching the given group name", + ?_f(erlcloud_ec2:describe_security_groups(["SSHAccess"])), + [{"Action", "DescribeSecurityGroups"}, + {"GroupName.1", "SSHAccess"}]}), + ?_ec2_test({ + "Describe security group(s) matching the given group ID(s)", + ?_f(erlcloud_ec2:describe_security_groups(["sg-9bf6ceff"], [], none, erlcloud_aws:default_config())), + [{"Action", "DescribeSecurityGroups"}, + {"GroupId.1", "sg-9bf6ceff"}]}), + ?_ec2_test({ + "Describe security group(s) matching the given set of filters", + ?_f(erlcloud_ec2:describe_security_groups([], [], [{ + "ip-permission-ipv6-cidr", ["::/0"]}], erlcloud_aws:default_config())), + [{"Action", "DescribeSecurityGroups"}, + {"Filter.1.Name" ,"ip-permission-ipv6-cidr"}, + {"Filter.1.Value.1", "%3A%3A%2F0"}]}) + ], + Response = generate_security_group_response(), + input_tests(Response, Tests). + +describe_security_groups_output_tests(_) -> + Tests = [ + ?_ec2_test({ + "Coverage for DescribeSecurityGroup API Request", + generate_security_group_response(), + {ok, [[ + {owner_id, "123456789012"}, + {group_id, "sg-9bf6ceff"}, + {group_name, "SSHAccess"}, + {group_description, "Security group for SSH access"}, + {vpc_id, "vpc-31896b55"}, + {ip_permissions, [[ + {ip_protocol, tcp}, + {from_port, 22}, + {to_port, 22}, + {users, []}, + {groups, []}, + {ip_ranges, ["0.0.0.0/0"]}, + {ipv6_ranges, ["::/0"]}] + ]}, + {ip_permissions_egress, [[ + {ip_protocol, '-1'}, + {from_port, 0}, + {to_port, 0}, + {users, []}, + {groups, []}, + {ip_ranges, ["0.0.0.0/0"]}, + {ipv6_ranges, ["::/0"]}] + ]}, + {tag_set, []}]]} + }) + ], + output_tests(?_f(erlcloud_ec2:describe_security_groups()), Tests). + generate_one_instance(N) -> " r-69 From 36f3215bbc79428555c68f7cbeab78c5f8504a23 Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Thu, 23 Feb 2023 11:26:24 +0000 Subject: [PATCH 266/310] =?UTF-8?q?Extend=20erlcloud=20with=20Security=20H?= =?UTF-8?q?ub=20module=20that=20implements=20=E2=80=9CDescibe=E2=80=9D=20o?= =?UTF-8?q?peration.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- include/erlcloud_aws.hrl | 2 +- src/erlcloud_aws.erl | 4 +--- src/erlcloud_securityhub.erl | 4 +--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 3a0662ea9..1761b850a 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -20,7 +20,7 @@ -record(aws_config, { access_analyzer_host="access-analyzer.us-east-1.amazonaws.com"::string(), - security_hub_host="securityhub.us-east-1.amazonaws.com"::string(), + securityhub_host="securityhub.us-east-1.amazonaws.com"::string(), as_host="autoscaling.amazonaws.com"::string(), ec2_protocol="https"::string(), ec2_host="ec2.amazonaws.com"::string(), diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 4e6b4a33e..29fe856d9 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -668,11 +668,9 @@ service_config( Service, Region, Config ) when is_atom(Region) -> service_config( Service, atom_to_binary(Region, latin1), Config ); service_config( Service, Region, Config ) when is_list(Region) -> service_config( Service, list_to_binary(Region), Config ); -service_config( <<"security_hub">>, Region, Config ) -> - service_config( <<"securityhub">>, Region, Config ); service_config( <<"securityhub">> = Service, Region, Config ) -> Host = service_host( Service, Region ), - Config#aws_config{security_hub_host = Host}; + Config#aws_config{securityhub_host = Host}; service_config( <<"access_analyzer">>, Region, Config ) -> service_config( <<"access-analyzer">>, Region, Config ); service_config( <<"access-analyzer">> = Service, Region, Config ) -> diff --git a/src/erlcloud_securityhub.erl b/src/erlcloud_securityhub.erl index 4746fb6cc..65484e5c4 100644 --- a/src/erlcloud_securityhub.erl +++ b/src/erlcloud_securityhub.erl @@ -36,8 +36,6 @@ describe_hub(AwsConfig, Params) -> {ok, Response}; {error, {<<"ResourceNotFoundException">>, _Message}} -> {error, not_found}; - {error, {<<"InvalidAccessException">>, _Message}} -> - {error, permission_denied}; {error, Reason} -> {error, Reason} end. @@ -62,7 +60,7 @@ request(AwsConfig0, Method, Path, Params, RequestBody) -> end. init_request(AwsConfig, Method, Path, Params, Payload) -> - Host = AwsConfig#aws_config.security_hub_host, + Host = AwsConfig#aws_config.securityhub_host, Service = "securityhub", NormPath = norm_path(Path), NormParams = norm_params(Params), From 9b2bc154febbd858ac0b60a648124189f8e8a002 Mon Sep 17 00:00:00 2001 From: Jakub 'King' Kubiak Date: Thu, 30 Mar 2023 19:47:12 +0100 Subject: [PATCH 267/310] typo in s3 mfa_delete attribute --- src/erlcloud_s3.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index b278ad386..c03760c5b 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -621,7 +621,7 @@ get_bucket_attribute(BucketName, AttributeName, Config) [{enabled, true}|erlcloud_xml:decode(Attributes, LoggingEnabled)] end; mfa_delete -> - case erlcloud_xml:get_text("/VersioningConfiguration/MFADelete", Doc) of + case erlcloud_xml:get_text("/VersioningConfiguration/MfaDelete", Doc) of "Enabled" -> enabled; _ -> disabled end; From 199b464abc3830ce0a04521f534adf378ab9e817 Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Mon, 3 Apr 2023 14:58:55 +0100 Subject: [PATCH 268/310] added create_secret_ functions --- src/erlcloud_sm.erl | 132 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl index 3c4f81ae9..acd81f8e2 100644 --- a/src/erlcloud_sm.erl +++ b/src/erlcloud_sm.erl @@ -9,6 +9,8 @@ %%% API -export([ + create_secret_binary/3, create_secret_binary/4, create_secret_binary/5, + create_secret_string/3, create_secret_string/4, create_secret_string/5, get_secret_value/2, get_secret_value/3 ]). @@ -21,6 +23,21 @@ -type get_secret_value_option() :: {version_id | version_stage, binary()}. -type get_secret_value_options() :: [get_secret_value_option()]. +%% replica region is expected to be a proplist of the following tuples: +%% [{<<"KmsKeyId">>, binary()}, {<<"Region">>, binary()}] +-type replica_region() :: [proplist()]. +-type replica_regions() :: [replica_region()]. + +-type create_secret_option() :: {add_replica_regions, replica_regions()} + | {client_request_token, binary()} + | {description, binary()} + | {force_overwrite_replica_secret, boolean()} + | {kms_key_id, binary()} + | {secret_binary, binary()} + | {secret_string, binary()} + | {tags, proplist()}. +-type create_secret_options() :: [create_secret_option()]. + %%%------------------------------------------------------------------------------ %%% Library initialization. %%%------------------------------------------------------------------------------ @@ -51,6 +68,99 @@ new(AccessKeyID, SecretAccessKey, Host, Port) -> sm_port = Port }. + +%%------------------------------------------------------------------------------ +%% CreateSecret - SecretBinary +%%------------------------------------------------------------------------------ +%% @doc +%% Creates a new secret binary. The function internally base64-encodes the binary +%% as it is expected by the AWS SecretManager API, so raw blob is expected +%% to be passed as an attribute. +%% +%% ClientRequestToken is used by AWS for secret versioning purposes. +%% It is recommended to be a UUID type value, and is requred to be between +%% 32 and 64 characters. +%% +%% To store a text secret use CreateSecret - SecretString version of the function +%% instead. +%% +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_CreateSecret.html] +%% +%% Example: +%% Name = <<"my-secret-binary">>, +%% ClientRequestToken = <<"7537a353-0de0-4b98-bf55-f8365821ed36">>, +%% %% some binary to store (say, an RSA private key's exponent) +%% {[_E, _Pub], [_E, _N, Priv, _P1, _P2, _E1, _E2, _C]} = crypto:generate_key(rsa, {2048,65537}), +%% erlcloud_sm:create_secret_binary(Name, ClientRequestToken, Priv). +%% @end +%%------------------------------------------------------------------------------ + +-spec create_secret_binary(Name :: binary(), ClientRequestToken :: binary(), + SecretBinary :: binary()) -> sm_response(). +create_secret_binary(Name, ClientRequestToken, SecretBinary) -> + create_secret_binary(Name, ClientRequestToken, SecretBinary, []). + +-spec create_secret_binary(Name :: binary(), ClientRequestToken :: binary(), + SecretBinary :: binary(), + Opts :: create_secret_options()) -> sm_response(). +create_secret_binary(Name, ClientRequestToken, SecretBinary, Opts) -> + create_secret_binary(Name, ClientRequestToken, SecretBinary, Opts, erlcloud_aws:default_config()). + +-spec create_secret_binary(Name :: binary(), ClientRequestToken :: binary(), + SecretBinary :: binary(), Opts :: create_secret_options(), + Config :: aws_config()) -> sm_response(). +create_secret_binary(Name, ClientRequestToken, SecretBinary, Opts, Config) -> + Secret = {secret_binary, base64:encode(SecretBinary)}, + create_secret(Name, ClientRequestToken, [Secret | Opts], Config). + +%%------------------------------------------------------------------------------ +%% CreateSecret - SecretString +%%------------------------------------------------------------------------------ +%% @doc +%% Creates a new secret string. The API expects SecretString is a text data to +%% encrypt and store in the SecretManager. It is recommended a JSON structure +%% of key/value pairs is used for the secret value. +%% +%% ClientRequestToken is used by AWS for secret versioning purposes. +%% It is recommended to be a UUID type value, and is requred to be between +%% 32 and 64 characters. +%% +%% To store a binary (which will be base64 encoded by the library, as it is +%% expected by AWS SecretManager API), use CreateSecret - SecretBinary version +%% of the function. +%% +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_CreateSecret.html] +%% +%% Example: +%% Name = <<"my-secret-string">>, +%% ClientRequestToken = <<"7537a353-0de0-4b98-bf55-f8365821ed37">>, +%% %% some user/password json to store +%% Secret = jsx:encode(#{<<"user">> => <<"my-user">>, <<"password">> => <<"superSecretPassword">>}), +%% erlcloud_sm:create_secret_string(Name, ClientRequestToken, Secret). +%% +%% @end +%%------------------------------------------------------------------------------ + +-spec create_secret_string(Name :: binary(), ClientRequestToken :: binary(), + SecretString :: binary()) -> sm_response(). +create_secret_string(Name, ClientRequestToken, SecretString) -> + create_secret_string(Name, ClientRequestToken, SecretString, []). + +-spec create_secret_string(Name :: binary(), ClientRequestToken :: binary(), + SecretString :: binary(), Opts :: create_secret_options()) -> sm_response(). +create_secret_string(Name, ClientRequestToken, SecretString, Opts) -> + create_secret_string(Name, ClientRequestToken, SecretString, Opts, erlcloud_aws:default_config()). + + +-spec create_secret_string(Name :: binary(), ClientRequestToken :: binary(), + SecretString :: binary(), Opts :: create_secret_options(), + Config :: aws_config()) -> sm_response(). +create_secret_string(Name, ClientRequestToken, SecretString, Opts, Config) -> + Secret = {secret_string, SecretString}, + create_secret(Name, ClientRequestToken, [Secret | Opts], Config). + %%------------------------------------------------------------------------------ %% GetSecretValue %%------------------------------------------------------------------------------ @@ -81,6 +191,12 @@ get_secret_value(SecretId, Opts, Config) -> %%% Internal Functions %%%------------------------------------------------------------------------------ +create_secret(SecretName, ClientRequestToken, Opts, Config) -> + Opts1 = [{client_request_token, ClientRequestToken} | Opts], + Json = create_secret_payload(SecretName, Opts1), + sm_request(Config, "secretsmanager.CreateSecret", Json). + + sm_request(Config, Operation, Body) -> case erlcloud_aws:update_config(Config) of {ok, Config1} -> @@ -132,3 +248,19 @@ sm_result_fun(#aws_request{response_type = error, sm_result_fun(#aws_request{response_type = error, error_type = aws} = Request) -> Request#aws_request{should_retry = false}. +create_secret_payload(SecretName, Opts) -> + Json = lists:map( + fun + ({add_replica_regions, Val}) -> {<<"AddReplicaRegions">>, Val}; + ({client_request_token, Val}) -> {<<"ClientRequestToken">>, Val}; + ({description, Val}) -> {<<"Description">>, Val}; + ({force_overwrite_replica_secret, Val}) -> {<<"ForceOverwriteReplicaSecret">>, Val}; + ({kms_key_id, Val}) -> {<<"KmsKeyId">>, Val}; + ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; %% note AWS accepts either SecretBinary or SecretString + ({secret_string, Val}) -> {<<"SecretString">>, Val}; %% not both at the same time + ({tags, Val}) -> {<<"Tags">>, Val}; + (Other) -> Other + end, + [{<<"Name">>, SecretName} | Opts]), + Json. + From cdb1fb3667ae7b9795e9fa0c1331d2bc41347900 Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Mon, 3 Apr 2023 16:44:38 +0100 Subject: [PATCH 269/310] Added delete_secret functions --- src/erlcloud_sm.erl | 43 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl index acd81f8e2..8d164e5c9 100644 --- a/src/erlcloud_sm.erl +++ b/src/erlcloud_sm.erl @@ -11,6 +11,7 @@ -export([ create_secret_binary/3, create_secret_binary/4, create_secret_binary/5, create_secret_string/3, create_secret_string/4, create_secret_string/5, + delete_secret/1, delete_secret/2, delete_secret/3, get_secret_value/2, get_secret_value/3 ]). @@ -33,11 +34,15 @@ | {description, binary()} | {force_overwrite_replica_secret, boolean()} | {kms_key_id, binary()} - | {secret_binary, binary()} - | {secret_string, binary()} + | {secret_binary, binary()} %% Note AWS accepts either SecretBinary or SecretString, + | {secret_string, binary()} %% not both at the same time | {tags, proplist()}. -type create_secret_options() :: [create_secret_option()]. +-type delete_secret_option() :: {force_delete_without_recovery, boolean()} %% Note you can't use both this parameter and RecoveryWindowInDays. + | {recovery_window_in_days, pos_integer()}. %% If none of these two options are specified then SM defaults to 30 day recovery window +-type delete_secret_options() :: [delete_secret_option()]. + %%%------------------------------------------------------------------------------ %%% Library initialization. %%%------------------------------------------------------------------------------ @@ -161,6 +166,36 @@ create_secret_string(Name, ClientRequestToken, SecretString, Opts, Config) -> Secret = {secret_string, SecretString}, create_secret(Name, ClientRequestToken, [Secret | Opts], Config). +%%------------------------------------------------------------------------------ +%% DeleteSecret +%%------------------------------------------------------------------------------ +%% @doc +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_DeleteSecret.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec delete_secret(SecretId :: binary()) -> sm_response(). +delete_secret(SecretId) -> + delete_secret(SecretId, []). + +-spec delete_secret(SecretId :: binary(), Opts :: delete_secret_options()) -> sm_response(). +delete_secret(SecretId, Opts) -> + delete_secret(SecretId, Opts, erlcloud_aws:default_config()). + +-spec delete_secret(SecretId :: binary(), Opts :: delete_secret_options(), + Config :: aws_config()) -> sm_response(). +delete_secret(SecretId, Opts, Config) -> + Json = lists:map( + fun + ({force_delete_without_recovery, Val}) -> {<<"ForceDeleteWithoutRecovery">>, Val}; + ({recovery_window_in_days, Val}) -> {<<"RecoveryWindowInDays">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId} | Opts]), + sm_request(Config, "secretsmanager.DeleteSecret", Json). + + %%------------------------------------------------------------------------------ %% GetSecretValue %%------------------------------------------------------------------------------ @@ -256,8 +291,8 @@ create_secret_payload(SecretName, Opts) -> ({description, Val}) -> {<<"Description">>, Val}; ({force_overwrite_replica_secret, Val}) -> {<<"ForceOverwriteReplicaSecret">>, Val}; ({kms_key_id, Val}) -> {<<"KmsKeyId">>, Val}; - ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; %% note AWS accepts either SecretBinary or SecretString - ({secret_string, Val}) -> {<<"SecretString">>, Val}; %% not both at the same time + ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; + ({secret_string, Val}) -> {<<"SecretString">>, Val}; ({tags, Val}) -> {<<"Tags">>, Val}; (Other) -> Other end, From 23fab25497aac644eaf7f8b8baf123c46b917d83 Mon Sep 17 00:00:00 2001 From: KingBrewer Date: Tue, 4 Apr 2023 10:33:58 +0100 Subject: [PATCH 270/310] delete_resource_policy, describe_secret, get_resource_policy, put_resource_policy --- src/erlcloud_sm.erl | 94 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl index 8d164e5c9..c73d41db3 100644 --- a/src/erlcloud_sm.erl +++ b/src/erlcloud_sm.erl @@ -11,8 +11,12 @@ -export([ create_secret_binary/3, create_secret_binary/4, create_secret_binary/5, create_secret_string/3, create_secret_string/4, create_secret_string/5, + delete_resource_policy/1, delete_resource_policy/2, delete_secret/1, delete_secret/2, delete_secret/3, - get_secret_value/2, get_secret_value/3 + describe_secret/1, describe_secret/2, + get_resource_policy/1, get_resource_policy/2, + get_secret_value/2, get_secret_value/3, + put_resource_policy/2, put_resource_policy/3, put_resource_policy/4 ]). %%%------------------------------------------------------------------------------ @@ -43,6 +47,9 @@ | {recovery_window_in_days, pos_integer()}. %% If none of these two options are specified then SM defaults to 30 day recovery window -type delete_secret_options() :: [delete_secret_option()]. +-type put_resource_policy_option() :: {block_public_policy, boolean()}. +-type put_resource_policy_options() :: [put_resource_policy_option()]. + %%%------------------------------------------------------------------------------ %%% Library initialization. %%%------------------------------------------------------------------------------ @@ -166,6 +173,24 @@ create_secret_string(Name, ClientRequestToken, SecretString, Opts, Config) -> Secret = {secret_string, SecretString}, create_secret(Name, ClientRequestToken, [Secret | Opts], Config). +%%------------------------------------------------------------------------------ +%% DeleteResourcePolicy +%%------------------------------------------------------------------------------ +%% @doc +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_DeleteResourcePolicy.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec delete_resource_policy(SecretId :: binary()) -> sm_response(). +delete_resource_policy(SecretId) -> + delete_resource_policy(SecretId, erlcloud_aws:default_config()). + +-spec delete_resource_policy(SecretId :: binary(), Config :: aws_config()) -> sm_response(). +delete_resource_policy(SecretId, Config) -> + Json = [{<<"SecretId">>, SecretId}], + sm_request(Config, "secretsmanager.DeleteResourcePolicy", Json). + %%------------------------------------------------------------------------------ %% DeleteSecret %%------------------------------------------------------------------------------ @@ -195,6 +220,41 @@ delete_secret(SecretId, Opts, Config) -> [{<<"SecretId">>, SecretId} | Opts]), sm_request(Config, "secretsmanager.DeleteSecret", Json). +%%------------------------------------------------------------------------------ +%% DescribeSecret +%%------------------------------------------------------------------------------ +%% @doc +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_DescibeSecret.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec describe_secret(SecretId :: binary()) -> sm_response(). +describe_secret(SecretId) -> + describe_secret(SecretId, erlcloud_aws:default_config()). + +-spec describe_secret(SecretId :: binary(), Config :: aws_config()) -> sm_response(). +describe_secret(SecretId, Config) -> + Json = [{<<"SecretId">>, SecretId}], + sm_request(Config, "secretsmanager.DescribeSecret", Json). + +%%------------------------------------------------------------------------------ +%% GetResourcePolicy +%%------------------------------------------------------------------------------ +%% @doc +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetResourcePolicy.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec get_resource_policy(SecretId :: binary()) -> sm_response(). +get_resource_policy(SecretId) -> + get_resource_policy(SecretId, erlcloud_aws:default_config()). + +-spec get_resource_policy(SecretId :: binary(), Config :: aws_config()) -> sm_response(). +get_resource_policy(SecretId, Config) -> + Json = [{<<"SecretId">>, SecretId}], + sm_request(Config, "secretsmanager.GetResourcePolicy", Json). %%------------------------------------------------------------------------------ %% GetSecretValue @@ -209,7 +269,6 @@ delete_secret(SecretId, Opts, Config) -> get_secret_value(SecretId, Opts) -> get_secret_value(SecretId, Opts, erlcloud_aws:default_config()). - -spec get_secret_value(SecretId :: binary(), Opts :: get_secret_value_options(), Config :: aws_config()) -> sm_response(). get_secret_value(SecretId, Opts, Config) -> @@ -222,6 +281,37 @@ get_secret_value(SecretId, Opts, Config) -> [{<<"SecretId">>, SecretId} | Opts]), sm_request(Config, "secretsmanager.GetSecretValue", Json). +%%------------------------------------------------------------------------------ +%% PutResourcePolicy +%%------------------------------------------------------------------------------ +%% @doc +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_PutResourcePolicy.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec put_resource_policy(SecretId :: binary(), ResourcePolicy :: binary()) -> sm_response(). +put_resource_policy(SecretId, ResourcePolicy) -> + put_resource_policy(SecretId, ResourcePolicy, []). + +-spec put_resource_policy(SecretId :: binary(), ResourcePolicy :: binary(), + Opts :: put_resource_policy_options()) -> sm_response(). +put_resource_policy(SecretId, ResourcePolicy, Opts) -> + put_resource_policy(SecretId, ResourcePolicy, Opts, erlcloud_aws:default_config()). + +-spec put_resource_policy(SecretId :: binary(), ResourcePolicy :: binary(), + Opts :: put_resource_policy_options(), + Config :: aws_config()) -> sm_response(). +put_resource_policy(SecretId, ResourcePolicy, Opts, Config) -> + Json = lists:map( + fun + ({block_public_policy, Val}) -> {<<"BlockPublicPolicy">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId}, {<<"ResourcePolicy">>, ResourcePolicy} | Opts]), + sm_request(Config, "secretsmanager.PutResourcePolicy", Json). + + %%%------------------------------------------------------------------------------ %%% Internal Functions %%%------------------------------------------------------------------------------ From 0fa91a88294fff223aca40308a349b18817654bb Mon Sep 17 00:00:00 2001 From: Jakub 'King' Kubiak Date: Tue, 4 Apr 2023 11:04:13 +0100 Subject: [PATCH 271/310] Update README.md (#749) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7e968df4e..125f82ca2 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ Service APIs implemented: - Simple Notification Service (SNS) - Web Application Firewall (WAF) - AWS Cost and Usage Report API +- AWS Secrets Manager - AWS Systems Manager (SSM) - and more to come From 71492ee5d0cd8aef1058820fa187521b33e7a237 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 16 May 2023 20:12:55 +0300 Subject: [PATCH 272/310] Remove rebar2 compatibility --- .github/workflows/ci.yml | 7 +-- Makefile | 92 ++++------------------------------------ rebar.config.script | 25 ----------- 3 files changed, 10 insertions(+), 114 deletions(-) delete mode 100644 rebar.config.script diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f78254c4f..e944a9028 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ on: - master jobs: ci: - name: Run checks and tests over ${{matrix.otp_vsn}} and ${{matrix.os}} (rebar2? ${{matrix.force_rebar2}}) + name: Run checks and tests over ${{matrix.otp_vsn}} and ${{matrix.os}} runs-on: ${{matrix.os}} container: image: erlang:${{matrix.otp_vsn}} @@ -17,13 +17,10 @@ jobs: matrix: otp_vsn: [19.3, 20.3, 21.3, 22.3, 23.3, 24.0] os: [ubuntu-latest] - force_rebar2: [true, false] - env: - FORCE_REBAR2: ${{matrix.force_rebar2}} steps: - uses: actions/checkout@v2 - run: export - - run: make travis-install + - run: make rebar3-install - run: make warnings - run: make check - run: make eunit diff --git a/Makefile b/Makefile index d9bbad732..61a32fa18 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,6 @@ .PHONY: all get-deps clean compile run eunit check check-eunit doc -# determine which Rebar we want to be running -REBAR2=$(shell which rebar || echo ./rebar) -REBAR3=$(shell which rebar3 || echo ./rebar3) -ifeq ($(FORCE_REBAR2),true) - REBAR=$(REBAR2) - REBAR_VSN=2 -else ifeq ($(REBAR3),) - REBAR=$(REBAR2) - REBAR_VSN=2 -else - REBAR=$(REBAR3) - REBAR_VSN=3 -endif +REBAR=$(shell which rebar3 || echo ./rebar3) CHECK_FILES=\ ebin/*.beam @@ -20,13 +8,7 @@ CHECK_FILES=\ CHECK_EUNIT_FILES=\ .eunit/*.beam - -all: get-deps compile - -get-deps: -ifeq ($(REBAR_VSN),2) - @$(REBAR) get-deps -endif +all: compile clean: @$(REBAR) clean @@ -35,87 +17,29 @@ compile: @$(REBAR) compile run: -ifeq ($(REBAR_VSN),2) - erl -pa deps/*/ebin -pa ./ebin -else $(REBAR) shell -endif - -deps: get-deps -check_warnings: deps -ifeq ($(REBAR_VSN),2) - @$(REBAR) compile -else +check_warnings: @$(REBAR) as warnings compile -endif -warnings: deps -ifeq ($(REBAR_VSN),2) - @WARNINGS_AS_ERRORS=true $(REBAR) compile - @WARNINGS_AS_ERRORS=true $(REBAR) compile_only=true eunit -else +warnings: @$(REBAR) as test compile -endif -eunit: deps -ifeq ($(REBAR_VSN),2) - $(MAKE) compile - @$(REBAR) eunit skip_deps=true -else +eunit: @ERL_FLAGS="-config $(PWD)/eunit" $(REBAR) eunit -endif .dialyzer_plt: - dialyzer --build_plt -r deps \ + dialyzer --build_plt -r _build/default \ --apps erts kernel stdlib inets crypto public_key ssl xmerl \ --fullpath \ --output_plt .dialyzer_plt -check: deps -ifeq ($(REBAR_VSN),2) - $(MAKE) compile - @$(REBAR) compile_only=true eunit - $(MAKE) .dialyzer_plt - dialyzer --no_check_plt --fullpath \ - $(CHECK_EUNIT_FILES) \ - -I include \ - --plt .dialyzer_plt -else +check: .dialyzer_plt @$(REBAR) as test dialyzer -endif doc: -ifeq ($(REBAR_VSN),2) - @$(REBAR) doc skip_deps=true -else @$(REBAR) edoc -endif -# The "install" step for Travis -travis-install: -ifeq ($(FORCE_REBAR2),true) - rebar get-deps -else +rebar3-install: wget https://s3.amazonaws.com/rebar3/rebar3 chmod a+x rebar3 -endif - -travis-publish: - @echo Create directories - mkdir -p ~/.hex - mkdir -p ~/.config/rebar3 - - @echo Decrypt secrets - @openssl aes-256-cbc -K $encrypted_9abc06b32f03_key -iv $encrypted_9abc06b32f03_iv -in hex.config.enc -out ~/.hex/hex.config -d - - @echo Create global config - echo '{plugins, [rebar3_hex]}.' > ~/.config/rebar3/rebar.config - - @echo Edit version tag in app.src - vi -e -c '%s/{vsn, *.*}/{vsn, "'${TRAVIS_TAG}'"}/g|w|q' src/erlcloud.app.src - - @echo Publish to Hex - echo 'Y' | ./rebar3 hex publish - - @echo Done diff --git a/rebar.config.script b/rebar.config.script deleted file mode 100644 index 618083bf3..000000000 --- a/rebar.config.script +++ /dev/null @@ -1,25 +0,0 @@ -%% -*- mode: erlang; -*- -case erlang:function_exported(rebar3, main, 1) of - true -> % rebar3 - CONFIG; - false -> % rebar 2.x or older - %% Use git-based deps - %% profiles - CONFIG_DEPS = - [{deps, [{meck, ".*",{git, "https://github.com/eproxus/meck.git", {tag, "0.9.0"}}}, - {jsx, ".*", {git, "https://github.com/talentdeficit/jsx.git", {tag, "v2.11.0"}}}, - %% {hackney, ".*", {git, "git://github.com/benoitc/hackney.git", {tag, "1.2.0"}}}, - {eini, ".*", {git, "https://github.com/erlcloud/eini.git", {tag, "1.2.9"}}}, - {lhttpc, ".*", {git, "https://github.com/erlcloud/lhttpc.git", {tag, "1.6.2"}}}, - {base16, ".*", {git, "https://github.com/goj/base16.git", {tag, "1.0.0"}}}]} - | lists:keydelete(deps, 1, CONFIG)], - CONFIG_NEW = - case os:getenv("WARNINGS_AS_ERRORS") of - "true" -> - [{erl_opts, [warnings_as_errors | proplists:get_value(erl_opts, CONFIG_DEPS)]} - | lists:keydelete(erl_opts, 1, CONFIG_DEPS)]; - _ -> - CONFIG_DEPS - end, - CONFIG_NEW -end. From 85e7c9864cb571effcc0b9f2bcfbff3b362bed0a Mon Sep 17 00:00:00 2001 From: Dinis Rosario Date: Mon, 29 May 2023 09:45:20 +0100 Subject: [PATCH 273/310] Support IMDSv2 session token authentication --- src/erlcloud_ec2_meta.erl | 64 ++++++++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 14 deletions(-) diff --git a/src/erlcloud_ec2_meta.erl b/src/erlcloud_ec2_meta.erl index aaae48c70..a9d8d3e3e 100644 --- a/src/erlcloud_ec2_meta.erl +++ b/src/erlcloud_ec2_meta.erl @@ -3,11 +3,29 @@ -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). --export([get_instance_metadata/0, get_instance_metadata/1, get_instance_metadata/2, - get_instance_user_data/0, get_instance_user_data/1, - get_instance_dynamic_data/0, get_instance_dynamic_data/1, get_instance_dynamic_data/2]). +-export([generate_session_token/1, generate_session_token/2, + get_instance_metadata/0, get_instance_metadata/1, get_instance_metadata/2, get_instance_metadata/3, + get_instance_user_data/0, get_instance_user_data/1, get_instance_user_data/2, + get_instance_dynamic_data/0, get_instance_dynamic_data/1, get_instance_dynamic_data/2, get_instance_dynamic_data/3]). +-spec generate_session_token(DurationSecs :: non_neg_integer()) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. +generate_session_token(DurationSecs) -> + generate_session_token(DurationSecs, erlcloud_aws:default_config()). + +%%%--------------------------------------------------------------------------- +-spec generate_session_token(DurationSecs :: non_neg_integer(), Config:: aws_config()) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. +%%%--------------------------------------------------------------------------- +%% @doc Generate session token Will fail if not an EC2 instance. +%% +%% This convenience function will generate the IMDSv2 session token from the AWS metadata available at +%% http:///latest/latest/api/token +%% ItemPath allows fetching specific pieces of metadata. +%% defaults to 169.254.169.254 +generate_session_token(DurationSecs, Config) -> + Header = [{"X-aws-ec2-metadata-token-ttl-seconds", integer_to_binary(DurationSecs)}], + MetaDataPath = "http://" ++ ec2_meta_host_port() ++ "/latest/api/token", + erlcloud_aws:http_body(erlcloud_httpc:request(MetaDataPath, put, Header, <<>>, erlcloud_aws:get_timeout(Config), Config)). -spec get_instance_metadata() -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. @@ -18,40 +36,49 @@ get_instance_metadata() -> get_instance_metadata(Config) -> get_instance_metadata("", Config). +-spec get_instance_metadata( ItemPath :: string(), Config :: aws_config() ) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. +get_instance_metadata(ItemPath, Config) -> + get_instance_metadata(ItemPath, Config, undefined). %%%--------------------------------------------------------------------------- --spec get_instance_metadata( ItemPath :: string(), Config :: aws_config() ) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. +-spec get_instance_metadata( ItemPath :: string(), Config :: aws_config(), Token :: undefined | binary()) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. %%%--------------------------------------------------------------------------- %% @doc Retrieve the instance meta data for the instance this code is running on. Will fail if not an EC2 instance. %% -%% This convenience function will retrieve the instance id from the AWS metadata available at +%% This convenience function will retrieve the instance id from the AWS metadata available at %% http:///latest/meta-data/* %% ItemPath allows fetching specific pieces of metadata. %% defaults to 169.254.169.254 %% %% -get_instance_metadata(ItemPath, Config) -> +get_instance_metadata(ItemPath, Config, Token) -> MetaDataPath = "http://" ++ ec2_meta_host_port() ++ "/latest/meta-data/" ++ ItemPath, - erlcloud_aws:http_body(erlcloud_httpc:request(MetaDataPath, get, [], <<>>, erlcloud_aws:get_timeout(Config), Config)). + Header = maybe_token_header(Token), + erlcloud_aws:http_body(erlcloud_httpc:request(MetaDataPath, get, Header, <<>>, erlcloud_aws:get_timeout(Config), Config)). -spec get_instance_user_data() -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. get_instance_user_data() -> get_instance_user_data(erlcloud_aws:default_config()). -%%%--------------------------------------------------------------------------- -spec get_instance_user_data( Config :: aws_config() ) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. +get_instance_user_data(Config) -> + get_instance_user_data(Config, undefined). + +%%%--------------------------------------------------------------------------- +-spec get_instance_user_data( Config :: aws_config(), Token :: undefined | binary() ) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. %%%--------------------------------------------------------------------------- %% @doc Retrieve the user data for the instance this code is running on. Will fail if not an EC2 instance. %% -%% This convenience function will retrieve the user data the instance was started with, i.e. what's available at +%% This convenience function will retrieve the user data the instance was started with, i.e. what's available at %% http:///latest/user-data %% defaults to 169.254.169.254 %% %% -get_instance_user_data(Config) -> +get_instance_user_data(Config, Token) -> UserDataPath = "http://" ++ ec2_meta_host_port() ++ "/latest/user-data/", - erlcloud_aws:http_body(erlcloud_httpc:request(UserDataPath, get, [], <<>>, erlcloud_aws:get_timeout(Config), Config)). + Header = maybe_token_header(Token), + erlcloud_aws:http_body(erlcloud_httpc:request(UserDataPath, get, Header, <<>>, erlcloud_aws:get_timeout(Config), Config)). -spec get_instance_dynamic_data() -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. @@ -62,14 +89,18 @@ get_instance_dynamic_data() -> get_instance_dynamic_data(Config) -> get_instance_dynamic_data("", Config). +-spec get_instance_dynamic_data( ItemPath :: string(), Config :: aws_config() ) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. +get_instance_dynamic_data(ItemPath, Config) -> + get_instance_dynamic_data(ItemPath, Config, undefined). %%%--------------------------------------------------------------------------- --spec get_instance_dynamic_data( ItemPath :: string(), Config :: aws_config() ) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. +-spec get_instance_dynamic_data( ItemPath :: string(), Config :: aws_config(), Token :: undefined | binary() ) -> {ok, binary()} | {error, erlcloud_aws:httpc_result_error()}. %%%--------------------------------------------------------------------------- -get_instance_dynamic_data(ItemPath, Config) -> +get_instance_dynamic_data(ItemPath, Config, Token) -> DynamicDataPath = "http://" ++ ec2_meta_host_port() ++ "/latest/dynamic/" ++ ItemPath, - erlcloud_aws:http_body(erlcloud_httpc:request(DynamicDataPath, get, [], <<>>, erlcloud_aws:get_timeout(Config), Config)). + Header = maybe_token_header(Token), + erlcloud_aws:http_body(erlcloud_httpc:request(DynamicDataPath, get, Header, <<>>, erlcloud_aws:get_timeout(Config), Config)). %%%------------------------------------------------------------------------------ %%% Internal functions. @@ -78,3 +109,8 @@ get_instance_dynamic_data(ItemPath, Config) -> ec2_meta_host_port() -> {ok, EC2MetaHostPort} = application:get_env(erlcloud, ec2_meta_host_port), EC2MetaHostPort. + +maybe_token_header(undefined) -> + []; +maybe_token_header(Token) -> + [{"X-aws-ec2-metadata-token", Token}]. From b6b63e295ce64925b82a6ee9e17e3949b20702c1 Mon Sep 17 00:00:00 2001 From: Dinis Rosario Date: Wed, 7 Jun 2023 11:33:50 +0100 Subject: [PATCH 274/310] Use IMDSv2 to access EC2 metadata --- src/erlcloud_aws.erl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 29fe856d9..86e555f7d 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -931,7 +931,9 @@ get_availability_zone() -> cache_instance_metadata_availability_zone() -> % it fine to use default here - no IAM is used, only for http client % one cannot use auto_config()/default_cfg() as it creates an infinite recursion. - case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", #aws_config{}) of + Config = #aws_config{}, + IMDSv2Token = maybe_imdsv2_session_token(Config), + case erlcloud_ec2_meta:get_instance_metadata("placement/availability-zone", Config, IMDSv2Token) of {ok, AZ} = OkResult -> application:set_env(erlcloud, availability_zone, AZ), OkResult; @@ -974,13 +976,14 @@ timestamp_to_gregorian_seconds(Timestamp) -> get_credentials_from_metadata(Config) -> %% TODO this function should retry on errors getting credentials %% First get the list of roles - case erlcloud_ec2_meta:get_instance_metadata("iam/security-credentials/", Config) of + IMDSv2Token = maybe_imdsv2_session_token(Config), + case erlcloud_ec2_meta:get_instance_metadata("iam/security-credentials/", Config, IMDSv2Token) of {error, Reason} -> {error, Reason}; {ok, Body} -> %% Always use the first role [Role | _] = binary:split(Body, <<$\n>>), - case erlcloud_ec2_meta:get_instance_metadata("iam/security-credentials/" ++ binary_to_list(Role), Config) of + case erlcloud_ec2_meta:get_instance_metadata("iam/security-credentials/" ++ binary_to_list(Role), Config, IMDSv2Token) of {error, Reason} -> {error, Reason}; {ok, Json} -> @@ -1562,3 +1565,10 @@ get_env_for_role_credentials(Arn, ExtId) -> set_env_for_role_credentials(Arn, ExtId, Val) -> % application:set_env is undocumented in regards to type(Par) =/= atom() application:set_env(erlcloud, {role_credentials, Arn, ExtId}, Val). + +-spec maybe_imdsv2_session_token(aws_config()) -> binary() | undefined. +maybe_imdsv2_session_token(Config) -> + case erlcloud_ec2_meta:generate_session_token(60, Config) of + {ok, Token} -> Token; + _Error -> undefined + end. From a544adf544de7a15197c6732eac15e8dd36c3c94 Mon Sep 17 00:00:00 2001 From: Dinis Rosario Date: Wed, 7 Jun 2023 13:34:17 +0100 Subject: [PATCH 275/310] Fix broken eunit test --- test/erlcloud_aws_tests.erl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/erlcloud_aws_tests.erl b/test/erlcloud_aws_tests.erl index b6e60d21e..6796b8947 100644 --- a/test/erlcloud_aws_tests.erl +++ b/test/erlcloud_aws_tests.erl @@ -915,8 +915,12 @@ service_config_waf_test() -> get_host_vpc_endpoint_setup_fun() -> meck:new(erlcloud_ec2_meta), + meck:expect(erlcloud_ec2_meta, generate_session_token, + fun(60, #aws_config{}) -> + {ok, <<"60_seconds_imdsv2_token">>} + end), meck:expect(erlcloud_ec2_meta, get_instance_metadata, - fun("placement/availability-zone", #aws_config{}) -> + fun("placement/availability-zone", #aws_config{}, <<"60_seconds_imdsv2_token">>) -> {ok, <<"us-east-1a">>} end). From 3d303e76fc891b05ad878f2073cacf39f793edb9 Mon Sep 17 00:00:00 2001 From: Paulo Zulato Date: Wed, 12 Jul 2023 16:15:07 -0300 Subject: [PATCH 276/310] feat(kinesis): make `Retry` configurable --- include/erlcloud_aws.hrl | 2 +- src/erlcloud_kinesis_impl.erl | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 1761b850a..0068854ed 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -79,7 +79,7 @@ kinesis_scheme="https://"::string(), kinesis_host="kinesis.us-east-1.amazonaws.com"::string(), kinesis_port=80::non_neg_integer(), - kinesis_retry=fun erlcloud_kinesis_impl:retry/2::erlcloud_kinesis_impl:retry_fun(), + kinesis_retry=fun erlcloud_kinesis_impl:retry/3::erlcloud_kinesis_impl:retry_fun(), glue_scheme="https://"::string(), glue_host="glue.us-east-1.amazonaws.com"::string(), glue_port=443::non_neg_integer(), diff --git a/src/erlcloud_kinesis_impl.erl b/src/erlcloud_kinesis_impl.erl index bdc36f42a..04490e5cf 100644 --- a/src/erlcloud_kinesis_impl.erl +++ b/src/erlcloud_kinesis_impl.erl @@ -38,7 +38,7 @@ -include("erlcloud_aws.hrl"). %% Helpers --export([backoff/1, retry/2]). +-export([backoff/1, retry/2, retry/3]). %% Internal impl api -export([request/3, request/4]). @@ -89,11 +89,15 @@ backoff(Attempt) -> timer:sleep(erlcloud_util:rand_uniform((1 bsl (Attempt - 1)) * 100)). -type attempt() :: {attempt, pos_integer()} | {error, term()}. --type retry_fun() :: fun((pos_integer(), term()) -> attempt()). +-type retry_fun() :: fun((pos_integer(), non_neg_integer(), term()) -> attempt()). -spec retry(pos_integer(), term()) -> attempt(). -retry(Attempt, Reason) when Attempt >= ?NUM_ATTEMPTS -> +retry(Attempt, Reason) -> + retry(Attempt, ?NUM_ATTEMPTS, Reason). + +-spec retry(pos_integer(), pos_integer(), term()) -> attempt(). +retry(Attempt, MaxAttempt, Reason) when Attempt > MaxAttempt -> {error, Reason}; -retry(Attempt, _) -> +retry(Attempt, _, _) -> backoff(Attempt), {attempt, Attempt + 1}. @@ -108,6 +112,7 @@ request_and_retry(_, _, _, _, {error, Reason}) -> {error, Reason}; request_and_retry(Config, Headers, Body, ShouldDecode, {attempt, Attempt}) -> RetryFun = Config#aws_config.kinesis_retry, + MaxAttempt = Config#aws_config.retry_num, case erlcloud_httpc:request( url(Config), post, [{<<"content-type">>, <<"application/x-amz-json-1.1">>} | Headers], @@ -123,20 +128,20 @@ request_and_retry(Config, Headers, Body, ShouldDecode, {attempt, Attempt}) -> {ok, {{Status, StatusLine}, _, RespBody}} when Status >= 400 andalso Status < 500 -> case client_error(Status, StatusLine, RespBody) of {retry, Reason} -> - request_and_retry(Config, Headers, Body, ShouldDecode, RetryFun(Attempt, Reason)); + request_and_retry(Config, Headers, Body, ShouldDecode, RetryFun(Attempt, MaxAttempt, Reason)); {error, Reason} -> {error, Reason} end; {ok, {{Status, StatusLine}, _, RespBody}} when Status >= 500 -> - request_and_retry(Config, Headers, Body, ShouldDecode, RetryFun(Attempt, {http_error, Status, StatusLine, RespBody})); + request_and_retry(Config, Headers, Body, ShouldDecode, RetryFun(Attempt, MaxAttempt, {http_error, Status, StatusLine, RespBody})); {ok, {{Status, StatusLine}, _, RespBody}} -> {error, {http_error, Status, StatusLine, RespBody}}; {error, Reason} -> %% TODO there may be some http errors, such as certificate error, that we don't want to retry - request_and_retry(Config, Headers, Body, ShouldDecode, RetryFun(Attempt, Reason)) + request_and_retry(Config, Headers, Body, ShouldDecode, RetryFun(Attempt, MaxAttempt, Reason)) end. -spec client_error(pos_integer(), string(), binary()) -> {retry, term()} | {error, term()}. From 1d1cc4a1b1136758d49d30a3b90f252b388d1825 Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Tue, 12 Sep 2023 09:45:14 +0100 Subject: [PATCH 277/310] Handle ipv6 destination cidr block for Route table discovery. --- src/erlcloud_ec2.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index dfa554687..564ae99f9 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -2005,6 +2005,7 @@ extract_route(Node) -> extract_route_set(Node) -> [ {destination_cidr_block, get_text("destinationCidrBlock", Node)}, + {destination_ipv6_cidr_block, get_text("destinationIpv6CidrBlock", Node)}, {gateway_id, get_text("gatewayId", Node)}, {nat_gateway_id, get_text("natGatewayId", Node)}, {instance_id, get_text("instanceId", Node)}, From 839bfa2650f5a3ce7e909f582e515b8cfd6d424c Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Tue, 12 Sep 2023 09:59:15 +0100 Subject: [PATCH 278/310] Handle ipv6 destination cidr block for Route table discovery. --- test/erlcloud_ec2_tests.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 702b11123..10e97c7eb 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -992,6 +992,7 @@ describe_route_tables_tests() -> {vpc_id,"vpc-012ff464df6bbf762"}, {route_set, [[{destination_cidr_block,"10.0.0.0/26"}, + {destination_ipv6_cidr_block,[]}, {gateway_id,"local"}, {nat_gateway_id,[]}, {instance_id,[]}, @@ -1000,6 +1001,7 @@ describe_route_tables_tests() -> {state,"active"}, {origin,"CreateRouteTable"}], [{destination_cidr_block,"0.0.0.0/8"}, + {destination_ipv6_cidr_block,[]}, {gateway_id,[]}, {nat_gateway_id,"nat-06e45944bb42d3ba2"}, {instance_id,[]}, @@ -1008,6 +1010,7 @@ describe_route_tables_tests() -> {state,"active"}, {origin,"CreateRoute"}], [{destination_cidr_block,"0.0.0.0/0"}, + {destination_ipv6_cidr_block,[]}, {gateway_id,"vgw-09fea3ed190b9ea1e"}, {nat_gateway_id,[]}, {instance_id,[]}, From ed3d2fefc246fd1ead237ca8ef6f2ef9b64fb42a Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Wed, 13 Sep 2023 10:53:11 +0100 Subject: [PATCH 279/310] Handle ipv6 destination cidr block for Route table discovery. --- test/erlcloud_ec2_tests.erl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 10e97c7eb..e3f073b1d 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -956,6 +956,12 @@ describe_route_tables_tests() -> active CreateRoute + + ::/64 + igw-1456db71 + active + CreateRoute + @@ -1017,7 +1023,16 @@ describe_route_tables_tests() -> {vpc_peering_conn_id,[]}, {network_interface_id,[]}, {state,"active"}, - {origin,"CreateRoute"}]]}, + {origin,"CreateRoute"}], + [{destination_cidr_block,[]}, + {destination_ipv6_cidr_block,"::/64"}, + {gateway_id,"igw-1456db71"}, + {nat_gateway_id,[]}, + {instance_id,[]}, + {vpc_peering_conn_id,[]}, + {network_interface_id,[]}, + {state,"active"}, + {origin,"CreateRoute"}]]}, {association_set, [[{route_table_association_id,"rtbassoc-123"}, {route_table_id,"rtb-567"}, From d60b547f92501aed65d33c5c56e0ab9ee9aa3b66 Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Wed, 13 Sep 2023 11:17:27 +0100 Subject: [PATCH 280/310] Handle ipv6 destination cidr block for Route table discovery. --- test/erlcloud_ec2_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index e3f073b1d..f342c6e02 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -957,7 +957,7 @@ describe_route_tables_tests() -> CreateRoute - ::/64 + ::/64 igw-1456db71 active CreateRoute From de3a26a6b6605d8ed9c6334bf6ade4454d45b3bc Mon Sep 17 00:00:00 2001 From: "hardi.rachh" Date: Wed, 13 Sep 2023 11:27:35 +0100 Subject: [PATCH 281/310] Handle ipv6 destination cidr block for Route table discovery. --- test/erlcloud_ec2_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index f342c6e02..4b241583c 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -957,7 +957,7 @@ describe_route_tables_tests() -> CreateRoute - ::/64 + ::/64 igw-1456db71 active CreateRoute From e5e24cbd84af5d9115097648a2b7265d62d32b78 Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Thu, 21 Sep 2023 10:31:44 -0500 Subject: [PATCH 282/310] Add hex.pm publishing section to readme, badges, make target * Add hex.pm version and license badges to readme * Add short hex.pm publishing section to readme * Add hex-publish make target to publish a new hex package from the latest version of the libary --- Makefile | 5 ++++- README.md | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 61a32fa18..ea73a90a9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all get-deps clean compile run eunit check check-eunit doc +.PHONY: all get-deps clean compile run eunit check check-eunit doc hex-publish rebar3-install REBAR=$(shell which rebar3 || echo ./rebar3) @@ -40,6 +40,9 @@ check: .dialyzer_plt doc: @$(REBAR) edoc +hex-publish: + @$(REBAR) hex publish + rebar3-install: wget https://s3.amazonaws.com/rebar3/rebar3 chmod a+x rebar3 diff --git a/README.md b/README.md index 125f82ca2..013596034 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # erlcloud: AWS APIs library for Erlang -[![Build Status](https://github.com/erlcloud/erlcloud/workflows/build/badge.svg)](https://github.com/erlcloud/erlcloud) +[![Build Status](https://github.com/erlcloud/erlcloud/workflows/build/badge.svg)](https://github.com/erlcloud/erlcloud/actions/workflows/ci.yml) +[![Hex.pm](https://img.shields.io/hexpm/v/erlcloud)](https://hex.pm/packages/erlcloud) +[![Hex.pm](https://img.shields.io/hexpm/l/erlcloud)](COPYRIGHT) + This library is not developed or maintained by AWS thus lots of functionality is still missing comparing to [aws-cli](https://aws.amazon.com/cli/) or @@ -105,8 +108,10 @@ make make run ``` -If you're using erlcloud in your application, add it as a dependency in your -application's configuration file. To use erlcloud in the shell, you can start +To use erlcloud in your application, add it as a dependency in your +application's configuration file. erlcloud is also available as a [Hex](https://hex.pm) package, refer to the Hex [`mix` usage docs](https://hex.pm/docs/usage) or [`rebar3` usage docs](https://hex.pm/docs/rebar3-usage) for more help including dependencies using Hex syntax. + +To use erlcloud in the shell, you can start it by calling: ```erlang @@ -284,3 +289,10 @@ as provided by OTP team. ddb2, ecs. - Library should not expose any long running or stateful processes - no gen_servers, no caches and etc. + + +## Publishing to hex.pm + +erlcloud is available as a [Hex](https://hex.pm) package. A new version of the package can be published by maintainers using mix or rebar3. A `hex-publish` make target that uses rebar3 is provided for maintainers to use or reference when publishing a new version of the package. + +Github Actions will eventually be used to automatically publish new versions. \ No newline at end of file From 236e5fe631ab9fd6777ec7e3702b88a5dc363383 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Carvalho?= Date: Thu, 26 Oct 2023 14:15:53 +0100 Subject: [PATCH 283/310] Add secretsmanager.PutSecretValue --- src/erlcloud_sm.erl | 67 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl index c73d41db3..af1bd0372 100644 --- a/src/erlcloud_sm.erl +++ b/src/erlcloud_sm.erl @@ -16,7 +16,9 @@ describe_secret/1, describe_secret/2, get_resource_policy/1, get_resource_policy/2, get_secret_value/2, get_secret_value/3, - put_resource_policy/2, put_resource_policy/3, put_resource_policy/4 + put_resource_policy/2, put_resource_policy/3, put_resource_policy/4, + put_secret_binary/3, put_secret_binary/4, put_secret_binary/5, + put_secret_string/3, put_secret_string/4, put_secret_string/5 ]). %%%------------------------------------------------------------------------------ @@ -50,6 +52,12 @@ -type put_resource_policy_option() :: {block_public_policy, boolean()}. -type put_resource_policy_options() :: [put_resource_policy_option()]. +-type put_secret_value_option() :: {client_request_token, binary()} + | {secret_binary, binary()} + | {secret_string, binary()} + | {version_stages, [binary()]}. +-type put_secret_value_options() :: [put_secret_value_option()]. + %%%------------------------------------------------------------------------------ %%% Library initialization. %%%------------------------------------------------------------------------------ @@ -311,6 +319,47 @@ put_resource_policy(SecretId, ResourcePolicy, Opts, Config) -> [{<<"SecretId">>, SecretId}, {<<"ResourcePolicy">>, ResourcePolicy} | Opts]), sm_request(Config, "secretsmanager.PutResourcePolicy", Json). +%%------------------------------------------------------------------------------ +%% PutSecretValue +%%------------------------------------------------------------------------------ +%% @doc +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_PutSecretValue.html] +%% @end +%%------------------------------------------------------------------------------ +-spec put_secret_string(SecretId :: binary(), ClientRequestToken :: binary(), + SecretString :: binary()) -> sm_response(). +put_secret_string(SecretId, ClientRequestToken, SecretString) -> + put_secret_string(SecretId, ClientRequestToken, SecretString, []). + +-spec put_secret_string(SecretId :: binary(), ClientRequestToken :: binary(), + SecretString :: binary(), Opts :: put_secret_value_options()) -> sm_response(). +put_secret_string(SecretId, ClientRequestToken, SecretString, Opts) -> + put_secret_string(SecretId, ClientRequestToken, SecretString, Opts, erlcloud_aws:default_config()). + +-spec put_secret_string(SecretId :: binary(), ClientRequestToken :: binary(), + SecretString :: binary(), Opts :: put_secret_value_options(), + Config :: aws_config()) -> sm_response(). +put_secret_string(SecretId, ClientRequestToken, SecretString, Opts, Config) -> + Secret = {secret_string, SecretString}, + put_secret(SecretId, ClientRequestToken, [Secret | Opts], Config). + +-spec put_secret_binary(SecretId :: binary(), ClientRequestToken :: binary(), + SecretBinary :: binary()) -> sm_response(). +put_secret_binary(SecretId, ClientRequestToken, SecretBinary) -> + put_secret_binary(SecretId, ClientRequestToken, SecretBinary, []). + +-spec put_secret_binary(SecretId :: binary(), ClientRequestToken :: binary(), + SecretBinary :: binary(), Opts :: put_secret_value_options()) -> sm_response(). +put_secret_binary(SecretId, ClientRequestToken, SecretBinary, Opts) -> + put_secret_binary(SecretId, ClientRequestToken, SecretBinary, Opts, erlcloud_aws:default_config()). + +-spec put_secret_binary(SecretId :: binary(), ClientRequestToken :: binary(), + SecretBinary :: binary(), Opts :: put_secret_value_options(), + Config :: aws_config()) -> sm_response(). +put_secret_binary(SecretId, ClientRequestToken, SecretBinary, Opts, Config) -> + Secret = {secret_binary, base64:encode(SecretBinary)}, + put_secret(SecretId, ClientRequestToken, [Secret | Opts], Config). %%%------------------------------------------------------------------------------ %%% Internal Functions @@ -321,6 +370,11 @@ create_secret(SecretName, ClientRequestToken, Opts, Config) -> Json = create_secret_payload(SecretName, Opts1), sm_request(Config, "secretsmanager.CreateSecret", Json). +put_secret(SecretId, ClientRequestToken, Opts, Config) -> + Opts1 = [{client_request_token, ClientRequestToken} | Opts], + Json = put_secret_payload(SecretId, Opts1), + sm_request(Config, "secretsmanager.PutSecretValue", Json). + sm_request(Config, Operation, Body) -> case erlcloud_aws:update_config(Config) of @@ -389,3 +443,14 @@ create_secret_payload(SecretName, Opts) -> [{<<"Name">>, SecretName} | Opts]), Json. +put_secret_payload(SecretId, Opts) -> + Json = lists:map( + fun + ({client_request_token, Val}) -> {<<"ClientRequestToken">>, Val}; + ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; + ({secret_string, Val}) -> {<<"SecretString">>, Val}; + ({version_stages, Val}) -> {<<"VersionStages">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId} | Opts]), + Json. From a07a7730c6738b6b6a8417362f270cdadb0b270b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Carvalho?= Date: Thu, 2 Nov 2023 20:19:49 +0000 Subject: [PATCH 284/310] Handle put secret options inside put_secret method. --- src/erlcloud_sm.erl | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl index af1bd0372..c1189dda5 100644 --- a/src/erlcloud_sm.erl +++ b/src/erlcloud_sm.erl @@ -371,8 +371,14 @@ create_secret(SecretName, ClientRequestToken, Opts, Config) -> sm_request(Config, "secretsmanager.CreateSecret", Json). put_secret(SecretId, ClientRequestToken, Opts, Config) -> - Opts1 = [{client_request_token, ClientRequestToken} | Opts], - Json = put_secret_payload(SecretId, Opts1), + Json = lists:map( + fun + ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; + ({secret_string, Val}) -> {<<"SecretString">>, Val}; + ({version_stages, Val}) -> {<<"VersionStages">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId}, {<<"ClientRequestToken">>, ClientRequestToken} | Opts]), sm_request(Config, "secretsmanager.PutSecretValue", Json). @@ -442,15 +448,3 @@ create_secret_payload(SecretName, Opts) -> end, [{<<"Name">>, SecretName} | Opts]), Json. - -put_secret_payload(SecretId, Opts) -> - Json = lists:map( - fun - ({client_request_token, Val}) -> {<<"ClientRequestToken">>, Val}; - ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; - ({secret_string, Val}) -> {<<"SecretString">>, Val}; - ({version_stages, Val}) -> {<<"VersionStages">>, Val}; - (Other) -> Other - end, - [{<<"SecretId">>, SecretId} | Opts]), - Json. From cd76feae566572a2d38c2041491e504ebce642d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Carvalho?= Date: Mon, 13 Nov 2023 16:01:49 +0000 Subject: [PATCH 285/310] Add unit tests for put_secret_ string and binary --- test/erlcloud_sm_tests.erl | 49 +++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/test/erlcloud_sm_tests.erl b/test/erlcloud_sm_tests.erl index 2d37fd2c4..7761c988c 100644 --- a/test/erlcloud_sm_tests.erl +++ b/test/erlcloud_sm_tests.erl @@ -23,6 +23,9 @@ %%%=================================================================== -define(SECRET_ID, <<"MyTestDatabaseSecret">>). +-define(SECRET_STRING, <<"{\"username\":\"sergio\",\"password\":\"SECRET-PASSWORD\"}">>). +-define(SECRET_BINARY, base64:encode(?SECRET_STRING)). +-define(CLIENT_REQUEST_TOKEN, <<"EXAMPLE2-90ab-cdef-fedc-ba987EXAMPLE">>). -define(VERSION_ID, <<"EXAMPLE1-90ab-cdef-fedc-ba987SECRET1">>). -define(VERSION_STAGE, <<"AWSPREVIOUS">>). @@ -36,7 +39,9 @@ operation_test_() -> fun stop/1, [ fun get_secret_value_input_tests/1, - fun get_secret_value_output_tests/1 + fun get_secret_value_output_tests/1, + fun put_secret_value_input_tests/1, + fun put_secret_value_output_tests/1 ]}. start() -> @@ -185,3 +190,45 @@ get_secret_value_output_tests(_) -> output_tests(?_f(erlcloud_sm:get_secret_value(?SECRET_ID, [{version_id, ?VERSION_ID}])), Tests), output_tests(?_f(erlcloud_sm:get_secret_value(?SECRET_ID, [{version_stage, ?VERSION_STAGE}])), Tests). + +put_secret_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"put_secret_string input test", + ?_f(erlcloud_sm:put_secret_string(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}, + {<<"SecretString">>,?SECRET_STRING}]) + }), + ?_sm_test( + {"put_secret_binary input test", + ?_f(erlcloud_sm:put_secret_binary(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}, + {<<"SecretBinary">>,?SECRET_BINARY}]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + +-define(PUT_SECRET_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"Name">>, ?SECRET_ID}, + {<<"VersionId">>, ?VERSION_ID}, + {<<"VersionStages">>, [?VERSION_STAGE]} +]). + + +put_secret_value_output_tests(_) -> + Tests = [?_sm_test( + {"put_secret_string output test", + jsx:encode(?PUT_SECRET_VALUE_RESP), + {ok, ?PUT_SECRET_VALUE_RESP}} + )], + + output_tests(?_f(erlcloud_sm:put_secret_string(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), Tests), + output_tests(?_f(erlcloud_sm:put_secret_binary(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), Tests). From 8d181f8ab01c08f6ada9b81722d533499522f5f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9rgio=20Carvalho?= Date: Mon, 13 Nov 2023 16:03:22 +0000 Subject: [PATCH 286/310] Add secret string as from the AWS examples. --- test/erlcloud_sm_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/erlcloud_sm_tests.erl b/test/erlcloud_sm_tests.erl index 7761c988c..7c88ef7a5 100644 --- a/test/erlcloud_sm_tests.erl +++ b/test/erlcloud_sm_tests.erl @@ -23,7 +23,7 @@ %%%=================================================================== -define(SECRET_ID, <<"MyTestDatabaseSecret">>). --define(SECRET_STRING, <<"{\"username\":\"sergio\",\"password\":\"SECRET-PASSWORD\"}">>). +-define(SECRET_STRING, <<"{\"username\":\"david\",\"password\":\"SECRET-PASSWORD\"}">>). -define(SECRET_BINARY, base64:encode(?SECRET_STRING)). -define(CLIENT_REQUEST_TOKEN, <<"EXAMPLE2-90ab-cdef-fedc-ba987EXAMPLE">>). -define(VERSION_ID, <<"EXAMPLE1-90ab-cdef-fedc-ba987SECRET1">>). From eabe97cc1770fc8cdeb7fede743cf7362f38e05c Mon Sep 17 00:00:00 2001 From: Ariel Date: Sat, 9 Dec 2023 00:59:09 +0100 Subject: [PATCH 287/310] Fix for https://github.com/erlcloud/erlcloud/issues/741 --- src/erlcloud_s3.erl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index c03760c5b..7835e7197 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -2161,6 +2161,10 @@ s3_endpoint_for_region(RegionName) -> case RegionName of "us-east-1" -> "s3-external-1.amazonaws.com"; + "cn-northwest-1" -> + "s3.cn-northwest-1.amazonaws.com.cn"; + "cn-north-1" -> + "s3.cn-north-1.amazonaws.com.cn"; _ -> lists:flatten(["s3-", RegionName, ".amazonaws.com"]) end. From 791fe2cd4e9a06e26878a3bca85ba5ce0dd986f6 Mon Sep 17 00:00:00 2001 From: Ariel Date: Sun, 10 Dec 2023 23:34:30 +0100 Subject: [PATCH 288/310] Fix for https://github.com/erlcloud/erlcloud/issues/721 --- src/erlcloud_cloudformation.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_cloudformation.erl b/src/erlcloud_cloudformation.erl index 24592c733..97887dd79 100644 --- a/src/erlcloud_cloudformation.erl +++ b/src/erlcloud_cloudformation.erl @@ -491,8 +491,9 @@ list_all(Fun, Params, Config, Acc) -> undefined -> lists:foldl(fun erlang:'++'/2, [], [Data | Acc]); _ -> - list_all(Fun, [{next_token, NextToken} | Params], - Config, [Data | Acc]) + list_all(Fun, + [convert_param({next_token, NextToken}) | Params], + Config, [Data | Acc]) end; {error, Reason} -> {error, Reason} From a398a53cd2670159c66e8b1e32aa7237e37757c3 Mon Sep 17 00:00:00 2001 From: Ariel Otilibili Date: Thu, 21 Dec 2023 19:59:18 +0100 Subject: [PATCH 289/310] Instead of SHA-1, sign_get/4 uses SHA256; solves #705 --- src/erlcloud_s3.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 7835e7197..674bd99d6 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -1076,7 +1076,7 @@ sign_get(Expire_time, BucketName, Key, Config) SecurityToken -> "x-amz-security-token:" ++ SecurityToken ++ "\n" end, To_sign = lists:flatten(["GET\n\n\n", Expires, "\n", SecurityTokenToSign, "/", BucketName, "/", Key]), - Sig = base64:encode(erlcloud_util:sha_mac(Config#aws_config.secret_access_key, To_sign)), + Sig = base64:encode(erlcloud_util:sha256_mac(Config#aws_config.secret_access_key, To_sign)), {Sig, Expires}. -spec make_link(integer(), string(), string()) -> {integer(), string(), string()}. From 70bef29f8fa8614b73e60140b5fe3df9eae22dc6 Mon Sep 17 00:00:00 2001 From: Carl Isom Date: Wed, 17 Jan 2024 17:43:26 -0600 Subject: [PATCH 290/310] Add support for DeletionProtectionEnabled in erlcloud_ddb2 --- include/erlcloud_ddb2.hrl | 1 + src/erlcloud_ddb2.erl | 14 ++++- test/erlcloud_ddb2_tests.erl | 104 ++++++++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/include/erlcloud_ddb2.hrl b/include/erlcloud_ddb2.hrl index 390971539..8f63d9060 100644 --- a/include/erlcloud_ddb2.hrl +++ b/include/erlcloud_ddb2.hrl @@ -156,6 +156,7 @@ {attribute_definitions :: undefined | erlcloud_ddb2:attr_defs(), billing_mode_summary :: undefined | #ddb2_billing_mode_summary{}, creation_date_time :: undefined | number(), + deletion_protection_enabled :: undefined | boolean(), global_secondary_indexes :: undefined | [#ddb2_global_secondary_index_description{}], global_table_version :: undefined | binary(), item_count :: undefined | integer(), diff --git a/src/erlcloud_ddb2.erl b/src/erlcloud_ddb2.erl index 9ee1f75ce..13ac0b528 100644 --- a/src/erlcloud_ddb2.erl +++ b/src/erlcloud_ddb2.erl @@ -168,6 +168,7 @@ delete_item_opts/0, delete_item_return/0, delete_table_return/0, + deletion_protection_enabled/0, describe_global_table_return/0, describe_global_table_settings_return/0, describe_table_return/0, @@ -382,6 +383,8 @@ default_config() -> erlcloud_aws:default_config(). {attr_name(), [in_attr_value(),...], in}. -type conditions() :: maybe_list(condition()). +-type deletion_protection_enabled() :: boolean_opt(deletion_protection_enabled). + -type select() :: all_attributes | all_projected_attributes | count | specific_attributes. -type return_consumed_capacity() :: none | total | indexes. @@ -1439,6 +1442,7 @@ table_description_record() -> [{<<"AttributeDefinitions">>, #ddb2_table_description.attribute_definitions, fun undynamize_attr_defs/2}, {<<"BillingModeSummary">>, #ddb2_table_description.billing_mode_summary, fun undynamize_billing_mode_summary/2}, {<<"CreationDateTime">>, #ddb2_table_description.creation_date_time, fun id/2}, + {<<"DeletionProtectionEnabled">>, #ddb2_table_description.deletion_protection_enabled, fun id/2}, {<<"GlobalSecondaryIndexes">>, #ddb2_table_description.global_secondary_indexes, fun(V, Opts) -> [undynamize_record(global_secondary_index_description_record(), I, Opts) || I <- V] end}, {<<"GlobalTableVersion">>, #ddb2_table_description.global_table_version, fun id/2}, @@ -1892,7 +1896,8 @@ dynamize_sse_specification({enabled, Enabled}) when is_boolean(Enabled) -> {global_secondary_indexes, global_secondary_indexes()} | {provisioned_throughput, {read_units(), write_units()}} | {sse_specification, sse_specification()} | - {stream_specification, stream_specification()}. + {stream_specification, stream_specification()} | + boolean_opt(deletion_protection_enabled). -type create_table_opts() :: [create_table_opt()]. -spec create_table_opts(key_schema()) -> opt_table(). @@ -1904,7 +1909,8 @@ create_table_opts(KeySchema) -> fun dynamize_global_secondary_indexes/1}, {provisioned_throughput, <<"ProvisionedThroughput">>, fun dynamize_provisioned_throughput/1}, {sse_specification, <<"SSESpecification">>, fun dynamize_sse_specification/1}, - {stream_specification, <<"StreamSpecification">>, fun dynamize_stream_specification/1}]. + {stream_specification, <<"StreamSpecification">>, fun dynamize_stream_specification/1}, + {deletion_protection_enabled, <<"DeletionProtectionEnabled">>, fun id/1}]. -spec create_table_record() -> record_desc(). create_table_record() -> @@ -4271,6 +4277,7 @@ dynamize_replication_group_updates(Updates) -> {attribute_definitions, attr_defs()} | {global_secondary_index_updates, global_secondary_index_updates()} | {stream_specification, stream_specification()} | + boolean_opt(deletion_protection_enabled) | out_opt(). -type update_table_opts() :: [update_table_opt()]. @@ -4282,7 +4289,8 @@ update_table_opts() -> {global_secondary_index_updates, <<"GlobalSecondaryIndexUpdates">>, fun dynamize_global_secondary_index_updates/1}, {stream_specification, <<"StreamSpecification">>, fun dynamize_stream_specification/1}, - {replica_updates, <<"ReplicaUpdates">>, fun dynamize_replication_group_updates/1}]. + {replica_updates, <<"ReplicaUpdates">>, fun dynamize_replication_group_updates/1}, + {deletion_protection_enabled, <<"DeletionProtectionEnabled">>, fun id/1}]. -spec update_table_record() -> record_desc(). update_table_record() -> diff --git a/test/erlcloud_ddb2_tests.erl b/test/erlcloud_ddb2_tests.erl index c82047f99..cb72fdc3b 100644 --- a/test/erlcloud_ddb2_tests.erl +++ b/test/erlcloud_ddb2_tests.erl @@ -1371,6 +1371,95 @@ create_table_input_tests(_) -> \"KeyType\": \"HASH\" } ] +}" + }), + ?_ddb_test( + {"CreateTable with deletion_protection_enabled = true", + ?_f(erlcloud_ddb2:create_table( + <<"Thread">>, + [{<<"ForumName">>, s}, + {<<"Subject">>, s}, + {<<"LastPostDateTime">>, s}], + {<<"ForumName">>, <<"Subject">>}, + 5, + 5, + [{local_secondary_indexes, + [{<<"LastPostIndex">>, <<"LastPostDateTime">>, keys_only}]}, + {global_secondary_indexes, + [{<<"SubjectIndex">>, {<<"Subject">>, <<"LastPostDateTime">>}, keys_only, 10, 5}]}, + {deletion_protection_enabled, true}] + )), " +{ + \"AttributeDefinitions\": [ + { + \"AttributeName\": \"ForumName\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"Subject\", + \"AttributeType\": \"S\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"AttributeType\": \"S\" + } + ], + \"GlobalSecondaryIndexes\": [ + { + \"IndexName\": \"SubjectIndex\", + \"KeySchema\": [ + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"ProjectionType\": \"KEYS_ONLY\" + }, + \"ProvisionedThroughput\": { + \"ReadCapacityUnits\": 10, + \"WriteCapacityUnits\": 5 + } + } + ], + \"DeletionProtectionEnabled\": true, + \"TableName\": \"Thread\", + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"Subject\", + \"KeyType\": \"RANGE\" + } + ], + \"LocalSecondaryIndexes\": [ + { + \"IndexName\": \"LastPostIndex\", + \"KeySchema\": [ + { + \"AttributeName\": \"ForumName\", + \"KeyType\": \"HASH\" + }, + { + \"AttributeName\": \"LastPostDateTime\", + \"KeyType\": \"RANGE\" + } + ], + \"Projection\": { + \"ProjectionType\": \"KEYS_ONLY\" + } + } + ], + \"ProvisionedThroughput\": { + \"ReadCapacityUnits\": 5, + \"WriteCapacityUnits\": 5 + } }" }) ], @@ -3086,7 +3175,8 @@ describe_table_output_tests(_) -> }, \"TableName\": \"Thread\", \"TableSizeBytes\": 0, - \"TableStatus\": \"ACTIVE\" + \"TableStatus\": \"ACTIVE\", + \"DeletionProtectionEnabled\": true } }", {ok, #ddb2_table_description @@ -3132,7 +3222,8 @@ describe_table_output_tests(_) -> replica_status = active}], table_name = <<"Thread">>, table_size_bytes = 0, - table_status = active}}}) + table_status = active, + deletion_protection_enabled = true}}}) ], output_tests(?_f(erlcloud_ddb2:describe_table(<<"name">>)), Tests). @@ -6688,6 +6779,15 @@ update_table_input_tests(_) -> { \"TableName\": \"Thread\", \"BillingMode\": \"PAY_PER_REQUEST\" +}" + }), + ?_ddb_test( + {"UpdateTable example request deletion_protection_enabled = false", + ?_f(erlcloud_ddb2:update_table(<<"Thread">>, + [{deletion_protection_enabled, false}])), " +{ + \"TableName\": \"Thread\", + \"DeletionProtectionEnabled\": false }" }) ], From 2cbada9866c5af4713ca99c918a933d6a8d38953 Mon Sep 17 00:00:00 2001 From: Ariel Otilibili Date: Thu, 18 Jan 2024 13:37:33 +0100 Subject: [PATCH 291/310] Removed self-references Found with this command: ``` git grep -n ':' | perl -nE 'print if m;/([\w_]+)\.erl:.*[^\w_]\g1:[\w_]+\(; && !/%/' ``` --- src/erlcloud_ddb.erl | 2 +- src/erlcloud_kinesis.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_ddb.erl b/src/erlcloud_ddb.erl index 46590c89c..f1d095069 100644 --- a/src/erlcloud_ddb.erl +++ b/src/erlcloud_ddb.erl @@ -664,7 +664,7 @@ batch_get_item_record() -> end} ]}. --type batch_get_item_return() :: ddb_return(#ddb_batch_get_item{}, [erlcloud_ddb:out_item()]). +-type batch_get_item_return() :: ddb_return(#ddb_batch_get_item{}, [out_item()]). -spec batch_get_item(batch_get_item_request_items()) -> batch_get_item_return(). batch_get_item(RequestItems) -> diff --git a/src/erlcloud_kinesis.erl b/src/erlcloud_kinesis.erl index 15f1c9aa0..533b3be27 100644 --- a/src/erlcloud_kinesis.erl +++ b/src/erlcloud_kinesis.erl @@ -969,7 +969,7 @@ list_all_tags_pagination_test_() -> meck:sequence(EK, request, 3, [{ok, [{<<"HasMoreTags">>, true}, {<<"Tags">>, Tags1}]}, {ok, [{<<"HasMoreTags">>, false}, {<<"Tags">>, Tags2}]}]), - Result = erlcloud_kinesis:list_all_tags_for_stream(<<"stream">>), + Result = list_all_tags_for_stream(<<"stream">>), meck:unload(EK), ?_assertEqual({ok, [{<<"k1">>, <<"v1">>}, {<<"k2">>, <<"v2">>}, From 1671aee0e9bac9ea3029ddb8bb978ebe4e91ce38 Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Wed, 30 Oct 2024 15:46:51 -0500 Subject: [PATCH 292/310] Bump lhttpc to 1.7.1 * Pull in lhttpc ssl fixes for later versions of OTP --- rebar.config | 2 +- rebar.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index 499ecd44c..26be7dad7 100644 --- a/rebar.config +++ b/rebar.config @@ -18,7 +18,7 @@ {deps, [ {jsx, "2.11.0"}, - {lhttpc, "1.6.2"}, + {lhttpc, "1.7.1"}, {eini, "1.2.9"}, {base16, "1.0.0"} ]}. diff --git a/rebar.lock b/rebar.lock index a955d139a..9d23881cd 100644 --- a/rebar.lock +++ b/rebar.lock @@ -2,11 +2,11 @@ [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.9">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.11.0">>},0}, - {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.6.2">>},0}]}. + {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.7.1">>},0}]}. [ {pkg_hash,[ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, {<<"eini">>, <<"FCC3CBD49BBDD9A1D9735C7365DAFFCD84481CCE81E6CB80537883AA44AC4895">>}, {<<"jsx">>, <<"08154624050333919B4AC1B789667D5F4DB166DC50E190C4D778D1587F102EE0">>}, - {<<"lhttpc">>, <<"044F16F0018C7AA7E945E9E9406C7F6035E0B8BC08BF77B00C78CE260E1071E3">>}]} + {<<"lhttpc">>, <<"8522AF9877765C33318A3AE486BE69BC165E835D05C3334A8166FD7B318D446B">>}]} ]. From 7ba9f7b45a974f7bc2a8c8d5d036d162867cadb5 Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Wed, 30 Oct 2024 15:56:20 -0500 Subject: [PATCH 293/310] Bump actions/checkout to v4 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e944a9028..ccfd576b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: otp_vsn: [19.3, 20.3, 21.3, 22.3, 23.3, 24.0] os: [ubuntu-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - run: export - run: make rebar3-install - run: make warnings From 1b84baed69e76b7e04bf6441abc55eb33e7ad4da Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Wed, 30 Oct 2024 15:57:52 -0500 Subject: [PATCH 294/310] Update OTP versions in GHA runner due to GLIBC issues * There are clashes between expected GLIBC libraries in Github Actions' checkout action and the runner environments that limit the runs to newer versions of OTP --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccfd576b3..20788f7ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: image: erlang:${{matrix.otp_vsn}} strategy: matrix: - otp_vsn: [19.3, 20.3, 21.3, 22.3, 23.3, 24.0] + otp_vsn: [24, 25, 26, 27] os: [ubuntu-latest] steps: - uses: actions/checkout@v4 From 2b8ce3fe86a675c5f4d72e38eaf0fb58fab0b372 Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Wed, 30 Oct 2024 22:12:26 -0500 Subject: [PATCH 295/310] Update for OTP26+ dialyzer, multi-strings * Dialyzer refactor: wrap with rebar3's dialyzer config and use a rebar3 profile with hackney included for checking, use map/0 type instead of maps:map/0 type * Stop using split strings that are ambiguous with OTP 27+ triple-quoted strings (only used currently in unit tests) --- Makefile | 12 +- rebar.config | 15 +- rebar.lock | 9 +- src/erlcloud_cognito_user_pools.erl | 24 +- src/erlcloud_util.erl | 8 +- test/erlcloud_lambda_tests.erl | 2001 +++++++++++++++++---------- 6 files changed, 1284 insertions(+), 785 deletions(-) diff --git a/Makefile b/Makefile index ea73a90a9..0fa960c03 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all get-deps clean compile run eunit check check-eunit doc hex-publish rebar3-install +.PHONY: all get-deps clean compile run eunit check doc hex-publish rebar3-install REBAR=$(shell which rebar3 || echo ./rebar3) @@ -28,14 +28,8 @@ warnings: eunit: @ERL_FLAGS="-config $(PWD)/eunit" $(REBAR) eunit -.dialyzer_plt: - dialyzer --build_plt -r _build/default \ - --apps erts kernel stdlib inets crypto public_key ssl xmerl \ - --fullpath \ - --output_plt .dialyzer_plt - -check: .dialyzer_plt - @$(REBAR) as test dialyzer +check: + @$(REBAR) as dialyzer do dialyzer --update-plt doc: @$(REBAR) edoc diff --git a/rebar.config b/rebar.config index 26be7dad7..937a7ceac 100644 --- a/rebar.config +++ b/rebar.config @@ -32,10 +32,19 @@ {profiles, [ - {test, [{deps, [{meck, "0.9.0"}]}, {erl_opts, [warnings_as_errors]}]} - ,{warnings, [{erl_opts, [warnings_as_errors]}]} + {dialyzer, [{deps, [hackney]}, {erl_opts, [warnings_as_errors]}]}, + {test, [{deps, [{meck, "0.9.0"}]}, {erl_opts, [warnings_as_errors]}]}, + {warnings, [{erl_opts, [warnings_as_errors]}]} ]}. -{post_hooks, [{clean, "rm -f .dialyzer_plt"}]}. +{project_plugins, [erlfmt]}. + +{post_hooks, [{clean, "rm -rf .dialyzer_plt"}]}. {pre_hooks, [{clean, "rm -rf erl_crash.dump *.log"}]}. + +{dialyzer, [ + {plt_location, ".dialyzer_plt"}, + {plt_apps, all_deps}, + {plt_extra_apps, [hackney]} + ]}. \ No newline at end of file diff --git a/rebar.lock b/rebar.lock index 9d23881cd..12f8f3b74 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,4 +1,4 @@ -{"1.1.0", +{"1.2.0", [{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.9">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.11.0">>},0}, @@ -8,5 +8,10 @@ {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, {<<"eini">>, <<"FCC3CBD49BBDD9A1D9735C7365DAFFCD84481CCE81E6CB80537883AA44AC4895">>}, {<<"jsx">>, <<"08154624050333919B4AC1B789667D5F4DB166DC50E190C4D778D1587F102EE0">>}, - {<<"lhttpc">>, <<"8522AF9877765C33318A3AE486BE69BC165E835D05C3334A8166FD7B318D446B">>}]} + {<<"lhttpc">>, <<"8522AF9877765C33318A3AE486BE69BC165E835D05C3334A8166FD7B318D446B">>}]}, +{pkg_hash_ext,[ + {<<"base16">>, <<"02AFD0827E61A7B07093873E063575CA3A2B07520567C7F8CEC7C5D42F052D76">>}, + {<<"eini">>, <<"DA64AE8DB7C2F502E6F20CDF44CD3D9BE364412B87FF49FEBF282540F673DFCB">>}, + {<<"jsx">>, <<"EED26A0D04D217F9EECEFFFB89714452556CF90EB38F290A27A4D45B9988F8C0">>}, + {<<"lhttpc">>, <<"154EEB27692482B52BE86406DCD1D18A2405CAFCE0E8DAA4A1A7BFA7FE295896">>}]} ]. diff --git a/src/erlcloud_cognito_user_pools.erl b/src/erlcloud_cognito_user_pools.erl index 8ffa263ce..11efb89b7 100644 --- a/src/erlcloud_cognito_user_pools.erl +++ b/src/erlcloud_cognito_user_pools.erl @@ -254,13 +254,13 @@ admin_get_user(UserName, UserPoolId, Config) -> admin_create_user(UserName, UserPoolId) -> admin_create_user(UserName, UserPoolId, #{}). --spec admin_create_user(binary(), binary(), maps:maps()) -> +-spec admin_create_user(binary(), binary(), map()) -> {ok, map()} | {error, any()}. admin_create_user(UserName, UserPoolId, OptionalArgs) -> Config = erlcloud_aws:default_config(), admin_create_user(UserName, UserPoolId, OptionalArgs, Config). --spec admin_create_user(binary(), binary(), maps:maps(), aws_config()) -> +-spec admin_create_user(binary(), binary(), map(), aws_config()) -> {ok, map()} | {error, any()}. admin_create_user(UserName, UserPoolId, OptionalArgs, Config) -> Body = OptionalArgs#{ @@ -661,13 +661,13 @@ admin_forget_device(UserPoolId, Username, DeviceKey, Config) -> admin_confirm_signup(UserPoolId, Username) -> admin_confirm_signup(UserPoolId, Username, #{}). --spec admin_confirm_signup(binary(), binary(), maps:map()) -> +-spec admin_confirm_signup(binary(), binary(), map()) -> {ok, map()} | {error, any()}. admin_confirm_signup(UserPoolId, Username, ClientMetadata) -> Config = erlcloud_aws:default_config(), admin_confirm_signup(UserPoolId, Username, ClientMetadata, Config). --spec admin_confirm_signup(binary(), binary(), maps:map(), aws_config()) -> +-spec admin_confirm_signup(binary(), binary(), map(), aws_config()) -> {ok, map()} | {error, any()}. admin_confirm_signup(UserPoolId, Username, ClientMetadata, Config) -> Body = #{ @@ -677,21 +677,21 @@ admin_confirm_signup(UserPoolId, Username, ClientMetadata, Config) -> }, request(Config, "AdminConfirmSignUp", Body). --spec admin_initiate_auth(binary(), binary(), binary(), maps:map()) -> +-spec admin_initiate_auth(binary(), binary(), binary(), map()) -> {ok, map()} | {error, any()}. admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams) -> Cfg = erlcloud_aws:default_config(), admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, Cfg). -spec admin_initiate_auth(binary(), binary(), binary(), - maps:map(), aws_config()) -> + map(), aws_config()) -> {ok, map()} | {error, any()}. admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, Cfg) -> admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, #{}, #{}, #{}, Cfg). --spec admin_initiate_auth(binary(), binary(), binary(), maps:map(), - maps:map(), maps:map(), maps:map(), aws_config()) -> +-spec admin_initiate_auth(binary(), binary(), binary(), map(), + map(), map(), map(), aws_config()) -> {ok, map()} | {error, any()}. admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, AnalyticsMeta, ClientMeta, ContextData, Cfg) -> @@ -708,14 +708,14 @@ admin_initiate_auth(PoolId, ClientId, AuthFlow, AuthParams, }, request(Cfg, "AdminInitiateAuth", make_request_body(Mandatory, Optional)). --spec respond_to_auth_challenge(binary(), binary(), maps:map(), binary()) -> +-spec respond_to_auth_challenge(binary(), binary(), map(), binary()) -> {ok, map()} | {error, any()}. respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, Session) -> Cfg = erlcloud_aws:default_config(), respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, Session, Cfg). --spec respond_to_auth_challenge(binary(), binary(), maps:map(), binary(), +-spec respond_to_auth_challenge(binary(), binary(), map(), binary(), aws_config()) -> {ok, map()} | {error, any()}. respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, @@ -723,8 +723,8 @@ respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, Session, #{}, #{}, #{}, Cfg). --spec respond_to_auth_challenge(binary(), binary(), maps:map(), binary(), - maps:map(), maps:map(), maps:map(), +-spec respond_to_auth_challenge(binary(), binary(), map(), binary(), + map(), map(), map(), aws_config()) -> {ok, map()} | {error, any()}. respond_to_auth_challenge(ClientId, ChallengeName, ChallengeResponses, diff --git a/src/erlcloud_util.erl b/src/erlcloud_util.erl index 201803b96..6802d23a4 100644 --- a/src/erlcloud_util.erl +++ b/src/erlcloud_util.erl @@ -198,18 +198,18 @@ next_token(Path, XML) -> ok end. --spec filter_undef(proplists:proplist() | maps:map()) -> - proplists:proplist() | maps:map(). +-spec filter_undef(proplists:proplist() | map()) -> + proplists:proplist() | map(). filter_undef(List) when is_list(List) -> lists:filter(fun({_Name, Value}) -> Value =/= undefined end, List); filter_undef(Map) when is_map(Map) -> maps:filter(fun(_Key, Value) -> Value =/= undefined end, Map). --spec filter_empty_map(maps:map()) -> maps:map(). +-spec filter_empty_map(map()) -> map(). filter_empty_map(Map) when is_map(Map) -> maps:filter(fun(_Key, Value) -> Value =/= #{} end, Map). --spec filter_empty_list(maps:map()) -> maps:map(). +-spec filter_empty_list(map()) -> map(). filter_empty_list(Map) when is_map(Map) -> maps:filter(fun(_Key, Value) -> Value =/= [] end, Map). diff --git a/test/erlcloud_lambda_tests.erl b/test/erlcloud_lambda_tests.erl index 027546e36..542814d35 100644 --- a/test/erlcloud_lambda_tests.erl +++ b/test/erlcloud_lambda_tests.erl @@ -7,43 +7,42 @@ -define(BASE_URL, "https://lambda.us-east-1.amazonaws.com:443/2015-03-31/"). route53_test_() -> - {foreach, - fun setup/0, - fun meck:unload/1, - [ - fun api_tests/1 - ]}. + {foreach, fun setup/0, fun meck:unload/1, [ + fun api_tests/1 + ]}. mocks() -> [ - mocked_create_alias(), - mocked_create_event_source_mapping(), - mocked_create_function(), - mocked_delete_event_source_mapping(), - mocked_delete_function(), - mocked_delete_function_qualifier(), - mocked_get_alias(), - mocked_get_event_source_mapping(), - mocked_get_function(), - mocked_get_function_configuration(), - mocked_invoke(), - mocked_invoke_alias(), - mocked_invoke_qualifier(), - mocked_list_aliases(), - mocked_list_event_source_mappings(), - mocked_list_function(), - mocked_list_versions_by_function(), - mocked_publish_version(), - mocked_update_alias(), - mocked_update_event_source_mapping(), - mocked_update_function_code(), - mocked_update_function_configuration() + mocked_create_alias(), + mocked_create_event_source_mapping(), + mocked_create_function(), + mocked_delete_event_source_mapping(), + mocked_delete_function(), + mocked_delete_function_qualifier(), + mocked_get_alias(), + mocked_get_event_source_mapping(), + mocked_get_function(), + mocked_get_function_configuration(), + mocked_invoke(), + mocked_invoke_alias(), + mocked_invoke_qualifier(), + mocked_list_aliases(), + mocked_list_event_source_mappings(), + mocked_list_function(), + mocked_list_versions_by_function(), + mocked_publish_version(), + mocked_update_alias(), + mocked_update_event_source_mapping(), + mocked_update_function_code(), + mocked_update_function_configuration() ]. setup() -> - Config = #aws_config{access_key_id="AccessId", - secret_access_key="secret", - security_token="token"}, + Config = #aws_config{ + access_key_id = "AccessId", + secret_access_key = "secret", + security_token = "token" + }, meck:new(ECA = erlcloud_aws, [non_strict, passthrough]), meck:expect(ECA, default_config, 0, Config), meck:expect(ECA, update_config, 1, {ok, Config}), @@ -53,806 +52,1298 @@ setup() -> mocked_create_alias() -> { - [?BASE_URL ++ "functions/name/aliases", post, '_', - <<"{\"FunctionVersion\":\"$LATEST\",\"Name\":\"aliasName1\"}">>, '_', '_'], - make_response(<<"{\"AliasArn\":\"arn:aws:lambda:us-east-1:352283894008:" -"function:name:aliasName1\",\"Description\":\"\",\"FunctionVersion\":\"$LATEST" -"\",\"Name\":\"aliasName1\"}">>) + [ + ?BASE_URL ++ "functions/name/aliases", + post, + '_', + <<"{\"FunctionVersion\":\"$LATEST\",\"Name\":\"aliasName1\"}">>, + '_', + '_' + ], + make_response([ + {<<"AliasArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName1">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName1">>} + ]) }. mocked_create_event_source_mapping() -> { - [?BASE_URL ++ "event-source-mappings", post, '_', - <<"{\"EventSourceArn\":\"arn:aws:kinesis:us-east-1:352283894008:stream" + [ + ?BASE_URL ++ "event-source-mappings", + post, + '_', + <<"{\"EventSourceArn\":\"arn:aws:kinesis:us-east-1:352283894008:stream" "/eholland-test\",\"FunctionName\":\"name\",\"StartingPosition\":\"TRI" - "M_HORIZON\"}">>, '_', '_'], - make_response(<<"{\"BatchSize\":100,\"EventSourceArn\":\"arn:aws:kinesi" -"s:us-east-1:352283894008:stream/eholland-test\",\"FunctionArn\":\"arn:aws:lam" -"bda:us-east-1:352283894008:function:name\",\"LastModified\":1449845416.123,\"" -"LastProcessingResult\":\"No records processed\",\"State\":\"Creating\",\"Stat" -"eTransitionReason\":\"User action\",\"UUID\":\"3f303f86-7395-43f3-9902-f5c80f" -"0a5382\"}">>) + "M_HORIZON\"}">>, + '_', + '_' + ], + make_response( + [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449845416.123}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Creating">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"3f303f86-7395-43f3-9902-f5c80f0a5382">>} + ] + ) }. mocked_create_function() -> { - [?BASE_URL ++ "functions", post, '_', - <<"{\"Code\":{\"S3Bucket\":\"bi-lambda\",\"S3Key\":\"local_transform/bi-a" -"ssets-environment-create-environment_1-0-0_latest.zip\"},\"FunctionName\":\"name" -"\",\"Handler\":\"index.process\",\"Role\":\"arn:aws:iam::352283894008:role/lambd" -"a_basic_execution\",\"Runtime\":\"nodejs\"}">> , '_', '_'], - make_response(<<"{\"CodeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4Ic" -"piPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"FunctionArn\":\"arn:aws:la" -"mbda:us-east-1:352283894008:function:name3\",\"FunctionName\":\"name3\",\"Han" -"dler\":\"index.process\",\"LastModified\":\"2015-12-11T13:45:31.924+0000\",\"" -"MemorySize\":128,\"Role\":\"arn:aws:iam::352283894008:role/lambda_basic_execu" -"tion\",\"Runtime\":\"nodejs\",\"Timeout\":3,\"Version\":\"$LATEST\",\"VpcConf" -"ig\":null}">>) + [ + ?BASE_URL ++ "functions", + post, + '_', + jsx:encode([ + {<<"Code">>, [ + {<<"S3Bucket">>, <<"bi-lambda">>}, + {<<"S3Key">>, + <<"local_transform/bi-assets-environment-create-environment_1-0-0_latest.zip">>} + ]}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"nodejs">>} + ]), + '_', + '_' + ], + make_response([ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name3">>}, + {<<"FunctionName">>, <<"name3">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-11T13:45:31.924+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ] + ) }. + mocked_delete_event_source_mapping() -> { - [?BASE_URL ++ "event-source-mappings/6554f300-551b-46a6-829c-41b6af6022c6", - delete, '_', <<>>, '_', '_'], - make_response(<<"{\"BatchSize\":100,\"EventSourceArn\":\"arn:aws:kinesi" -"s:us-east-1:352283894008:stream/eholland-test\",\"FunctionArn\":\"arn:aws:lam" -"bda:us-east-1:352283894008:function:name\",\"LastModified\":1449843960.0,\"La" -"stProcessingResult\":\"No records processed\",\"State\":\"Deleting\",\"StateT" -"ransitionReason\":\"User action\",\"UUID\":\"a45b58ec-a539-4c47-929e-174b4dd2" -"d963\"}">>) + [ + ?BASE_URL ++ "event-source-mappings/6554f300-551b-46a6-829c-41b6af6022c6", + delete, + '_', + <<>>, + '_', + '_' + ], + make_response( + [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449843960.0}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Deleting">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>} + ] + ) }. mocked_delete_function() -> { - [?BASE_URL ++ "functions/name", - delete, '_', <<>>, '_', '_'], - make_response({204,"No Content"}, <<"">>) + [ + ?BASE_URL ++ "functions/name", + delete, + '_', + <<>>, + '_', + '_' + ], + make_response({204, "No Content"}, <<"">>) }. mocked_delete_function_qualifier() -> { - [?BASE_URL ++ "functions/name_qualifier?Qualifier=123", - delete, '_', <<>>, '_', '_'], - make_response({204,"No Content"}, <<"">>) + [ + ?BASE_URL ++ "functions/name_qualifier?Qualifier=123", + delete, + '_', + <<>>, + '_', + '_' + ], + make_response({204, "No Content"}, <<"">>) }. mocked_get_alias() -> - { - [?BASE_URL ++ "functions/name/aliases/aliasName", - get, '_', <<>>, '_', '_'], - make_response(<<"{\"AliasArn\":\"arn:aws:lambda:us-east-1:352283894008:" -"function:name:aliasName\",\"Description\":\"\",\"FunctionVersion\":\"$LATEST\" -"",\"Name\":\"aliasName\"}">>) + { + [ + ?BASE_URL ++ "functions/name/aliases/aliasName", + get, + '_', + <<>>, + '_', + '_' + ], + make_response([ + {<<"AliasArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName">>} + ]) }. mocked_get_event_source_mapping() -> - { - [?BASE_URL ++ "event-source-mappings/a45b58ec-a539-4c47-929e-174b4dd2d963", - get, '_', <<>>, '_', '_'], - make_response(<<"{\"BatchSize\":100,\"EventSourceArn\":\"arn:aws:kinesi" -"s:us-east-1:352283894008:stream/eholland-test\",\"FunctionArn\":\"arn:aws:lam" -"bda:us-east-1:352283894008:function:name\",\"LastModified\":1449841860.0,\"La" -"stProcessingResult\":\"No records processed\",\"State\":\"Enabled\",\"StateTr" -"ansitionReason\":\"User action\",\"UUID\":\"a45b58ec-a539-4c47-929e-174b4dd2d" -"963\"}">>) + { + [ + ?BASE_URL ++ "event-source-mappings/a45b58ec-a539-4c47-929e-174b4dd2d963", + get, + '_', + <<>>, + '_', + '_' + ], + make_response( + [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449841860.0}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Enabled">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>} + ] + ) }. mocked_get_function() -> { - [?BASE_URL ++ "functions/name", - get, '_', <<>>, '_', '_'], - make_response(<<"{\"Code\":{\"Location\":\"https://awslambda-us-east-1-" -"tasks.s3-us-east-1.amazonaws.com/snapshots/352283894008/name-69237aec-bae9-40" -"86-af73-a6610c2f8eb8?x-amz-security-token=AQoDYXdzENb%2F%2F%2F%2F%2F%2F%2F%2F" -"%2F%2FwEa4APJSJNHsYWnObKlElri5ytVzNg0t%2F0zwADHn40jy%2F1ZDW9%2FYWGi8dTp1l6LWB" -"I9TwJi0LLcgp%2FlCxJIh7hsAPftYX62J9r9lRcmgd9RnYssg1%2Fkpfyjya90epxKg2zdHm%2BuZ" -"GukHYDcmAE1IQcHwsaQbvGAjXPCpFyxClbV6gMcFIsaBtfMxoMcbTCXG9m8l56nKgcX6Mi60vRNaB" -"83AeNVrKMhB8EBUUbYbaB%2BG0iJg32i2HBF6VJMxamOLIEf1GJp1tWt%2FSAHfEkdTwcwtGINH3T" -"NRv%2BY3ddsXs8pJ49eY49NCHANPC%2Bq0JzNQydbIK1shz8w1nozXYQo6%2BNh9tqOlaJNFgfFbt" -"JkUDXv4rFqgVsfgJKJSQBeYUKmlNvIPQIoHWhRjjRzQUGmYDc3eEug7vELsNcHZixI4nNVycH%2BJ" -"ZwaBvswy4eE7gBv3HwHi3SVlg9iXFTrfWTK%2FlCybC7mZIjAmPGiLCG5Pu8SoCgaGdHp8HmSeXXu" -"s4VUFcVTUw1qn7E%2BSaRFg3MTpCdu1f1Nqh4pNu7GpZacyLbH%2BSocuPyTjyYGL8sk0C3rjIWZi" -"pJcfUZsleS1cXLEGw%2FPAK5eg0RNWlVsaF1WgVimqQh3LtoRZ%2BwYCHRXsHjhCyQ8VhoguJCrsw" -"U%3D&AWSAccessKeyId=ASIAIAUY4IMA6JRJ3FSA&Expires=1449843004&Signature=5KUmber" -"1cOBSChlKtnLaI%2FUemU4%3D\",\"RepositoryType\":\"S3\"},\"Configuration\":{\"C" -"odeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=\",\"CodeSize\":848" -",\"Description\":\"\",\"FunctionArn\":\"arn:aws:lambda:us-east-1:352283894008" -":function:name\",\"FunctionName\":\"name\",\"Handler\":\"index.process\",\"La" -"stModified\":\"2015-12-10T13:57:48.214+0000\",\"MemorySize\":512,\"Role\":\"a" -"rn:aws:iam::352283894008:role/lambda_kinesis_role\",\"Runtime\":\"nodejs\",\"" -"Timeout\":30,\"Version\":\"$LATEST\",\"VpcConfig\":null}}">>) + [ + ?BASE_URL ++ "functions/name", + get, + '_', + <<>>, + '_', + '_' + ], + make_response([ + {<<"Code">>, [ + {<<"Location">>, + <<"https://awslambda-us-east-1-tasks.s3-us-east-1.amazonaws.com/snapshots/352283894008/name-69237aec-bae9-4086-af73-a6610c2f8eb8?x-amz-security-token=AQoDYXdzENb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEa4APJSJNHsYWnObKlElri5ytVzNg0t%2F0zwADHn40jy%2F1ZDW9%2FYWGi8dTp1l6LWBI9TwJi0LLcgp%2FlCxJIh7hsAPftYX62J9r9lRcmgd9RnYssg1%2Fkpfyjya90epxKg2zdHm%2BuZGukHYDcmAE1IQcHwsaQbvGAjXPCpFyxClbV6gMcFIsaBtfMxoMcbTCXG9m8l56nKgcX6Mi60vRNaB83AeNVrKMhB8EBUUbYbaB%2BG0iJg32i2HBF6VJMxamOLIEf1GJp1tWt%2FSAHfEkdTwcwtGINH3TNRv%2BY3ddsXs8pJ49eY49NCHANPC%2Bq0JzNQydbIK1shz8w1nozXYQo6%2BNh9tqOlaJNFgfFbtJkUDXv4rFqgVsfgJKJSQBeYUKmlNvIPQIoHWhRjjRzQUGmYDc3eEug7vELsNcHZixI4nNVycH%2BJZwaBvswy4eE7gBv3HwHi3SVlg9iXFTrfWTK%2FlCybC7mZIjAmPGiLCG5Pu8SoCgaGdHp8HmSeXXus4VUFcVTUw1qn7E%2BSaRFg3MTpCdu1f1Nqh4pNu7GpZacyLbH%2BSocuPyTjyYGL8sk0C3rjIWZipJcfUZsleS1cXLEGw%2FPAK5eg0RNWlVsaF1WgVimqQh3LtoRZ%2BwYCHRXsHjhCyQ8VhoguJCrswU%3D&AWSAccessKeyId=ASIAIAUY4IMA6JRJ3FSA&Expires=1449843004&Signature=5KUmber1cOBSChlKtnLaI%2FUemU4%3D">>}, + {<<"RepositoryType">>, <<"S3">>} + ]}, + {<<"Configuration">>, [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ]} + ]) }. mocked_get_function_configuration() -> { - [?BASE_URL ++ "functions/name/configuration", get, '_', <<>>, '_', '_'], - make_response(<<"{\"CodeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4Ic" -"piPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"FunctionArn\":\"arn:aws:la" -"mbda:us-east-1:352283894008:function:name\",\"FunctionName\":\"name\",\"Handl" -"er\":\"index.process\",\"LastModified\":\"2015-12-10T13:57:48.214+0000\",\"Me" -"morySize\":512,\"Role\":\"arn:aws:iam::352283894008:role/lambda_kinesis_role\" -"",\"Runtime\":\"nodejs\",\"Timeout\":30,\"Version\":\"$LATEST\",\"VpcConfig\"" -":null}">>) + [?BASE_URL ++ "functions/name/configuration", get, '_', <<>>, '_', '_'], + make_response( + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ] + ) }. mocked_invoke() -> - { - [?BASE_URL ++ "functions/name/invocations", post, '_', <<"{}">>, '_', '_'], - make_response(<<"{\"message\":\"Hello World!\"}">>) - }. + { + [?BASE_URL ++ "functions/name/invocations", post, '_', <<"{}">>, '_', '_'], + make_response(<<"{\"message\":\"Hello World!\"}">>) + }. mocked_invoke_alias() -> - { - [?BASE_URL ++ "functions/name%3Aalias/invocations", post, '_', <<"{}">>, '_', '_'], - make_response(<<"{\"message\":\"Hello World!\"}">>) - }. - + { + [?BASE_URL ++ "functions/name%3Aalias/invocations", post, '_', <<"{}">>, '_', '_'], + make_response(<<"{\"message\":\"Hello World!\"}">>) + }. mocked_invoke_qualifier() -> - { - [?BASE_URL ++ "functions/name_qualifier/invocations?Qualifier=123", post, - '_', <<"{}">>, '_', '_'], - make_response(<<"{\"message\":\"Hello World!\"}">>) - }. - + { + [ + ?BASE_URL ++ "functions/name_qualifier/invocations?Qualifier=123", + post, + '_', + <<"{}">>, + '_', + '_' + ], + make_response(<<"{\"message\":\"Hello World!\"}">>) + }. mocked_list_aliases() -> { - [?BASE_URL ++ "functions/name/aliases", get, '_', <<>>, '_', '_'], - make_response(<<"{\"Aliases\":[{\"AliasArn\":\"arn:aws:lambda:us-east-1" -":352283894008:function:name:aliasName\",\"Description\":\"\",\"FunctionVersio" -"n\":\"$LATEST\",\"Name\":\"aliasName\"},{\"AliasArn\":\"arn:aws:lambda:us-eas" -"t-1:352283894008:function:name:aliasName1\",\"Description\":\"\",\"FunctionVe" -"rsion\":\"$LATEST\",\"Name\":\"aliasName1\"}],\"NextMarker\":null}">>) + [?BASE_URL ++ "functions/name/aliases", get, '_', <<>>, '_', '_'], + make_response([ + {<<"Aliases">>, [ + [ + {<<"AliasArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName">>} + ], + [ + {<<"AliasArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName1">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName1">>} + ] + ]}, + {<<"NextMarker">>, null} + ]) }. mocked_list_event_source_mappings() -> { - [?BASE_URL ++ "event-source-mappings/?EventSourceArn=arn%3Aaws%3Akinesis%" - "3Aus-east-1%3A352283894008%3Astream%2Feholland-test&FunctionName=name", - get, '_', <<>>, '_', '_'], - make_response(<<"{\"EventSourceMappings\":[{\"BatchSize\":100,\"EventSo" -"urceArn\":\"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test\",\"F" -"unctionArn\":\"arn:aws:lambda:us-east-1:352283894008:function:name\",\"LastMo" -"dified\":1449841860.0,\"LastProcessingResult\":\"No records processed\",\"Sta" -"te\":\"Enabled\",\"StateTransitionReason\":\"User action\",\"UUID\":\"a45b58e" -"c-a539-4c47-929e-174b4dd2d963\"}],\"NextMarker\":null}">>) + [ + ?BASE_URL ++ + "event-source-mappings/?EventSourceArn=arn%3Aaws%3Akinesis%" + "3Aus-east-1%3A352283894008%3Astream%2Feholland-test&FunctionName=name", + get, + '_', + <<>>, + '_', + '_' + ], + make_response([ + {<<"EventSourceMappings">>, [ + [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449841860.0}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Enabled">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>} + ] + ]}, + {<<"NextMarker">>, null} + ]) }. mocked_list_function() -> { - [?BASE_URL ++ "functions/", - get, '_', <<>>, '_', '_'], - make_response(<<"{\"Functions\":[{\"CodeSha256\":\"XmLDAZXEkl5KbA8ezZpw" -"FU+bjgTXBehUmWGOScl4F2A=\",\"CodeSize\":5561,\"Description\":\"\",\"FunctionA" -"rn\":\"arn:aws:lambda:us-east-1:352283894008:function:bi-assets-asset-create-" -"host\",\"FunctionName\":\"bi-assets-asset-create-host\",\"Handler\":\"index.h" -"andler\",\"LastModified\":\"2015-11-27T09:55:12.973+0000\",\"MemorySize\":128" -",\"Role\":\"arn:aws:iam::352283894008:role/lambda_basic_execution\",\"Runtime" -"\":\"nodejs\",\"Timeout\":3,\"Version\":\"$LATEST\",\"VpcConfig\":null},{\"Co" -"deSha256\":\"XmLDAZXEkl5KbA8ezZpwFU+bjgTXBehUmWGOScl4F2A=\",\"CodeSize\":5561" -",\"Description\":\"\",\"FunctionArn\":\"arn:aws:lambda:us-east-1:352283894008" -":function:bi-assets-asset-create-host1\",\"FunctionName\":\"bi-assets-asset-c" -"reate-host1\",\"Handler\":\"index.handler\",\"LastModified\":\"2015-12-01T11:" -"20:44.464+0000\",\"MemorySize\":128,\"Role\":\"arn:aws:iam::352283894008:role" -"/lambda_kinesis_role\",\"Runtime\":\"nodejs\",\"Timeout\":10,\"Version\":\"$L" -"ATEST\",\"VpcConfig\":null},{\"CodeSha256\":\"tZJ+kUZVD1vGYwMIUvAoaZmvS4I9NHV" -"c7a/267eChYY=\",\"CodeSize\":132628,\"Description\":\"\",\"FunctionArn\":\"ar" -"n:aws:lambda:us-east-1:352283894008:function:bi-driver\",\"FunctionName\":\"b" -"i-driver\",\"Handler\":\"index.handler\",\"LastModified\":\"2015-12-03T13:59:" -"05.219+0000\",\"MemorySize\":1024,\"Role\":\"arn:aws:iam::352283894008:role/l" -"ambda_kinesis_role\",\"Runtime\":\"nodejs\",\"Timeout\":59,\"Version\":\"$LAT" -"EST\",\"VpcConfig\":null},{\"CodeSha256\":\"QS10seyYGXrrhAnGMbJcTi+JOa4HWLaD+" -"9YCLYG3+VE=\",\"CodeSize\":121486,\"Description\":\"An Amazon Kinesis stream " -"processor that logs the data being published.\",\"FunctionArn\":\"arn:aws:lam" -"bda:us-east-1:352283894008:function:eholland-js-router\",\"FunctionName\":\"e" -"holland-js-router\",\"Handler\":\"index.handler\",\"LastModified\":\"2015-12-" -"02T14:50:41.923+0000\",\"MemorySize\":1024,\"Role\":\"arn:aws:iam::3522838940" -"08:role/lambda_kinesis_role\",\"Runtime\":\"nodejs\",\"Timeout\":60,\"Version" -"\":\"$LATEST\",\"VpcConfig\":null},{\"CodeSha256\":\"ey+6CSWe750XoPSdVSQditxx" -"oHNWmPFwve/MLPNs/Do=\",\"CodeSize\":253,\"Description\":\"\",\"FunctionArn\":" -"\"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-aims-user\",\"" -"FunctionName\":\"eholland-test-aims-user\",\"Handler\":\"lambda_function.lamb" -"da_handler\",\"LastModified\":\"2015-11-09T17:32:46.030+0000\",\"MemorySize\"" -":128,\"Role\":\"arn:aws:iam::352283894008:role/lambda_basic_execution\",\"Run" -"time\":\"python2.7\",\"Timeout\":3,\"Version\":\"$LATEST\",\"VpcConfig\":null" -"},{\"CodeSha256\":\"R9HaKKCPPAem0ikKrkMGpDtX95M8egDCDd+4Ws+Kk5c=\",\"CodeSize" -"\":253,\"Description\":\"aims users transform function\",\"FunctionArn\":\"ar" -"n:aws:lambda:us-east-1:352283894008:function:eholland-test-aims-users\",\"Fun" -"ctionName\":\"eholland-test-aims-users\",\"Handler\":\"lambda_function.lambda" -"_handler\",\"LastModified\":\"2015-11-02T13:59:32.509+0000\",\"MemorySize\":1" -"28,\"Role\":\"arn:aws:iam::352283894008:role/lambda_basic_execution\",\"Runti" -"me\":\"python2.7\",\"Timeout\":3,\"Version\":\"$LATEST\",\"VpcConfig\":null}," -"{\"CodeSha256\":\"PGG1vCAQpc8J7e4HL1z1Pv4DGSZEmhnJaNPTGDS29kk=\",\"CodeSize\"" -":32372,\"Description\":\"An Amazon Kinesis stream processor that logs the dat" -"a being published.\",\"FunctionArn\":\"arn:aws:lambda:us-east-1:352283894008:" -"function:eholland-test-router\",\"FunctionName\":\"eholland-test-router\",\"H" -"andler\":\"ProcessKinesisRecords.lambda_handler\",\"LastModified\":\"2015-11-" -"09T10:58:50.458+0000\",\"MemorySize\":256,\"Role\":\"arn:aws:iam::35228389400" -"8:role/lambda_kinesis_role\",\"Runtime\":\"python2.7\",\"Timeout\":59,\"Versi" -"on\":\"$LATEST\",\"VpcConfig\":null},{\"CodeSha256\":\"8twXwGaAXyr8Li81rGhM6v" -"lMh+dvXglwqgIshfaio+U=\",\"CodeSize\":708740,\"Description\":\"Lambda Router " -"Function for ETL Service\",\"FunctionArn\":\"arn:aws:lambda:us-east-1:3522838" -"94008:function:us-east-1_base_eholland_master_lambda_route\",\"FunctionName\"" -":\"us-east-1_base_eholland_master_lambda_route\",\"Handler\":\"ProcessKinesis" -"Records.handler\",\"LastModified\":\"2015-11-11T10:13:19.043+0000\",\"MemoryS" -"ize\":512,\"Role\":\"arn:aws:iam::352283894008:role/lambda_kinesis_role\",\"R" -"untime\":\"nodejs\",\"Timeout\":59,\"Version\":\"$LATEST\",\"VpcConfig\":null" -"},{\"CodeSha256\":\"aDwLJhljsMVHsaN+LM4jRmsSLKKBdS+eFrAhKmJ/zbQ=\",\"CodeSize" -"\":253,\"Description\":\"\",\"FunctionArn\":\"arn:aws:lambda:us-east-1:352283" -"894008:function:us-east-1_base_eholland_master_lambda_route-aims-user-create\"" -",\"FunctionName\":\"us-east-1_base_eholland_master_lambda_route-aims-user-cr" -"eate\",\"Handler\":\"lambda_function.lambda_handler\",\"LastModified\":\"2015" -"-11-11T09:42:10.303+0000\",\"MemorySize\":128,\"Role\":\"arn:aws:iam::3522838" -"94008:role/lambda_basic_execution\",\"Runtime\":\"python2.7\",\"Timeout\":3," -"\"Version\":\"$LATEST\",\"VpcConfig\":null},{\"CodeSha256\":\"zeoBX1hIWJBHk1mu" -"Je1iFyS1CcAmsT0Ct4IcpiPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"Functi" -"onArn\":\"arn:aws:lambda:us-east-1:352283894008:function:name\",\"FunctionNam" -"e\":\"name\",\"Handler\":\"index.process\",\"LastModified\":\"2015-12-10T13:5" -"7:48.214+0000\",\"MemorySize\":512,\"Role\":\"arn:aws:iam::352283894008:role/" -"lambda_kinesis_role\",\"Runtime\":\"nodejs\",\"Timeout\":30,\"Version\":\"$LA" -"TEST\",\"VpcConfig\":null},{\"CodeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0C" -"t4IcpiPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"FunctionArn\":\"arn:aw" -"s:lambda:us-east-1:352283894008:function:name2\",\"FunctionName\":\"name2\"," -"\"Handler\":\"index.process\",\"LastModified\":\"2015-12-10T11:31:21.106+0000" -"\",\"MemorySize\":128,\"Role\":\"arn:aws:iam::352283894008:role/lambda_basic_e" -"xecution\",\"Runtime\":\"nodejs\",\"Timeout\":3,\"Version\":\"$LATEST\",\"Vpc" -"Config\":null}],\"NextMarker\":null}">>) + [ + ?BASE_URL ++ "functions/", + get, + '_', + <<>>, + '_', + '_' + ], + make_response([ + {<<"Functions">>, [ + [ + {<<"CodeSha256">>, <<"XmLDAZXEkl5KbA8ezZpwFU+bjgTXBehUmWGOScl4F2A=">>}, + {<<"CodeSize">>, 5561}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:bi-assets-asset-create-host">>}, + {<<"FunctionName">>, <<"bi-assets-asset-create-host">>}, + {<<"Handler">>, <<"index.handler">>}, + {<<"LastModified">>, <<"2015-11-27T09:55:12.973+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"XmLDAZXEkl5KbA8ezZpwFU+bjgTXBehUmWGOScl4F2A=">>}, + {<<"CodeSize">>, 5561}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:bi-assets-asset-create-host1">>}, + {<<"FunctionName">>, <<"bi-assets-asset-create-host1">>}, + {<<"Handler">>, <<"index.handler">>}, + {<<"LastModified">>, <<"2015-12-01T11:20:44.464+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 10}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"tZJ+kUZVD1vGYwMIUvAoaZmvS4I9NHVc7a/267eChYY=">>}, + {<<"CodeSize">>, 132628}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:bi-driver">>}, + {<<"FunctionName">>, <<"bi-driver">>}, + {<<"Handler">>, <<"index.handler">>}, + {<<"LastModified">>, <<"2015-12-03T13:59:05.219+0000">>}, + {<<"MemorySize">>, 1024}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 59}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"QS10seyYGXrrhAnGMbJcTi+JOa4HWLaD+9YCLYG3+VE=">>}, + {<<"CodeSize">>, 121486}, + {<<"Description">>, + <<"An Amazon Kinesis stream processor that logs the data being published.">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:eholland-js-router">>}, + {<<"FunctionName">>, <<"eholland-js-router">>}, + {<<"Handler">>, <<"index.handler">>}, + {<<"LastModified">>, <<"2015-12-02T14:50:41.923+0000">>}, + {<<"MemorySize">>, 1024}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 60}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"ey+6CSWe750XoPSdVSQditxxoHNWmPFwve/MLPNs/Do=">>}, + {<<"CodeSize">>, 253}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-aims-user">>}, + {<<"FunctionName">>, <<"eholland-test-aims-user">>}, + {<<"Handler">>, <<"lambda_function.lambda_handler">>}, + {<<"LastModified">>, <<"2015-11-09T17:32:46.030+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"python2.7">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"R9HaKKCPPAem0ikKrkMGpDtX95M8egDCDd+4Ws+Kk5c=">>}, + {<<"CodeSize">>, 253}, + {<<"Description">>, <<"aims users transform function">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-aims-users">>}, + {<<"FunctionName">>, <<"eholland-test-aims-users">>}, + {<<"Handler">>, <<"lambda_function.lambda_handler">>}, + {<<"LastModified">>, <<"2015-11-02T13:59:32.509+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"python2.7">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"PGG1vCAQpc8J7e4HL1z1Pv4DGSZEmhnJaNPTGDS29kk=">>}, + {<<"CodeSize">>, 32372}, + {<<"Description">>, + <<"An Amazon Kinesis stream processor that logs the data being published.">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-router">>}, + {<<"FunctionName">>, <<"eholland-test-router">>}, + {<<"Handler">>, <<"ProcessKinesisRecords.lambda_handler">>}, + {<<"LastModified">>, <<"2015-11-09T10:58:50.458+0000">>}, + {<<"MemorySize">>, 256}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"python2.7">>}, + {<<"Timeout">>, 59}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"8twXwGaAXyr8Li81rGhM6vlMh+dvXglwqgIshfaio+U=">>}, + {<<"CodeSize">>, 708740}, + {<<"Description">>, <<"Lambda Router Function for ETL Service">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:us-east-1_base_eholland_master_lambda_route">>}, + {<<"FunctionName">>, <<"us-east-1_base_eholland_master_lambda_route">>}, + {<<"Handler">>, <<"ProcessKinesisRecords.handler">>}, + {<<"LastModified">>, <<"2015-11-11T10:13:19.043+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 59}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"aDwLJhljsMVHsaN+LM4jRmsSLKKBdS+eFrAhKmJ/zbQ=">>}, + {<<"CodeSize">>, 253}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:us-east-1_base_eholland_master_lambda_route-aims-user-create">>}, + {<<"FunctionName">>, + <<"us-east-1_base_eholland_master_lambda_route-aims-user-create">>}, + {<<"Handler">>, <<"lambda_function.lambda_handler">>}, + {<<"LastModified">>, <<"2015-11-11T09:42:10.303+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"python2.7">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name2">>}, + {<<"FunctionName">>, <<"name2">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T11:31:21.106+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ] + ]}, + {<<"NextMarker">>, null} + ]) }. mocked_list_versions_by_function() -> { - [?BASE_URL ++ "functions/name/versions", get, '_', <<>>, '_', '_'], - make_response(<<"{\"NextMarker\":null,\"Versions\":[{\"CodeSha256\":\"z" -"eoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=\",\"CodeSize\":848,\"Description" -"\":\"\",\"FunctionArn\":\"arn:aws:lambda:us-east-1:352283894008:function:name:" -"$LATEST\",\"FunctionName\":\"name\",\"Handler\":\"index.process\",\"LastModif" -"ied\":\"2015-12-10T13:57:48.214+0000\",\"MemorySize\":512,\"Role\":\"arn:aws:" -"iam::352283894008:role/lambda_kinesis_role\",\"Runtime\":\"nodejs\",\"Timeout" -"\":30,\"Version\":\"$LATEST\",\"VpcConfig\":null},{\"CodeSha256\":\"zeoBX1hIW" -"JBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=\",\"CodeSize\":848,\"Description\":\"\"," -"\"FunctionArn\":\"arn:aws:lambda:us-east-1:352283894008:function:name:1\",\"Fu" -"nctionName\":\"name\",\"Handler\":\"index.process\",\"LastModified\":\"2015-1" -"2-10T11:36:12.776+0000\",\"MemorySize\":128,\"Role\":\"arn:aws:iam::352283894" -"008:role/lambda_kinesis_role\",\"Runtime\":\"nodejs\",\"Timeout\":3,\"Version" -"\":\"1\",\"VpcConfig\":null},{\"CodeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT" -"0Ct4IcpiPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"FunctionArn\":\"arn:" -"aws:lambda:us-east-1:352283894008:function:name:2\",\"FunctionName\":\"name\"" -",\"Handler\":\"index.process\",\"LastModified\":\"2015-12-10T13:56:43.171+000" -"0\",\"MemorySize\":128,\"Role\":\"arn:aws:iam::352283894008:role/lambda_kines" -"is_role\",\"Runtime\":\"nodejs\",\"Timeout\":3,\"Version\":\"2\",\"VpcConfig" -"\":null}]}">>) + [?BASE_URL ++ "functions/name/versions", get, '_', <<>>, '_', '_'], + make_response([ + {<<"NextMarker">>, null}, + {<<"Versions">>, [ + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:$LATEST">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:1">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T11:36:12.776+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"1">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:2">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:56:43.171+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"2">>}, + {<<"VpcConfig">>, null} + ] + ]} + ]) }. mocked_publish_version() -> { - [?BASE_URL ++ "functions/name/versions", post, '_', <<"{}">>, '_', '_'], - make_response(<<"{\"CodeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4Ic" -"piPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"FunctionArn\":\"arn:aws:la" -"mbda:us-east-1:352283894008:function:name:3\",\"FunctionName\":\"name\",\"Han" -"dler\":\"index.process\",\"LastModified\":\"2015-12-10T13:57:48.214+0000\",\"" -"MemorySize\":512,\"Role\":\"arn:aws:iam::352283894008:role/lambda_kinesis_rol" -"e\",\"Runtime\":\"nodejs\",\"Timeout\":30,\"Version\":\"3\",\"VpcConfig\":nul" -"l}">>) + [?BASE_URL ++ "functions/name/versions", post, '_', <<"{}">>, '_', '_'], + make_response([ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name:3">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"3">>}, + {<<"VpcConfig">>, null} + ]) }. mocked_update_alias() -> { - [?BASE_URL ++ "functions/name/aliases/aliasName", put, - '_', <<"{}">>, '_', '_'], - make_response(<<"{\"AliasArn\":\"arn:aws:lambda:us-east-1:352283894008:" -"function:name:aliasName\",\"Description\":\"\",\"FunctionVersion\":\"$LATEST" -"\",\"Name\":\"aliasName\"}">>) + [ + ?BASE_URL ++ "functions/name/aliases/aliasName", + put, + '_', + <<"{}">>, + '_', + '_' + ], + make_response([ + {<<"AliasArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName">>} + ]) }. mocked_update_event_source_mapping() -> { - [?BASE_URL ++ "event-source-mappings/a45b58ec-a539-4c47-929e-174b4dd2d963", - put, '_', <<"{\"BatchSize\":100}">>, '_', '_'], - make_response(<<"{\"BatchSize\":100,\"EventSourceArn\":\"arn:aws:kinesi" -"s:us-east-1:352283894008:stream/eholland-test\",\"FunctionArn\":\"arn:aws:lam" -"bda:us-east-1:352283894008:function:name\",\"LastModified\":1449844011.991,\"" -"LastProcessingResult\":\"No records processed\",\"State\":\"Updating\",\"Stat" -"eTransitionReason\":\"User action\",\"UUID\":\"a45b58ec-a539-4c47-929e-174b4d" -"d2d963\"}">>) + [ + ?BASE_URL ++ "event-source-mappings/a45b58ec-a539-4c47-929e-174b4dd2d963", + put, + '_', + <<"{\"BatchSize\":100}">>, + '_', + '_' + ], + make_response([ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449844011.991}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Updating">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>} + ]) }. mocked_update_function_code() -> { - [?BASE_URL ++ "functions/name/code", put, '_', - <<"{\"Publish\":true,\"S3Bucket\":\"bi-lambda\",\"S3Key\":\"local_transf" + [ + ?BASE_URL ++ "functions/name/code", + put, + '_', + <<"{\"Publish\":true,\"S3Bucket\":\"bi-lambda\",\"S3Key\":\"local_transf" "orm/bi-assets-environment-create-environment_1-0-0_latest.zip\"}">>, - '_', '_'], - make_response(<<"{\"CodeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4Ic" -"piPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"FunctionArn\":\"arn:aws:la" -"mbda:us-east-1:352283894008:function:name:4\",\"FunctionName\":\"name\",\"Han" -"dler\":\"index.process\",\"LastModified\":\"2015-12-11T14:29:17.023+0000\",\"" -"MemorySize\":512,\"Role\":\"arn:aws:iam::352283894008:role/lambda_kinesis_rol" -"e\",\"Runtime\":\"nodejs\",\"Timeout\":30,\"Version\":\"4\",\"VpcConfig\":nul" -"l}">>) + '_', + '_' + ], + make_response([ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name:4">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-11T14:29:17.023+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"4">>}, + {<<"VpcConfig">>, null} + ]) }. mocked_update_function_configuration() -> { - [?BASE_URL ++ "functions/name/configuration", put, '_', - <<"{\"MemorySize\":512,\"Timeout\":30}">>, '_', '_'], - make_response(<<"{\"CodeSha256\":\"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4Ic" -"piPf8QM=\",\"CodeSize\":848,\"Description\":\"\",\"FunctionArn\":\"arn:aws:la" -"mbda:us-east-1:352283894008:function:name\",\"FunctionName\":\"name\",\"Handl" -"er\":\"index.process\",\"LastModified\":\"2015-12-11T14:31:52.034+0000\",\"Me" -"morySize\":512,\"Role\":\"arn:aws:iam::352283894008:role/lambda_kinesis_role\" -"",\"Runtime\":\"nodejs\",\"Timeout\":30,\"Version\":\"$LATEST\",\"VpcConfig\"" -":null}">>) + [ + ?BASE_URL ++ "functions/name/configuration", + put, + '_', + <<"{\"MemorySize\":512,\"Timeout\":30}">>, + '_', + '_' + ], + make_response([ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-11T14:31:52.034+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ]) }. api_tests(_) -> [ - fun() -> - Result = erlcloud_lambda:list_functions(), - Expected = - {ok, - [{<<"Functions">>, - [[{<<"CodeSha256">>,<<"XmLDAZXEkl5KbA8ezZpwFU+bjgTXBehUmWGOScl4F2A=">>}, - {<<"CodeSize">>,5561}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:bi-assets-asset-create-host">>}, - {<<"FunctionName">>,<<"bi-assets-asset-create-host">>}, - {<<"Handler">>,<<"index.handler">>}, - {<<"LastModified">>,<<"2015-11-27T09:55:12.973+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,3},{<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"XmLDAZXEkl5KbA8ezZpwFU+bjgTXBehUmWGOScl4F2A=">>}, - {<<"CodeSize">>,5561}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:bi-assets-asset-create-host1">>}, - {<<"FunctionName">>,<<"bi-assets-asset-create-host1">>}, - {<<"Handler">>,<<"index.handler">>}, - {<<"LastModified">>,<<"2015-12-01T11:20:44.464+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,10}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"tZJ+kUZVD1vGYwMIUvAoaZmvS4I9NHVc7a/267eChYY=">>}, - {<<"CodeSize">>,132628}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:bi-driver">>}, - {<<"FunctionName">>,<<"bi-driver">>}, - {<<"Handler">>,<<"index.handler">>}, - {<<"LastModified">>,<<"2015-12-03T13:59:05.219+0000">>}, - {<<"MemorySize">>,1024}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,59},{<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"QS10seyYGXrrhAnGMbJcTi+JOa4HWLaD+9YCLYG3+VE=">>}, - {<<"CodeSize">>,121486}, - {<<"Description">>,<<"An Amazon Kinesis stream processor that logs the data being published.">>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:eholland-js-router">>}, - {<<"FunctionName">>,<<"eholland-js-router">>}, - {<<"Handler">>,<<"index.handler">>}, - {<<"LastModified">>,<<"2015-12-02T14:50:41.923+0000">>}, - {<<"MemorySize">>,1024}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,60}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"ey+6CSWe750XoPSdVSQditxxoHNWmPFwve/MLPNs/Do=">>}, - {<<"CodeSize">>,253}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-aims-user">>}, - {<<"FunctionName">>,<<"eholland-test-aims-user">>}, - {<<"Handler">>,<<"lambda_function.lambda_handler">>}, - {<<"LastModified">>,<<"2015-11-09T17:32:46.030+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, - {<<"Runtime">>,<<"python2.7">>}, - {<<"Timeout">>,3}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"R9HaKKCPPAem0ikKrkMGpDtX95M8egDCDd+4Ws+Kk5c=">>}, - {<<"CodeSize">>,253}, - {<<"Description">>,<<"aims users transform function">>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-aims-users">>}, - {<<"FunctionName">>,<<"eholland-test-aims-users">>}, - {<<"Handler">>,<<"lambda_function.lambda_handler">>}, - {<<"LastModified">>,<<"2015-11-02T13:59:32.509+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, - {<<"Runtime">>,<<"python2.7">>}, - {<<"Timeout">>,3}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"PGG1vCAQpc8J7e4HL1z1Pv4DGSZEmhnJaNPTGDS29kk=">>}, - {<<"CodeSize">>,32372}, - {<<"Description">>,<<"An Amazon Kinesis stream processor that logs the data being published.">>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-router">>}, - {<<"FunctionName">>,<<"eholland-test-router">>}, - {<<"Handler">>,<<"ProcessKinesisRecords.lambda_handler">>}, - {<<"LastModified">>,<<"2015-11-09T10:58:50.458+0000">>}, - {<<"MemorySize">>,256}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"python2.7">>}, - {<<"Timeout">>,59}, - {<<"Version">>, <<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"8twXwGaAXyr8Li81rGhM6vlMh+dvXglwqgIshfaio+U=">>}, - {<<"CodeSize">>,708740}, - {<<"Description">>,<<"Lambda Router Function for ETL Service">>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:us-east-1_base_eholland_master_lambda_route">>}, - {<<"FunctionName">>,<<"us-east-1_base_eholland_master_lambda_route">>}, - {<<"Handler">>,<<"ProcessKinesisRecords.handler">>}, - {<<"LastModified">>,<<"2015-11-11T10:13:19.043+0000">>}, - {<<"MemorySize">>,512}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,59}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"aDwLJhljsMVHsaN+LM4jRmsSLKKBdS+eFrAhKmJ/zbQ=">>}, - {<<"CodeSize">>,253}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:us-east-1_base_eholland_master_lambda_route-aims-user-create">>}, - {<<"FunctionName">>,<<"us-east-1_base_eholland_master_lambda_route-aims-user-create">>}, - {<<"Handler">>,<<"lambda_function.lambda_handler">>}, - {<<"LastModified">>,<<"2015-11-11T09:42:10.303+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, - {<<"Runtime">>,<<"python2.7">>}, - {<<"Timeout">>,3}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-10T13:57:48.214+0000">>}, - {<<"MemorySize">>,512}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,30}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name2">>}, - {<<"FunctionName">>,<<"name2">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-10T11:31:21.106+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,3}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}]]}, - {<<"NextMarker">>,null}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:create_alias(<<"name">>, <<"$LATEST">>, - <<"aliasName1">>, []), - Expected = {ok,[{<<"AliasArn">>, - <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName1">>}, - {<<"Description">>,<<>>}, - {<<"FunctionVersion">>,<<"$LATEST">>}, - {<<"Name">>,<<"aliasName1">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:create_event_source_mapping( - <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>, - <<"name">>, <<"TRIM_HORIZON">>, []), - Expected = {ok, - [{<<"BatchSize">>,100}, - {<<"EventSourceArn">>,<<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"LastModified">>,1449845416.123}, - {<<"LastProcessingResult">>,<<"No records processed">>}, - {<<"State">>,<<"Creating">>}, - {<<"StateTransitionReason">>,<<"User action">>}, - {<<"UUID">>,<<"3f303f86-7395-43f3-9902-f5c80f0a5382">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:create_function( - #erlcloud_lambda_code{s3Bucket = <<"bi-lambda">>, - s3Key = <<"local_transform/bi-ass" - "ets-environment-create" - "-environment_1-0-0_lat" - "est.zip">> - }, - <<"name">>, <<"index.process">>, - <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>, - nodejs, []), - Expected = {ok,[{<<"CodeSha256">>, - <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>, - <<"arn:aws:lambda:us-east-1:352283894008:function:name3">>}, - {<<"FunctionName">>,<<"name3">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-11T13:45:31.924+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>, - <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,3}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:delete_event_source_mapping( - <<"6554f300-551b-46a6-829c-41b6af6022c6">>), - Expected = {ok, - [{<<"BatchSize">>,100}, - {<<"EventSourceArn">>,<<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"LastModified">>,1449843960.0}, - {<<"LastProcessingResult">>,<<"No records processed">>}, - {<<"State">>,<<"Deleting">>}, - {<<"StateTransitionReason">>,<<"User action">>}, - {<<"UUID">>,<<"a45b58ec-a539-4c47-929e-174b4dd2d963">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:delete_function( - <<"name">>), - Expected = {ok, []}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:delete_function( - <<"name_qualifier">>, <<"123">>, #aws_config{}), - Expected = {ok, []}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:get_alias(<<"name">>, <<"aliasName">>), - Expected = {ok, [{<<"AliasArn">>, - <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, - {<<"Description">>,<<>>}, - {<<"FunctionVersion">>,<<"$LATEST">>}, - {<<"Name">>,<<"aliasName">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:get_event_source_mapping( - <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>), - Expected = {ok, [{<<"BatchSize">>,100}, - {<<"EventSourceArn">>, - <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, - {<<"FunctionArn">>, - <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"LastModified">>,1449841860.0}, - {<<"LastProcessingResult">>,<<"No records processed">>}, - {<<"State">>,<<"Enabled">>}, - {<<"StateTransitionReason">>,<<"User action">>}, - {<<"UUID">>,<<"a45b58ec-a539-4c47-929e-174b4dd2d963">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:get_function(<<"name">>), - Expected = {ok, [{<<"Code">>, - [{<<"Location">>, <<"https://awslambda-us-east-1-tasks.s3-us-east-1.amazonaws.com/snapshots/352283894008/name-69237aec-bae9-4086-af73-a6610c2f8eb8?x-amz-security-token=AQoDYXdzENb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEa4APJSJNHsYWnObKlElri5ytVzNg0t%2F0zwADHn40jy%2F1ZDW9%2FYWGi8dTp1l6LWBI9TwJi0LLcgp%2FlCxJIh7hsAPftYX62J9r9lRcmgd9RnYssg1%2Fkpfyjya90epxKg2zdHm%2BuZGukHYDcmAE1IQcHwsaQbvGAjXPCpFyxClbV6gMcFIsaBtfMxoMcbTCXG9m8l56nKgcX6Mi60vRNaB83AeNVrKMhB8EBUUbYbaB%2BG0iJg32i2HBF6VJMxamOLIEf1GJp1tWt%2FSAHfEkdTwcwtGINH3TNRv%2BY3ddsXs8pJ49eY49NCHANPC%2Bq0JzNQydbIK1shz8w1nozXYQo6%2BNh9tqOlaJNFgfFbtJkUDXv4rFqgVsfgJKJSQBeYUKmlNvIPQIoHWhRjjRzQUGmYDc3eEug7vELsNcHZixI4nNVycH%2BJZwaBvswy4eE7gBv3HwHi3SVlg9iXFTrfWTK%2FlCybC7mZIjAmPGiLCG5Pu8SoCgaGdHp8HmSeXXus4VUFcVTUw1qn7E%2BSaRFg3MTpCdu1f1Nqh4pNu7GpZacyLbH%2BSocuPyTjyYGL8sk0C3rjIWZipJcfUZsleS1cXLEGw%2FPAK5eg0RNWlVsaF1WgVimqQh3LtoRZ%2BwYCHRXsHjhCyQ8VhoguJCrswU%3D&AWSAccessKeyId=ASIAIAUY4IMA6JRJ3FSA&Expires=1449843004&Signature=5KUmber1cOBSChlKtnLaI%2FUemU4%3D">>}, - {<<"RepositoryType">>,<<"S3">>}]}, - {<<"Configuration">>, - [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-10T13:57:48.214+0000">>}, - {<<"MemorySize">>,512}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,30}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}]}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:get_function_configuration(<<"name">>), - Expected = {ok, [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-10T13:57:48.214+0000">>}, - {<<"MemorySize">>,512}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,30}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:invoke(<<"name">>, []), - Expected = {ok, [{<<"message">>, <<"Hello World!">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:invoke(<<"name">>, [], [{show_headers, true}], #aws_config{}), - Expected = {ok, [], [{<<"message">>, <<"Hello World!">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:invoke(<<"name">>, [], [raw_response_body], #aws_config{}), - Expected = {ok, <<"{\"message\":\"Hello World!\"}">>}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:invoke(<<"name:alias">>, [], [raw_response_body], #aws_config{}), - Expected = {ok, <<"{\"message\":\"Hello World!\"}">>}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:invoke(<<"name_qualifier">>, [], [], <<"123">>), - Expected = {ok, [{<<"message">>, <<"Hello World!">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:list_aliases(<<"name">>), - Expected = {ok, [{<<"Aliases">>, - [[{<<"AliasArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, - {<<"Description">>,<<>>}, - {<<"FunctionVersion">>,<<"$LATEST">>}, - {<<"Name">>,<<"aliasName">>}], - [{<<"AliasArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName1">>}, - {<<"Description">>,<<>>}, - {<<"FunctionVersion">>,<<"$LATEST">>}, - {<<"Name">>,<<"aliasName1">>}]]}, - {<<"NextMarker">>,null}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:list_event_source_mappings( - <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>, - <<"name">>), - Expected = {ok, [{<<"EventSourceMappings">>, - [[{<<"BatchSize">>,100}, - {<<"EventSourceArn">>,<<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"LastModified">>,1449841860.0}, - {<<"LastProcessingResult">>,<<"No records processed">>}, - {<<"State">>,<<"Enabled">>}, - {<<"StateTransitionReason">>,<<"User action">>}, - {<<"UUID">>,<<"a45b58ec-a539-4c47-929e-174b4dd2d963">>}]]}, - {<<"NextMarker">>,null}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:list_versions_by_function(<<"name">>), - Expected = {ok, [{<<"NextMarker">>,null}, - {<<"Versions">>, - [[{<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>, 848}, - {<<"Description">>, <<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name:$LATEST">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-10T13:57:48.214+0000">>}, - {<<"MemorySize">>,512}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,30}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name:1">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-10T11:36:12.776+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,3}, - {<<"Version">>,<<"1">>}, - {<<"VpcConfig">>,null}], - [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name:2">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-10T13:56:43.171+0000">>}, - {<<"MemorySize">>,128}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,3}, - {<<"Version">>,<<"2">>}, - {<<"VpcConfig">>,null}]]}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:publish_version(<<"name">>), - Expected = {ok, [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name:3">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-10T13:57:48.214+0000">>}, - {<<"MemorySize">>,512}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,30}, - {<<"Version">>,<<"3">>}, - {<<"VpcConfig">>,null}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:update_alias(<<"name">>, <<"aliasName">>), - Expected = {ok, [{<<"AliasArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, - {<<"Description">>,<<>>}, - {<<"FunctionVersion">>,<<"$LATEST">>}, - {<<"Name">>,<<"aliasName">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:update_event_source_mapping( - <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>, - 100, undefined, undefined), - Expected = {ok, - [{<<"BatchSize">>,100}, - {<<"EventSourceArn">>,<<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"LastModified">>,1449844011.991}, - {<<"LastProcessingResult">>,<<"No records processed">>}, - {<<"State">>,<<"Updating">>}, - {<<"StateTransitionReason">>,<<"User action">>}, - {<<"UUID">>,<<"a45b58ec-a539-4c47-929e-174b4dd2d963">>}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:update_function_code( - <<"name">>, true, #erlcloud_lambda_code{ - s3Bucket = <<"bi-lambda">>, - s3Key = <<"local_transform/bi-assets-environment-create-environment_1-0-0_latest.zip">> - }), - Expected = {ok, - [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name:4">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-11T14:29:17.023+0000">>}, - {<<"MemorySize">>,512}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,30}, - {<<"Version">>,<<"4">>}, - {<<"VpcConfig">>,null}]}, - ?assertEqual(Expected, Result) - end, - fun() -> - Result = erlcloud_lambda:update_function_configuration( - <<"name">>, undefined, undefined, 512, undefined, 30), - Expected = {ok, [{<<"CodeSha256">>,<<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, - {<<"CodeSize">>,848}, - {<<"Description">>,<<>>}, - {<<"FunctionArn">>,<<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, - {<<"FunctionName">>,<<"name">>}, - {<<"Handler">>,<<"index.process">>}, - {<<"LastModified">>,<<"2015-12-11T14:31:52.034+0000">>}, - {<<"MemorySize">>,512}, - {<<"Role">>,<<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, - {<<"Runtime">>,<<"nodejs">>}, - {<<"Timeout">>,30}, - {<<"Version">>,<<"$LATEST">>}, - {<<"VpcConfig">>,null}]}, - ?assertEqual(Expected, Result) - end + fun() -> + Result = erlcloud_lambda:list_functions(), + Expected = + {ok, [ + {<<"Functions">>, [ + [ + {<<"CodeSha256">>, <<"XmLDAZXEkl5KbA8ezZpwFU+bjgTXBehUmWGOScl4F2A=">>}, + {<<"CodeSize">>, 5561}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:bi-assets-asset-create-host">>}, + {<<"FunctionName">>, <<"bi-assets-asset-create-host">>}, + {<<"Handler">>, <<"index.handler">>}, + {<<"LastModified">>, <<"2015-11-27T09:55:12.973+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, + <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"XmLDAZXEkl5KbA8ezZpwFU+bjgTXBehUmWGOScl4F2A=">>}, + {<<"CodeSize">>, 5561}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:bi-assets-asset-create-host1">>}, + {<<"FunctionName">>, <<"bi-assets-asset-create-host1">>}, + {<<"Handler">>, <<"index.handler">>}, + {<<"LastModified">>, <<"2015-12-01T11:20:44.464+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 10}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"tZJ+kUZVD1vGYwMIUvAoaZmvS4I9NHVc7a/267eChYY=">>}, + {<<"CodeSize">>, 132628}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:bi-driver">>}, + {<<"FunctionName">>, <<"bi-driver">>}, + {<<"Handler">>, <<"index.handler">>}, + {<<"LastModified">>, <<"2015-12-03T13:59:05.219+0000">>}, + {<<"MemorySize">>, 1024}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 59}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"QS10seyYGXrrhAnGMbJcTi+JOa4HWLaD+9YCLYG3+VE=">>}, + {<<"CodeSize">>, 121486}, + {<<"Description">>, + <<"An Amazon Kinesis stream processor that logs the data being published.">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:eholland-js-router">>}, + {<<"FunctionName">>, <<"eholland-js-router">>}, + {<<"Handler">>, <<"index.handler">>}, + {<<"LastModified">>, <<"2015-12-02T14:50:41.923+0000">>}, + {<<"MemorySize">>, 1024}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 60}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"ey+6CSWe750XoPSdVSQditxxoHNWmPFwve/MLPNs/Do=">>}, + {<<"CodeSize">>, 253}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-aims-user">>}, + {<<"FunctionName">>, <<"eholland-test-aims-user">>}, + {<<"Handler">>, <<"lambda_function.lambda_handler">>}, + {<<"LastModified">>, <<"2015-11-09T17:32:46.030+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, + <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"python2.7">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"R9HaKKCPPAem0ikKrkMGpDtX95M8egDCDd+4Ws+Kk5c=">>}, + {<<"CodeSize">>, 253}, + {<<"Description">>, <<"aims users transform function">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-aims-users">>}, + {<<"FunctionName">>, <<"eholland-test-aims-users">>}, + {<<"Handler">>, <<"lambda_function.lambda_handler">>}, + {<<"LastModified">>, <<"2015-11-02T13:59:32.509+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, + <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"python2.7">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"PGG1vCAQpc8J7e4HL1z1Pv4DGSZEmhnJaNPTGDS29kk=">>}, + {<<"CodeSize">>, 32372}, + {<<"Description">>, + <<"An Amazon Kinesis stream processor that logs the data being published.">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:eholland-test-router">>}, + {<<"FunctionName">>, <<"eholland-test-router">>}, + {<<"Handler">>, <<"ProcessKinesisRecords.lambda_handler">>}, + {<<"LastModified">>, <<"2015-11-09T10:58:50.458+0000">>}, + {<<"MemorySize">>, 256}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"python2.7">>}, + {<<"Timeout">>, 59}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"8twXwGaAXyr8Li81rGhM6vlMh+dvXglwqgIshfaio+U=">>}, + {<<"CodeSize">>, 708740}, + {<<"Description">>, <<"Lambda Router Function for ETL Service">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:us-east-1_base_eholland_master_lambda_route">>}, + {<<"FunctionName">>, <<"us-east-1_base_eholland_master_lambda_route">>}, + {<<"Handler">>, <<"ProcessKinesisRecords.handler">>}, + {<<"LastModified">>, <<"2015-11-11T10:13:19.043+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 59}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"aDwLJhljsMVHsaN+LM4jRmsSLKKBdS+eFrAhKmJ/zbQ=">>}, + {<<"CodeSize">>, 253}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:us-east-1_base_eholland_master_lambda_route-aims-user-create">>}, + {<<"FunctionName">>, + <<"us-east-1_base_eholland_master_lambda_route-aims-user-create">>}, + {<<"Handler">>, <<"lambda_function.lambda_handler">>}, + {<<"LastModified">>, <<"2015-11-11T09:42:10.303+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, + <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"python2.7">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name2">>}, + {<<"FunctionName">>, <<"name2">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T11:31:21.106+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, + <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ] + ]}, + {<<"NextMarker">>, null} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:create_alias( + <<"name">>, + <<"$LATEST">>, + <<"aliasName1">>, + [] + ), + Expected = + {ok, [ + {<<"AliasArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName1">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName1">>} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:create_event_source_mapping( + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>, + <<"name">>, + <<"TRIM_HORIZON">>, + [] + ), + Expected = + {ok, [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449845416.123}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Creating">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"3f303f86-7395-43f3-9902-f5c80f0a5382">>} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:create_function( + #erlcloud_lambda_code{ + s3Bucket = <<"bi-lambda">>, + s3Key = << + "local_transform/bi-ass" + "ets-environment-create" + "-environment_1-0-0_lat" + "est.zip" + >> + }, + <<"name">>, + <<"index.process">>, + <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>, + nodejs, + [] + ), + Expected = + {ok, [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name3">>}, + {<<"FunctionName">>, <<"name3">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-11T13:45:31.924+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_basic_execution">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:delete_event_source_mapping( + <<"6554f300-551b-46a6-829c-41b6af6022c6">> + ), + Expected = + {ok, [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449843960.0}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Deleting">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:delete_function( + <<"name">> + ), + Expected = {ok, []}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:delete_function( + <<"name_qualifier">>, <<"123">>, #aws_config{} + ), + Expected = {ok, []}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:get_alias(<<"name">>, <<"aliasName">>), + Expected = + {ok, [ + {<<"AliasArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName">>} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:get_event_source_mapping( + <<"a45b58ec-a539-4c47-929e-174b4dd2d963">> + ), + Expected = + {ok, [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449841860.0}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Enabled">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:get_function(<<"name">>), + Expected = + {ok, [ + {<<"Code">>, [ + {<<"Location">>, + <<"https://awslambda-us-east-1-tasks.s3-us-east-1.amazonaws.com/snapshots/352283894008/name-69237aec-bae9-4086-af73-a6610c2f8eb8?x-amz-security-token=AQoDYXdzENb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEa4APJSJNHsYWnObKlElri5ytVzNg0t%2F0zwADHn40jy%2F1ZDW9%2FYWGi8dTp1l6LWBI9TwJi0LLcgp%2FlCxJIh7hsAPftYX62J9r9lRcmgd9RnYssg1%2Fkpfyjya90epxKg2zdHm%2BuZGukHYDcmAE1IQcHwsaQbvGAjXPCpFyxClbV6gMcFIsaBtfMxoMcbTCXG9m8l56nKgcX6Mi60vRNaB83AeNVrKMhB8EBUUbYbaB%2BG0iJg32i2HBF6VJMxamOLIEf1GJp1tWt%2FSAHfEkdTwcwtGINH3TNRv%2BY3ddsXs8pJ49eY49NCHANPC%2Bq0JzNQydbIK1shz8w1nozXYQo6%2BNh9tqOlaJNFgfFbtJkUDXv4rFqgVsfgJKJSQBeYUKmlNvIPQIoHWhRjjRzQUGmYDc3eEug7vELsNcHZixI4nNVycH%2BJZwaBvswy4eE7gBv3HwHi3SVlg9iXFTrfWTK%2FlCybC7mZIjAmPGiLCG5Pu8SoCgaGdHp8HmSeXXus4VUFcVTUw1qn7E%2BSaRFg3MTpCdu1f1Nqh4pNu7GpZacyLbH%2BSocuPyTjyYGL8sk0C3rjIWZipJcfUZsleS1cXLEGw%2FPAK5eg0RNWlVsaF1WgVimqQh3LtoRZ%2BwYCHRXsHjhCyQ8VhoguJCrswU%3D&AWSAccessKeyId=ASIAIAUY4IMA6JRJ3FSA&Expires=1449843004&Signature=5KUmber1cOBSChlKtnLaI%2FUemU4%3D">>}, + {<<"RepositoryType">>, <<"S3">>} + ]}, + {<<"Configuration">>, [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ]} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:get_function_configuration(<<"name">>), + Expected = + {ok, [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:invoke(<<"name">>, []), + Expected = {ok, [{<<"message">>, <<"Hello World!">>}]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:invoke(<<"name">>, [], [{show_headers, true}], #aws_config{}), + Expected = {ok, [], [{<<"message">>, <<"Hello World!">>}]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:invoke(<<"name">>, [], [raw_response_body], #aws_config{}), + Expected = {ok, <<"{\"message\":\"Hello World!\"}">>}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:invoke( + <<"name:alias">>, [], [raw_response_body], #aws_config{} + ), + Expected = {ok, <<"{\"message\":\"Hello World!\"}">>}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:invoke(<<"name_qualifier">>, [], [], <<"123">>), + Expected = {ok, [{<<"message">>, <<"Hello World!">>}]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:list_aliases(<<"name">>), + Expected = + {ok, [ + {<<"Aliases">>, [ + [ + {<<"AliasArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName">>} + ], + [ + {<<"AliasArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName1">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName1">>} + ] + ]}, + {<<"NextMarker">>, null} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:list_event_source_mappings( + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>, + <<"name">> + ), + Expected = + {ok, [ + {<<"EventSourceMappings">>, [ + [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449841860.0}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Enabled">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>} + ] + ]}, + {<<"NextMarker">>, null} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:list_versions_by_function(<<"name">>), + Expected = + {ok, [ + {<<"NextMarker">>, null}, + {<<"Versions">>, [ + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:$LATEST">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:1">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T11:36:12.776+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"1">>}, + {<<"VpcConfig">>, null} + ], + [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:2">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:56:43.171+0000">>}, + {<<"MemorySize">>, 128}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 3}, + {<<"Version">>, <<"2">>}, + {<<"VpcConfig">>, null} + ] + ]} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:publish_version(<<"name">>), + Expected = + {ok, [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:3">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-10T13:57:48.214+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"3">>}, + {<<"VpcConfig">>, null} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:update_alias(<<"name">>, <<"aliasName">>), + Expected = + {ok, [ + {<<"AliasArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:aliasName">>}, + {<<"Description">>, <<>>}, + {<<"FunctionVersion">>, <<"$LATEST">>}, + {<<"Name">>, <<"aliasName">>} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:update_event_source_mapping( + <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>, + 100, + undefined, + undefined + ), + Expected = + {ok, [ + {<<"BatchSize">>, 100}, + {<<"EventSourceArn">>, + <<"arn:aws:kinesis:us-east-1:352283894008:stream/eholland-test">>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"LastModified">>, 1449844011.991}, + {<<"LastProcessingResult">>, <<"No records processed">>}, + {<<"State">>, <<"Updating">>}, + {<<"StateTransitionReason">>, <<"User action">>}, + {<<"UUID">>, <<"a45b58ec-a539-4c47-929e-174b4dd2d963">>} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:update_function_code( + <<"name">>, true, #erlcloud_lambda_code{ + s3Bucket = <<"bi-lambda">>, + s3Key = + <<"local_transform/bi-assets-environment-create-environment_1-0-0_latest.zip">> + } + ), + Expected = + {ok, [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, + <<"arn:aws:lambda:us-east-1:352283894008:function:name:4">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-11T14:29:17.023+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"4">>}, + {<<"VpcConfig">>, null} + ]}, + ?assertEqual(Expected, Result) + end, + fun() -> + Result = erlcloud_lambda:update_function_configuration( + <<"name">>, undefined, undefined, 512, undefined, 30 + ), + Expected = + {ok, [ + {<<"CodeSha256">>, <<"zeoBX1hIWJBHk1muJe1iFyS1CcAmsT0Ct4IcpiPf8QM=">>}, + {<<"CodeSize">>, 848}, + {<<"Description">>, <<>>}, + {<<"FunctionArn">>, <<"arn:aws:lambda:us-east-1:352283894008:function:name">>}, + {<<"FunctionName">>, <<"name">>}, + {<<"Handler">>, <<"index.process">>}, + {<<"LastModified">>, <<"2015-12-11T14:31:52.034+0000">>}, + {<<"MemorySize">>, 512}, + {<<"Role">>, <<"arn:aws:iam::352283894008:role/lambda_kinesis_role">>}, + {<<"Runtime">>, <<"nodejs">>}, + {<<"Timeout">>, 30}, + {<<"Version">>, <<"$LATEST">>}, + {<<"VpcConfig">>, null} + ]}, + ?assertEqual(Expected, Result) + end ]. +make_response(Value) when is_binary(Value) -> + make_response({200, <<"OK">>}, Value); make_response(Value) -> - make_response({200, <<"OK">>}, Value). + make_response({200, <<"OK">>}, jsx:encode(Value)). make_response(Status, Value) -> {ok, {Status, [], Value}}. From 8a172f15c36636d067c160ec26e9c8fcf3dcc58b Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Wed, 30 Oct 2024 22:21:59 -0500 Subject: [PATCH 296/310] Fix OTP 27+ warnings about updating literals --- rebar.config | 2 +- test/erlcloud_cognito_user_pools_tests.erl | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rebar.config b/rebar.config index 937a7ceac..f2da43ece 100644 --- a/rebar.config +++ b/rebar.config @@ -47,4 +47,4 @@ {plt_location, ".dialyzer_plt"}, {plt_apps, all_deps}, {plt_extra_apps, [hackney]} - ]}. \ No newline at end of file + ]}. diff --git a/test/erlcloud_cognito_user_pools_tests.erl b/test/erlcloud_cognito_user_pools_tests.erl index e2f7aaec5..c311a1f8d 100644 --- a/test/erlcloud_cognito_user_pools_tests.erl +++ b/test/erlcloud_cognito_user_pools_tests.erl @@ -585,7 +585,7 @@ test_list_users() -> do_test(Request, Expected, TestFun). test_list_all_users() -> - Mocked1 = ?LIST_USERS_RESP#{<<"PaginationToken">> => "1"}, + Mocked1 = maps:put(<<"PaginationToken">>, "1", ?LIST_USERS_RESP), Mocked2 = #{<<"Users">> => [ #{ <<"Attributes">> => @@ -858,7 +858,7 @@ test_list_user_pool_clients() -> do_test(Request, Expected, TestFun). test_list_all_user_pool_clients() -> - Mocked1 = ?LIST_USER_POOL_CLIENTS#{<<"NextToken">> => <<"next">>}, + Mocked1 = maps:put(<<"NextToken">>, <<"next">>, ?LIST_USER_POOL_CLIENTS), Mocked2 = ?LIST_USER_POOL_CLIENTS, meck:sequence(?EHTTPC, request, 6, [{ok, {{200, "OK"}, [], jsx:encode(Mocked1)}}, {ok, {{200, "OK"}, [], jsx:encode(Mocked2)}}]), @@ -877,9 +877,9 @@ test_admin_list_devices() -> do_test(Request, Expected, TestFun). test_list_all_devices() -> - Mocked1 = ?ADMIN_LIST_DEVICE#{<<"PaginationToken">> => <<"next page">>}, + Mocked1 = maps:put(<<"PaginationToken">>, <<"next page">>, ?ADMIN_LIST_DEVICE), Devices = hd(maps:get(<<"Devices">>, ?ADMIN_LIST_DEVICE)), - Mocked2 = ?ADMIN_LIST_DEVICE#{<<"Devices">> => [Devices#{<<"DeviceKey">> => <<"testKey2">>}]}, + Mocked2 = maps:put(<<"Devices">>, [Devices#{<<"DeviceKey">> => <<"testKey2">>}], ?ADMIN_LIST_DEVICE), meck:sequence(?EHTTPC, request, 6, [{ok, {{200, "OK"}, [], jsx:encode(Mocked1)}}, {ok, {{200, "OK"}, [], jsx:encode(Mocked2)}}]), Expected = {ok, ?LIST_ALL_DEVICES}, From 2949b13b4445043c4731c6368839b7f0f836b9cf Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Wed, 30 Oct 2024 23:34:20 -0500 Subject: [PATCH 297/310] Fix a few xmerl related type definitions for dialyzer --- include/erlcloud_xmerl.hrl | 2 +- src/erlcloud_cloudfront.erl | 6 +++--- src/erlcloud_ec2.erl | 4 ++++ src/erlcloud_elb.erl | 7 ++++--- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/include/erlcloud_xmerl.hrl b/include/erlcloud_xmerl.hrl index c3c481af9..08228ee04 100644 --- a/include/erlcloud_xmerl.hrl +++ b/include/erlcloud_xmerl.hrl @@ -3,7 +3,7 @@ -include_lib("xmerl/include/xmerl.hrl"). --type(xmerl_xpath_doc_nodes() :: #xmlElement{} | #xmlAttribute{} | #xmlText{} | #xmlPI{} | #xmlComment{} | #xmlNsNode{}). +-type(xmerl_xpath_doc_nodes() :: #xmlElement{} | #xmlAttribute{} | #xmlText{} | #xmlPI{} | #xmlComment{} | #xmlNsNode{} | #xmlObj{}). -type(xmerl_xpath_node_entity() :: #xmlDocument{} | xmerl_xpath_doc_nodes()). -type(xmerl_xpath_doc_entity() :: #xmlDocument{} | [xmerl_xpath_doc_nodes()]). diff --git a/src/erlcloud_cloudfront.erl b/src/erlcloud_cloudfront.erl index 702ce07da..be20478ad 100644 --- a/src/erlcloud_cloudfront.erl +++ b/src/erlcloud_cloudfront.erl @@ -20,7 +20,7 @@ -define(MAX_RESULTS, 100). --spec extract_distribution_summary(Node :: list()) -> proplist(). +-spec extract_distribution_summary(Node :: xmerl_xpath_doc_nodes()) -> proplist(). extract_distribution_summary(Node) -> erlcloud_xml:decode( [ @@ -58,7 +58,7 @@ extract_distribution(Node) -> ], Node) ++ extract_distribution_config(hd(xmerl_xpath:string("DistributionConfig", Node))). --spec extract_distribution_config(Node :: list()) -> proplist(). +-spec extract_distribution_config(Node :: xmerl_xpath_doc_nodes()) -> proplist(). extract_distribution_config(Node) -> erlcloud_xml:decode( [ @@ -79,7 +79,7 @@ extract_distribution_config(Node) -> ], Node). --spec extract_cache_behavior(Node :: list()) -> proplist(). +-spec extract_cache_behavior(Node :: xmerl_xpath_doc_nodes()) -> proplist(). extract_cache_behavior(Node) -> erlcloud_xml:decode( [ diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index 564ae99f9..ab033bf82 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -524,7 +524,11 @@ bundle_instance(InstanceID, Bucket, Prefix, AccessKeyID, UploadPolicy, Error end. +-spec extract_bundle_task(Nodes :: [xmerl_xpath_doc_nodes()]) -> proplist(); + (Node :: xmerl_xpath_doc_nodes()) -> proplist(). extract_bundle_task([Node]) -> + extract_bundle_task(Node); +extract_bundle_task(Node) -> [ {instance_id, get_text("instanceId", Node)}, {bundle_id, get_text("bundleId", Node)}, diff --git a/src/erlcloud_elb.erl b/src/erlcloud_elb.erl index 37a0af8e4..e7ba68f1b 100644 --- a/src/erlcloud_elb.erl +++ b/src/erlcloud_elb.erl @@ -34,6 +34,7 @@ -include("erlcloud.hrl"). -include("erlcloud_aws.hrl"). +-include("erlcloud_xmerl.hrl"). -define(API_VERSION, "2012-06-01"). @@ -471,11 +472,11 @@ create_load_balancer_policy(LB, PolicyName, PolicyTypeName, AttrList, Config) ok. --spec describe_load_balancer_attributes(string()) -> proplist(). +-spec describe_load_balancer_attributes(string()) -> proplist() | no_return(). describe_load_balancer_attributes(Name) -> describe_load_balancer_attributes(Name, default_config()). --spec describe_load_balancer_attributes(string(), aws_config()) -> proplist(). +-spec describe_load_balancer_attributes(string(), aws_config()) -> proplist() | no_return(). describe_load_balancer_attributes(Name, Config) -> Node = elb_request(Config, "DescribeLoadBalancerAttributes", @@ -486,7 +487,7 @@ describe_load_balancer_attributes(Name, Config) -> %%%=================================================================== %%% Internal functions %%%=================================================================== --spec extract_elb_attribs(proplist()) -> proplist(). +-spec extract_elb_attribs(xmerl_xpath_doc_nodes()) -> proplist() | no_return(). extract_elb_attribs(Node) -> RootPath = "DescribeLoadBalancerAttributesResult/LoadBalancerAttributes", erlcloud_xml:decode( From 1e59eb31bf37ff921af7e8cb29637e1f65956a1b Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Wed, 30 Oct 2024 23:40:02 -0500 Subject: [PATCH 298/310] Remove erlfmt from project_plugins, added it accidentally --- rebar.config | 2 -- 1 file changed, 2 deletions(-) diff --git a/rebar.config b/rebar.config index f2da43ece..264384b83 100644 --- a/rebar.config +++ b/rebar.config @@ -37,8 +37,6 @@ {warnings, [{erl_opts, [warnings_as_errors]}]} ]}. -{project_plugins, [erlfmt]}. - {post_hooks, [{clean, "rm -rf .dialyzer_plt"}]}. {pre_hooks, [{clean, "rm -rf erl_crash.dump *.log"}]}. From b89b7e47eadd9eb8805b8db70083f799ffc42a09 Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Thu, 31 Oct 2024 11:15:03 -0500 Subject: [PATCH 299/310] Update supported versions section of readme with latest tested versions --- README.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 013596034..fe1adc1cf 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,19 @@ At the moment we support the following OTP releases: - 20.3 - 21.3 - 22.3 - -It might still work on 17+ (primarily due to Erlang maps) but we do not -guarantee that. +- 23.3 +- 24.3 +- 25.3 +- 26.2 +- 27.1 + +This list is determined by ensuring eunit tests and dialyzer checks succeed for these versions, but not all of these +versions are in active use by library authors. Please report any issues discovered in actual use. + +The Github Actions test runners only support OTP 24+ due to runtime issues, but OTP 19-23 were tested locally with unmodified, +official [Erlang docker images](https://hub.docker.com/_/erlang). Dialyzer checks run against the latest hex-published hackney +for OTP 24+, but a previous versions of hackney (1.15.0) and its dependency parse_trans (3.2.0) were used to do dialyzer checks +for OTP 19-23 due to newer versions of parse_trans requiring OTP 21+. ## Getting started From 34b7f8866f15ae54bfd487274f45a8970a946c68 Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Thu, 31 Oct 2024 15:09:13 -0500 Subject: [PATCH 300/310] Add hex.pm publish on tag push * Add Github Actions job to publish to hex.pm on tag pushes --- .github/workflows/hex_publish.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/hex_publish.yml diff --git a/.github/workflows/hex_publish.yml b/.github/workflows/hex_publish.yml new file mode 100644 index 000000000..a2d9d6004 --- /dev/null +++ b/.github/workflows/hex_publish.yml @@ -0,0 +1,23 @@ +--- +name: Publish to Hex.pm +on: + push: + tags: + - '*' + +jobs: + publish: + runs-on: ubuntu-latest + container: erlang:27 + + steps: + - name: Check out + uses: actions/checkout@v4 + + - name: 'Fix up git directory ownership (see https://github.com/actions/checkout/issues/1049)' + run: git config --global --add safe.directory '*' + + - name: Publish to Hex.pm + env: + HEX_API_KEY: ${{ secrets.HEX_API_KEY }} + run: rebar3 hex publish package -r hexpm --yes From 438d88c51dedea5cba110a278b74bead79f4a45c Mon Sep 17 00:00:00 2001 From: Richard Palomino Date: Tue, 5 Nov 2024 21:05:57 -0600 Subject: [PATCH 301/310] Add rebar3_hex as a project_plugin for publishing * Add rebar3_hex as a project_plugin for publishing --- rebar.config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rebar.config b/rebar.config index 264384b83..d3ebdbe12 100644 --- a/rebar.config +++ b/rebar.config @@ -23,6 +23,8 @@ {base16, "1.0.0"} ]}. +{project_plugins, [rebar3_hex]}. + {overrides, [ %% do not pull in the covertool plugin or repo, cause it fetches rebar and From f997b37ecd760e9e0ba8e86f9e3315fe684b7309 Mon Sep 17 00:00:00 2001 From: Ariel Otilibili Date: Sat, 23 Nov 2024 14:38:10 +0100 Subject: [PATCH 302/310] Removed unused macros in Makefile ``` $ grep CHECK Makefile CHECK_FILES=\ CHECK_EUNIT_FILES=\ ``` Signed-off-by: Ariel Otilibili --- Makefile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Makefile b/Makefile index 0fa960c03..79672c001 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,6 @@ REBAR=$(shell which rebar3 || echo ./rebar3) -CHECK_FILES=\ - ebin/*.beam - -CHECK_EUNIT_FILES=\ - .eunit/*.beam - all: compile clean: From 137caac6269c05a90945675876412bdc34442ecc Mon Sep 17 00:00:00 2001 From: Nelson Vides Date: Wed, 4 Dec 2024 11:44:49 +0100 Subject: [PATCH 303/310] Upgrade base16 library --- rebar.config | 2 +- rebar.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rebar.config b/rebar.config index d3ebdbe12..2397ce3a7 100644 --- a/rebar.config +++ b/rebar.config @@ -20,7 +20,7 @@ {jsx, "2.11.0"}, {lhttpc, "1.7.1"}, {eini, "1.2.9"}, - {base16, "1.0.0"} + {base16, "2.0.1"} ]}. {project_plugins, [rebar3_hex]}. diff --git a/rebar.lock b/rebar.lock index 12f8f3b74..8e7b26280 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,16 +1,16 @@ {"1.2.0", -[{<<"base16">>,{pkg,<<"base16">>,<<"1.0.0">>},0}, +[{<<"base16">>,{pkg,<<"base16">>,<<"2.0.1">>},0}, {<<"eini">>,{pkg,<<"eini">>,<<"1.2.9">>},0}, {<<"jsx">>,{pkg,<<"jsx">>,<<"2.11.0">>},0}, {<<"lhttpc">>,{pkg,<<"lhttpc">>,<<"1.7.1">>},0}]}. [ {pkg_hash,[ - {<<"base16">>, <<"283644E2B21BD5915ACB7178BED7851FB07C6E5749B8FAD68A53C501092176D9">>}, + {<<"base16">>, <<"F0549F732E03BE8124ED0D19FD5EE52146CC8BE24C48CBC3F23AB44B157F11A2">>}, {<<"eini">>, <<"FCC3CBD49BBDD9A1D9735C7365DAFFCD84481CCE81E6CB80537883AA44AC4895">>}, {<<"jsx">>, <<"08154624050333919B4AC1B789667D5F4DB166DC50E190C4D778D1587F102EE0">>}, {<<"lhttpc">>, <<"8522AF9877765C33318A3AE486BE69BC165E835D05C3334A8166FD7B318D446B">>}]}, {pkg_hash_ext,[ - {<<"base16">>, <<"02AFD0827E61A7B07093873E063575CA3A2B07520567C7F8CEC7C5D42F052D76">>}, + {<<"base16">>, <<"06EA2D48343282E712160BA89F692B471DB8B36ABE8394F3445FF9032251D772">>}, {<<"eini">>, <<"DA64AE8DB7C2F502E6F20CDF44CD3D9BE364412B87FF49FEBF282540F673DFCB">>}, {<<"jsx">>, <<"EED26A0D04D217F9EECEFFFB89714452556CF90EB38F290A27A4D45B9988F8C0">>}, {<<"lhttpc">>, <<"154EEB27692482B52BE86406DCD1D18A2405CAFCE0E8DAA4A1A7BFA7FE295896">>}]} From 78ff6b07ec8732b9038fa64df97b6e6afa6d6a9c Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 31 Jan 2025 18:15:38 +0000 Subject: [PATCH 304/310] ec2: add `metadata_options` property to `describe_instances()` output --- src/erlcloud_ec2.erl | 3 +- test/erlcloud_ec2_tests.erl | 267 +++++++++++++++++++++++++++++++++++- 2 files changed, 268 insertions(+), 2 deletions(-) diff --git a/src/erlcloud_ec2.erl b/src/erlcloud_ec2.erl index ab033bf82..f37b1dc93 100644 --- a/src/erlcloud_ec2.erl +++ b/src/erlcloud_ec2.erl @@ -1512,6 +1512,7 @@ extract_instance(Node) -> {dns_name, get_text("dnsName", Node)}, {reason, get_text("reason", Node, none)}, {key_name, get_text("keyName", Node, none)}, + {metadata_options, transform_item_list(Node, "metadataOptions", fun extract_metadata_options/1)}, {ami_launch_index, list_to_integer(get_text("amiLaunchIndex", Node, "0"))}, {product_codes, get_list("productCodes/item/productCode", Node)}, {instance_type, get_text("instanceType", Node)}, @@ -4382,4 +4383,4 @@ extract_launch_template_data(Node) -> {security_group_set, [get_text(Item) || Item <- xmerl_xpath:string("securityGroupSet/item", Node)]}, {tag_specification_set, transform_item_list(Node, "tagSpecificationSet/item", fun extract_tag_specification/1)}, {user_data, get_text("userData", Node)} - ]. \ No newline at end of file + ]. diff --git a/test/erlcloud_ec2_tests.erl b/test/erlcloud_ec2_tests.erl index 4b241583c..feaa0945d 100644 --- a/test/erlcloud_ec2_tests.erl +++ b/test/erlcloud_ec2_tests.erl @@ -65,7 +65,8 @@ describe_test_() -> fun describe_launch_template_versions_input_tests/1, fun describe_launch_template_versions_output_tests/1, fun describe_security_groups_input_tests/1, - fun describe_security_groups_output_tests/1 + fun describe_security_groups_output_tests/1, + fun describe_instances_output_tests/1 ]}. start() -> @@ -1383,6 +1384,262 @@ describe_instances_test_() -> {timeout, 60, fun () -> test_pagination(Tests, generate_instances_response, describe_instances, [], [[]]) end} . +describe_instances_output_tests(_) -> + Tests = + [?_ec2_test( + {"This example describes all instances", " + + 95bd274e-560d-46c8-82da-bf41da8f2caf + + + r-000001 + 123456789012 + + + + i-000001 + ami-000001 + + 16 + running + + ip-10-0-1-1.ec2.internal + + + key + 0 + + + false + + m3.medium + 2025-01-31T09:01:15.000Z + + us-east-1a + + default + + + disabled + + subnet-00001 + vpc-00001 + 10.0.1.1 + 12.1.2.3 + true + + + sg-00001 + security-group-01 + + + x86_64 + ebs + /dev/sda1 + + + /dev/sda1 + + vol-00001 + attached + 2025-01-31T09:00:34.000Z + true + + + + hvm + + + + Name + test-instance + + + xen + + + eni-00001 + subnet-00001 + vpc-00001 + Primary network interface + 123456789012 + in-use + 02:bb:10:1a:1a:1a + 10.0.1.1 + true + + + sg-00001 + security-group-01 + + + + eni-attach-00001 + 0 + attached + 2025-01-31T09:00:34.000Z + true + 0 + + + 123.12.123.12 + + amazon + + + + 10.0.1.1 + true + + 123.12.123.12 + + amazon + + + + + interface + + false + + + + false + true + + 1 + 1 + + + open + + + false + + + false + + + applied + optional + 1 + enabled + disabled + disabled + + + default + default + + + default + + Linux/UNIX + RunInstances + 2025-01-31T09:00:34.000Z + + + + + + + ", + {ok, [[ + {reservation_id, "r-000001"}, + {owner_id, "123456789012"}, + {instances_set, [[ + {instance_id, "i-000001"}, + {group_set, [[ + {group_id, "sg-00001"}, + {group_name, "security-group-01"} + ]]}, + {image_id, "ami-000001"}, + {instance_state, [{code, 16}, {name, "running"}]}, + {private_dns_name, "ip-10-0-1-1.ec2.internal"}, + {dns_name, []}, + {reason, none}, + {key_name, "key"}, + {metadata_options, [[ + {http_endpoint, "enabled"}, + {http_protocol_ipv6, "disabled"}, + {http_put_response_hop_limit, 1}, + {http_tokens, "optional"}, + {instance_metadata_tags, "disabled"}, + {state, "applied"} + ]]}, + {ami_launch_index, 0}, + {product_codes, []}, + {instance_type, "m3.medium"}, + {launch_time, {{2025, 1, 31}, {9, 1, 15}}}, + {platform, []}, + {placement, [{availability_zone, "us-east-1a"}]}, + {kernel_id, []}, + {ramdisk_id, []}, + {monitoring, [{enabled, false}, {state, "disabled"}]}, + {subnet_id, "subnet-00001"}, + {vpc_id, "vpc-00001"}, + {private_ip_address, "10.0.1.1"}, + {ip_address, "12.1.2.3"}, + {state_reason, [{code, []}, {message, []}]}, + {architecture, "x86_64"}, + {root_device_type, "ebs"}, + {root_device_name, "/dev/sda1"}, + {block_device_mapping, [[ + {device_name, "/dev/sda1"}, + {volume_id, "vol-00001"}, + {status, "attached"}, + {attach_time, {{2025, 1, 31}, {9, 0, 34}}}, + {delete_on_termination, true} + ]]}, + {instance_lifecycle, none}, + {spot_instance_request_id, none}, + {iam_instance_profile, [{arn, []}, {id, []}]}, + {tag_set, [[{key, "Name"}, {value, "test-instance"}]]}, + {network_interface_set, [[ + {network_interface_id, "eni-00001"}, + {subnet_id, "subnet-00001"}, + {vpc_id, "vpc-00001"}, + {availability_zone, []}, + {description, "Primary network interface"}, + {owner_id, "123456789012"}, + {requester_managed, false}, + {status, "in-use"}, + {mac_address, "02:bb:10:1a:1a:1a"}, + {private_ip_address, "10.0.1.1"}, + {source_dest_check, true}, + {groups_set, [[ + {group_id, "sg-00001"}, + {group_name, "security-group-01"} + ]]}, + {attachment, [ + {attachment_id, "eni-attach-00001"}, + {instance_id, []}, + {instance_owner_id, []}, + {device_index, "0"}, + {status, "attached"}, + {attach_time, {{2025, 1, 31}, {9, 0, 34}}}, + {delete_on_termination, true} + ]}, + {association, [ + {public_ip, "123.12.123.12"}, + {public_dns_name, []}, + {ip_owner_id, "amazon"}, + {allocation_id, []}, + {association_id, []} + ]}, + {tag_set, []}, + {private_ip_addresses_set, [[ + {private_ip_address, "10.0.1.1"}, + {primary, true} + ]]} + ]]} + ]]} + ]]}} + ) + ], + output_tests(?_f(erlcloud_ec2:describe_instances()), Tests) +. + describe_instances_boundaries_test_() -> [ ?_assertException(error, function_clause, erlcloud_ec2:describe_instances([], 4, undefined)), @@ -1914,6 +2171,14 @@ generate_one_instance(N) -> xen false + + applied + optional + 1 + enabled + disabled + disabled +     ". From 620e22aae850bff5494545f2804b3564281c8eb4 Mon Sep 17 00:00:00 2001 From: Scott Jones Date: Fri, 14 Feb 2025 11:38:29 +0000 Subject: [PATCH 305/310] efs: Implement `DescribeFileSystems` API Introduce support for the `elasticfilesystem:DescribeFileSystems` operation via the `erlcloud_efs` module. --- include/erlcloud_aws.hrl | 3 + src/erlcloud_aws.erl | 3 + src/erlcloud_efs.erl | 214 ++++++++++++++++++++ test/erlcloud_efs_tests.erl | 379 ++++++++++++++++++++++++++++++++++++ 4 files changed, 599 insertions(+) create mode 100644 src/erlcloud_efs.erl create mode 100644 test/erlcloud_efs_tests.erl diff --git a/include/erlcloud_aws.hrl b/include/erlcloud_aws.hrl index 0068854ed..a7f9120b7 100644 --- a/include/erlcloud_aws.hrl +++ b/include/erlcloud_aws.hrl @@ -118,6 +118,9 @@ ecs_scheme="https://"::string(), ecs_host="ecs.us-east-1.amazonaws.com"::string(), ecs_port=443::non_neg_integer(), + efs_host="elasticfilesystem.us-east-1.amazonaws.com"::string(), + efs_port=443::non_neg_integer(), + efs_scheme="https://"::string(), mes_scheme="https://"::string(), mes_host="entitlement.marketplace.us-east-1.amazonaws.com"::string(), mes_port=443::non_neg_integer(), diff --git a/src/erlcloud_aws.erl b/src/erlcloud_aws.erl index 86e555f7d..c81e20b2e 100644 --- a/src/erlcloud_aws.erl +++ b/src/erlcloud_aws.erl @@ -712,6 +712,9 @@ service_config( <<"ec2">> = Service, Region, Config ) -> service_config( <<"ecs">> = Service, Region, Config ) -> Host = service_host( Service, Region ), Config#aws_config{ ecs_host = Host }; +service_config( <<"efs">> = Service, Region, Config ) -> + Host = service_host( Service, Region ), + Config#aws_config{ efs_host = Host }; service_config( <<"elasticloadbalancing">> = Service, Region, Config ) -> Host = service_host( Service, Region ), Config#aws_config{ elb_host = Host }; diff --git a/src/erlcloud_efs.erl b/src/erlcloud_efs.erl new file mode 100644 index 000000000..1cddd8035 --- /dev/null +++ b/src/erlcloud_efs.erl @@ -0,0 +1,214 @@ +-module(erlcloud_efs). + +-include("erlcloud.hrl"). +-include("erlcloud_aws.hrl"). + +%% API +-export([ + describe_file_systems/0, + describe_file_systems/1, + describe_file_systems/2, + describe_file_systems/3, + describe_file_systems_all/0, + describe_file_systems_all/1, + describe_file_systems_all/2, + describe_file_systems_all/3, + describe_file_systems_all/4 +]). + +-type params() :: [param_name() | {param_name(), param_value()}]. +-type param_name() :: binary() | string() | atom() | integer(). +-type param_value() :: binary() | string() | atom() | integer(). + +-type file_system() :: proplist(). +-type file_systems() :: [file_system()]. +-type token() :: binary() | undefined. + +-define(EFS_API_VERSION, "2015-02-01"). + +%% ----------------------------------------------------------------------------- +%% Exported functions +%% ----------------------------------------------------------------------------- + +-spec describe_file_systems() -> Result when + Result :: {ok, file_systems(), token()} | {error, term()}. +describe_file_systems() -> + AwsConfig = erlcloud_aws:default_config(), + describe_file_systems(AwsConfig). + +-spec describe_file_systems(Arg) -> Result when + Arg :: aws_config() | params(), + Result :: {ok, file_systems(), token()} | {error, term()}. +describe_file_systems(AwsConfig) when is_record(AwsConfig, aws_config) -> + describe_file_systems(AwsConfig, _Params = []); +describe_file_systems(Params) -> + AwsConfig = erlcloud_aws:default_config(), + describe_file_systems(AwsConfig, Params). + +-spec describe_file_systems(AwsConfig, Params) -> Result when + AwsConfig :: aws_config(), + Params :: params(), + Result :: {ok, file_systems(), token()} | {error, term()}. +describe_file_systems(AwsConfig, Params) when is_record(AwsConfig, aws_config) -> + Path = [?EFS_API_VERSION, "file-systems"], + case request(AwsConfig, _Method = get, Path, Params) of + {ok, Response} -> + FileSystems = proplists:get_value(<<"FileSystems">>, Response), + case proplists:get_value(<<"NextMarker">>, Response, null) of + null -> + {ok, FileSystems, undefined}; + Marker -> + {ok, FileSystems, Marker} + end; + {error, Reason} -> + {error, Reason} + end. + +-spec describe_file_systems(AwsConfig, Params, Token) -> Result when + AwsConfig :: aws_config(), + Params :: params(), + Token :: token(), + Result :: {ok, file_systems(), token()} | {error, term()}. +describe_file_systems(AwsConfig, Params, Token) when + is_record(AwsConfig, aws_config), is_binary(Token) +-> + describe_file_systems(AwsConfig, [{<<"Marker">>, Token} | Params]); +describe_file_systems(AwsConfig, Params, _Token = undefined) -> + describe_file_systems(AwsConfig, Params). + +-spec describe_file_systems_all() -> Result when + Result :: {ok, file_systems()} | {error, term()}. +describe_file_systems_all() -> + AwsConfig = erlcloud_aws:default_config(), + describe_file_systems_all(AwsConfig). + +-spec describe_file_systems_all(Arg) -> Result when + Arg :: aws_config() | params(), + Result :: {ok, file_systems()} | {error, term()}. +describe_file_systems_all(AwsConfig) when is_record(AwsConfig, aws_config) -> + describe_file_systems_all(AwsConfig, _Params = []); +describe_file_systems_all(Params) when is_list(Params) -> + AwsConfig = erlcloud_aws:default_config(), + describe_file_systems_all(AwsConfig, Params). + +-spec describe_file_systems_all(AwsConfig, Params) -> Result when + AwsConfig :: aws_config(), + Params :: params(), + Result :: {ok, file_systems()} | {error, term()}. +describe_file_systems_all(AwsConfig, Params) when is_record(AwsConfig, aws_config) -> + describe_file_systems_all(AwsConfig, Params, _Token = undefined). + +-spec describe_file_systems_all(AwsConfig, Params, Token) -> Result when + AwsConfig :: aws_config(), + Params :: params(), + Token :: token() | undefined, + Result :: {ok, file_systems()} | {error, term()}. +describe_file_systems_all(AwsConfig, Params, Token) -> + describe_file_systems_all(AwsConfig, Params, Token, _Acc = []). + +-spec describe_file_systems_all(AwsConfig, Params, Token, Acc) -> Result when + AwsConfig :: aws_config(), + Params :: params(), + Token :: token() | undefined, + Acc :: [file_systems()], + Result :: {ok, file_systems()} | {error, term()}. +describe_file_systems_all(AwsConfig, Params, Token, Acc) -> + case describe_file_systems(AwsConfig, Params, Token) of + {ok, FileSystems, undefined} -> + {ok, flatten_pages([FileSystems | Acc])}; + {ok, FileSystems, NextToken} -> + describe_file_systems_all(AwsConfig, Params, NextToken, [FileSystems | Acc]); + {error, Reason} -> + {error, Reason} + end. + +%% ----------------------------------------------------------------------------- +%% Local functions +%% ----------------------------------------------------------------------------- + +request(AwsConfig, Method, Path, Params) -> + request(AwsConfig, Method, Path, Params, _RequestBody = <<>>). + +request(AwsConfig0, Method, Path, Params, RequestBody) -> + case erlcloud_aws:update_config(AwsConfig0) of + {ok, AwsConfig1} -> + AwsRequest0 = init_request(AwsConfig1, Method, Path, Params, RequestBody), + AwsRequest1 = erlcloud_retry:request(AwsConfig1, AwsRequest0, fun should_retry/1), + case AwsRequest1#aws_request.response_type of + ok -> + decode_response(AwsRequest1); + error -> + decode_error(AwsRequest1) + end; + {error, Reason} -> + {error, Reason} + end. + +init_request(AwsConfig, Method, Path, Params, Payload) -> + Scheme = AwsConfig#aws_config.efs_scheme, + Host = AwsConfig#aws_config.efs_host, + Port = AwsConfig#aws_config.efs_port, + Service = "elasticfilesystem", + NormPath = norm_path(Path), + NormParams = norm_params(Params), + Region = erlcloud_aws:aws_region_from_host(Host), + Headers = [{"host", Host}, {"content-type", "application/json"}], + SignedHeaders = erlcloud_aws:sign_v4( + Method, NormPath, AwsConfig, Headers, Payload, Region, Service, Params + ), + #aws_request{ + service = efs, + method = Method, + uri = Scheme ++ Host ++ ":" ++ integer_to_list(Port) ++ NormPath ++ NormParams, + request_headers = SignedHeaders, + request_body = Payload + }. + +norm_path(Path) -> + binary_to_list(iolist_to_binary(["/" | lists:join("/", Path)])). + +norm_params([] = _Params) -> + ""; +norm_params(Params) -> + "?" ++ erlcloud_aws:canonical_query_string(Params). + +should_retry(#aws_request{response_type = ok} = AwsRequest) -> + AwsRequest; +should_retry(#aws_request{response_type = error, response_status = Status} = AwsRequest) when + Status == 429; Status >= 500 +-> + AwsRequest#aws_request{should_retry = true}; +should_retry(#aws_request{} = AwsRequest) -> + AwsRequest#aws_request{should_retry = false}. + +decode_response(#aws_request{response_body = <<>>}) -> + ok; +decode_response(#aws_request{response_body = ResponseBody}) -> + Json = jsx:decode(ResponseBody, [{return_maps, false}]), + {ok, Json}. + +decode_error(#aws_request{error_type = aws} = AwsRequest) -> + Type = extract_error_type(AwsRequest), + Message = extract_error_message(AwsRequest), + {error, {Type, Message}}; +decode_error(AwsRequest) -> + erlcloud_aws:request_to_return(AwsRequest). + +extract_error_type(#aws_request{response_body = ResponseBody} = AwsRequest) -> + ResponseObject = jsx:decode(ResponseBody, [{return_maps, false}]), + case proplists:get_value(<<"ErrorCode">>, ResponseObject) of + undefined -> + Headers = AwsRequest#aws_request.response_headers, + ErrorType = proplists:get_value("x-amzn-errortype", Headers), + iolist_to_binary(ErrorType); + Code -> + Code + end. + +extract_error_message(#aws_request{response_body = ResponseBody}) -> + Object = jsx:decode(ResponseBody, [{return_maps, false}]), + proplists:get_value(<<"Message">>, Object, <<>>). + +-spec flatten_pages([[any()]]) -> [any()]. +flatten_pages(Pages) -> + lists:append(lists:reverse(Pages)). diff --git a/test/erlcloud_efs_tests.erl b/test/erlcloud_efs_tests.erl new file mode 100644 index 000000000..aae035898 --- /dev/null +++ b/test/erlcloud_efs_tests.erl @@ -0,0 +1,379 @@ +-module(erlcloud_efs_tests). + +-include_lib("eunit/include/eunit.hrl"). +-include("erlcloud_aws.hrl"). + +-define(TEST_AWS_CONFIG, #aws_config{ + access_key_id = "TEST_ACCESS_KEY_ID", + secret_access_key = "TEST_ACCESS_KEY", + security_token = "TEST_SECURITY_TOKEN" +}). + +api_test_() -> + { + foreach, + fun() -> meck:new(erlcloud_httpc) end, + fun(_) -> meck:unload() end, + [ + fun describe_file_systems_tests/1 + ] + }. + +describe_file_systems_tests(_) -> + [ + { + "DescribeFileSystems", + fun() -> + EfsUrl = "https://elasticfilesystem.us-east-1.amazonaws.com:443/2015-02-01/file-systems", + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, + request, + fun(Url, Method, _Hdrs, Body, _Timeout, AwsCfg) when + Url =:= EfsUrl, Method =:= get, Body =:= <<>>, AwsCfg =:= AwsConfig + -> + ResponseContent = << + "{" + "\"FileSystems\":[" + "{" + "\"AvailabilityZoneId\":null," + "\"AvailabilityZoneName\":null," + "\"CreationTime\":1.7343072E9," + "\"CreationToken\":\"11111111-1111-1111-1111-111111111111\"," + "\"Encrypted\":true," + "\"FileSystemArn\":\"arn:aws:elasticfilesystem:us-east-1:111111111111:file-system/fs-00000000000000001\"," + "\"FileSystemId\":\"fs-00000000000000001\"," + "\"FileSystemProtection\":{" + "\"ReplicationOverwriteProtection\":\"ENABLED\"" + "}," + "\"KmsKeyId\":\"arn:aws:kms:us-east-1:111111111111:key/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa\"," + "\"LifeCycleState\":\"available\"," + "\"Name\":\"efs-001\"," + "\"NumberOfMountTargets\":0," + "\"OwnerId\":\"111111111111\"," + "\"PerformanceMode\":\"generalPurpose\"," + "\"ProvisionedThroughputInMibps\":null," + "\"SizeInBytes\":{" + "\"Timestamp\":null," + "\"Value\":6144," + "\"ValueInArchive\":0," + "\"ValueInIA\":0," + "\"ValueInStandard\":6144" + "}," + "\"Tags\":[" + "{" + "\"Key\":\"Name\"," + "\"Value\":\"efs-001\"" + "}" + "]," + "\"ThroughputMode\":\"elastic\"" + "}," + "{" + "\"AvailabilityZoneId\":null," + "\"AvailabilityZoneName\":null," + "\"CreationTime\":1.7343072E9," + "\"CreationToken\":\"22222222-2222-2222-2222-222222222222\"," + "\"Encrypted\":true," + "\"FileSystemArn\":\"arn:aws:elasticfilesystem:us-east-1:111111111111:file-system/fs-00000000000000002\"," + "\"FileSystemId\":\"fs-00000000000000002\"," + "\"FileSystemProtection\":{" + "\"ReplicationOverwriteProtection\":\"ENABLED\"" + "}," + "\"KmsKeyId\":\"arn:aws:kms:us-east-1:111111111111:key/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa\"," + "\"LifeCycleState\":\"available\"," + "\"Name\":\"efs-002\"," + "\"NumberOfMountTargets\":0," + "\"OwnerId\":\"111111111111\"," + "\"PerformanceMode\":\"generalPurpose\"," + "\"ProvisionedThroughputInMibps\":null," + "\"SizeInBytes\":{" + "\"Timestamp\":null," + "\"Value\":6144," + "\"ValueInArchive\":0," + "\"ValueInIA\":0," + "\"ValueInStandard\":6144" + "}," + "\"Tags\":[" + "{" + "\"Key\":\"Name\"," + "\"Value\":\"efs-002\"" + "}" + "]," + "\"ThroughputMode\":\"elastic\"" + "}" + "]," + "\"Marker\":null," + "\"NextMarker\":null" + "}" + >>, + {ok, {{200, "OK"}, [], ResponseContent}} + end + ), + Params = [], + Result = erlcloud_efs:describe_file_systems(AwsConfig, Params), + FileSystems = [ + [ + {<<"AvailabilityZoneId">>, null}, + {<<"AvailabilityZoneName">>, null}, + {<<"CreationTime">>, 1.7343072e9}, + {<<"CreationToken">>, <<"11111111-1111-1111-1111-111111111111">>}, + {<<"Encrypted">>, true}, + {<<"FileSystemArn">>, + <<"arn:aws:elasticfilesystem:us-east-1:111111111111:file-system/fs-00000000000000001">>}, + {<<"FileSystemId">>, <<"fs-00000000000000001">>}, + {<<"FileSystemProtection">>, [ + {<<"ReplicationOverwriteProtection">>, <<"ENABLED">>} + ]}, + {<<"KmsKeyId">>, + <<"arn:aws:kms:us-east-1:111111111111:key/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa">>}, + {<<"LifeCycleState">>, <<"available">>}, + {<<"Name">>, <<"efs-001">>}, + {<<"NumberOfMountTargets">>, 0}, + {<<"OwnerId">>, <<"111111111111">>}, + {<<"PerformanceMode">>, <<"generalPurpose">>}, + {<<"ProvisionedThroughputInMibps">>, null}, + {<<"SizeInBytes">>, [ + {<<"Timestamp">>, null}, + {<<"Value">>, 6144}, + {<<"ValueInArchive">>, 0}, + {<<"ValueInIA">>, 0}, + {<<"ValueInStandard">>, 6144} + ]}, + {<<"Tags">>, [[{<<"Key">>, <<"Name">>}, {<<"Value">>, <<"efs-001">>}]]}, + {<<"ThroughputMode">>, <<"elastic">>} + ], + [ + {<<"AvailabilityZoneId">>, null}, + {<<"AvailabilityZoneName">>, null}, + {<<"CreationTime">>, 1.7343072e9}, + {<<"CreationToken">>, <<"22222222-2222-2222-2222-222222222222">>}, + {<<"Encrypted">>, true}, + {<<"FileSystemArn">>, + <<"arn:aws:elasticfilesystem:us-east-1:111111111111:file-system/fs-00000000000000002">>}, + {<<"FileSystemId">>, <<"fs-00000000000000002">>}, + {<<"FileSystemProtection">>, [ + {<<"ReplicationOverwriteProtection">>, <<"ENABLED">>} + ]}, + {<<"KmsKeyId">>, + <<"arn:aws:kms:us-east-1:111111111111:key/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa">>}, + {<<"LifeCycleState">>, <<"available">>}, + {<<"Name">>, <<"efs-002">>}, + {<<"NumberOfMountTargets">>, 0}, + {<<"OwnerId">>, <<"111111111111">>}, + {<<"PerformanceMode">>, <<"generalPurpose">>}, + {<<"ProvisionedThroughputInMibps">>, null}, + {<<"SizeInBytes">>, [ + {<<"Timestamp">>, null}, + {<<"Value">>, 6144}, + {<<"ValueInArchive">>, 0}, + {<<"ValueInIA">>, 0}, + {<<"ValueInStandard">>, 6144} + ]}, + {<<"Tags">>, [[{<<"Key">>, <<"Name">>}, {<<"Value">>, <<"efs-002">>}]]}, + {<<"ThroughputMode">>, <<"elastic">>} + ] + ], + ?assertEqual({ok, FileSystems, undefined}, Result) + end + }, + { + "DescribeFileSystems [all]", + fun() -> + EfsUrl1 = "https://elasticfilesystem.us-east-1.amazonaws.com:443/2015-02-01/file-systems?MaxItems=1", + EfsUrl2 = "https://elasticfilesystem.us-east-1.amazonaws.com:443/2015-02-01/file-systems?Marker=TEST_NEXT_MARKER&MaxItems=1", + AwsConfig = ?TEST_AWS_CONFIG, + meck:expect( + erlcloud_httpc, + request, + fun + (Url, Method, _Hdrs, Body, _Timeout, AwsCfg) when + Url =:= EfsUrl1, Method =:= get, Body =:= <<>>, AwsCfg =:= AwsConfig + -> + ResponseContent = << + "{" + "\"FileSystems\":[" + "{" + "\"AvailabilityZoneId\":null," + "\"AvailabilityZoneName\":null," + "\"CreationTime\":1.7343072E9," + "\"CreationToken\":\"11111111-1111-1111-1111-111111111111\"," + "\"Encrypted\":true," + "\"FileSystemArn\":\"arn:aws:elasticfilesystem:us-east-1:111111111111:file-system/fs-00000000000000001\"," + "\"FileSystemId\":\"fs-00000000000000001\"," + "\"FileSystemProtection\":{" + "\"ReplicationOverwriteProtection\":\"ENABLED\"" + "}," + "\"KmsKeyId\":\"arn:aws:kms:us-east-1:111111111111:key/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa\"," + "\"LifeCycleState\":\"available\"," + "\"Name\":\"efs-001\"," + "\"NumberOfMountTargets\":0," + "\"OwnerId\":\"111111111111\"," + "\"PerformanceMode\":\"generalPurpose\"," + "\"ProvisionedThroughputInMibps\":null," + "\"SizeInBytes\":{" + "\"Timestamp\":null," + "\"Value\":6144," + "\"ValueInArchive\":0," + "\"ValueInIA\":0," + "\"ValueInStandard\":6144" + "}," + "\"Tags\":[" + "{" + "\"Key\":\"Name\"," + "\"Value\":\"efs-001\"" + "}" + "]," + "\"ThroughputMode\":\"elastic\"" + "}" + "]," + "\"Marker\":null," + "\"NextMarker\":\"TEST_NEXT_MARKER\"" + "}" + >>, + {ok, {{200, "OK"}, [], ResponseContent}}; + (Url, Method, _Hdrs, Body, _Timeout, AwsCfg) when + Url =:= EfsUrl2, Method =:= get, Body =:= <<>>, AwsCfg =:= AwsConfig + -> + ResponseContent = << + "{" + "\"FileSystems\":[" + "{" + "\"AvailabilityZoneId\":null," + "\"AvailabilityZoneName\":null," + "\"CreationTime\":1.7343072E9," + "\"CreationToken\":\"22222222-2222-2222-2222-222222222222\"," + "\"Encrypted\":true," + "\"FileSystemArn\":\"arn:aws:elasticfilesystem:us-east-1:111111111111:file-system/fs-00000000000000002\"," + "\"FileSystemId\":\"fs-00000000000000002\"," + "\"FileSystemProtection\":{" + "\"ReplicationOverwriteProtection\":\"ENABLED\"" + "}," + "\"KmsKeyId\":\"arn:aws:kms:us-east-1:111111111111:key/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa\"," + "\"LifeCycleState\":\"available\"," + "\"Name\":\"efs-002\"," + "\"NumberOfMountTargets\":0," + "\"OwnerId\":\"111111111111\"," + "\"PerformanceMode\":\"generalPurpose\"," + "\"ProvisionedThroughputInMibps\":null," + "\"SizeInBytes\":{" + "\"Timestamp\":null," + "\"Value\":6144," + "\"ValueInArchive\":0," + "\"ValueInIA\":0," + "\"ValueInStandard\":6144" + "}," + "\"Tags\":[" + "{" + "\"Key\":\"Name\"," + "\"Value\":\"efs-002\"" + "}" + "]," + "\"ThroughputMode\":\"elastic\"" + "}" + "]," + "\"Marker\":\"TEST_NEXT_MARKER\"," + "\"NextMarker\":null" + "}" + >>, + {ok, {{200, "OK"}, [], ResponseContent}} + end + ), + Params = [{<<"MaxItems">>, 1}], + Result = erlcloud_efs:describe_file_systems_all(AwsConfig, Params), + FileSystems = [ + [ + {<<"AvailabilityZoneId">>, null}, + {<<"AvailabilityZoneName">>, null}, + {<<"CreationTime">>, 1.7343072e9}, + {<<"CreationToken">>, <<"11111111-1111-1111-1111-111111111111">>}, + {<<"Encrypted">>, true}, + {<<"FileSystemArn">>, + <<"arn:aws:elasticfilesystem:us-east-1:111111111111:file-system/fs-00000000000000001">>}, + {<<"FileSystemId">>, <<"fs-00000000000000001">>}, + {<<"FileSystemProtection">>, [ + {<<"ReplicationOverwriteProtection">>, <<"ENABLED">>} + ]}, + {<<"KmsKeyId">>, + <<"arn:aws:kms:us-east-1:111111111111:key/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa">>}, + {<<"LifeCycleState">>, <<"available">>}, + {<<"Name">>, <<"efs-001">>}, + {<<"NumberOfMountTargets">>, 0}, + {<<"OwnerId">>, <<"111111111111">>}, + {<<"PerformanceMode">>, <<"generalPurpose">>}, + {<<"ProvisionedThroughputInMibps">>, null}, + {<<"SizeInBytes">>, [ + {<<"Timestamp">>, null}, + {<<"Value">>, 6144}, + {<<"ValueInArchive">>, 0}, + {<<"ValueInIA">>, 0}, + {<<"ValueInStandard">>, 6144} + ]}, + {<<"Tags">>, [[{<<"Key">>, <<"Name">>}, {<<"Value">>, <<"efs-001">>}]]}, + {<<"ThroughputMode">>, <<"elastic">>} + ], + [ + {<<"AvailabilityZoneId">>, null}, + {<<"AvailabilityZoneName">>, null}, + {<<"CreationTime">>, 1.7343072e9}, + {<<"CreationToken">>, <<"22222222-2222-2222-2222-222222222222">>}, + {<<"Encrypted">>, true}, + {<<"FileSystemArn">>, + <<"arn:aws:elasticfilesystem:us-east-1:111111111111:file-system/fs-00000000000000002">>}, + {<<"FileSystemId">>, <<"fs-00000000000000002">>}, + {<<"FileSystemProtection">>, [ + {<<"ReplicationOverwriteProtection">>, <<"ENABLED">>} + ]}, + {<<"KmsKeyId">>, + <<"arn:aws:kms:us-east-1:111111111111:key/aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa">>}, + {<<"LifeCycleState">>, <<"available">>}, + {<<"Name">>, <<"efs-002">>}, + {<<"NumberOfMountTargets">>, 0}, + {<<"OwnerId">>, <<"111111111111">>}, + {<<"PerformanceMode">>, <<"generalPurpose">>}, + {<<"ProvisionedThroughputInMibps">>, null}, + {<<"SizeInBytes">>, [ + {<<"Timestamp">>, null}, + {<<"Value">>, 6144}, + {<<"ValueInArchive">>, 0}, + {<<"ValueInIA">>, 0}, + {<<"ValueInStandard">>, 6144} + ]}, + {<<"Tags">>, [[{<<"Key">>, <<"Name">>}, {<<"Value">>, <<"efs-002">>}]]}, + {<<"ThroughputMode">>, <<"elastic">>} + ] + ], + ?assertEqual({ok, FileSystems}, Result) + end + }, + { + "DescribeFileSystems [filesystem not found]", + fun() -> + AwsConfig = ?TEST_AWS_CONFIG, + EfsUrl = "https://elasticfilesystem.us-east-1.amazonaws.com:443/2015-02-01/file-systems?FileSystemId=fs-1234bad", + meck:expect( + erlcloud_httpc, + request, + fun(Url, Method, _Hdrs, Body, _Timeout, AwsCfg) when + Url =:= EfsUrl, Method =:= get, Body =:= <<>>, AwsCfg =:= AwsConfig + -> + ResponseContent = << + "{" + "\"ErrorCode\":\"FileSystemNotFound\"," + "\"Message\":\"File system 'fs-1234bad' does not exist.\"" + "}" + >>, + ResponseHeaders = [ + {"x-amzn-errortype", "FileSystemNotFound:"}, + {"content-type", "application/json"} + ], + {ok, {{404, "Not Found"}, ResponseHeaders, ResponseContent}} + end + ), + Params = [{<<"FileSystemId">>, <<"fs-1234bad">>}], + Result = erlcloud_efs:describe_file_systems(AwsConfig, Params), + ErrorType = <<"FileSystemNotFound">>, + ErrorMessage = <<"File system 'fs-1234bad' does not exist.">>, + ?assertEqual({error, {ErrorType, ErrorMessage}}, Result) + end + } + ]. From 417ee97e401f21ac1bb539b5f1634bcfd768bc8c Mon Sep 17 00:00:00 2001 From: cody-friedrichsen Date: Mon, 21 Apr 2025 12:28:53 -0500 Subject: [PATCH 306/310] Implement S3 Head Bucket operation --- src/erlcloud_s3.erl | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index 674bd99d6..ce2f23968 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -23,6 +23,7 @@ delete_object/2, delete_object/3, delete_object_version/3, delete_object_version/4, head_object/2, head_object/3, head_object/4, + head_bucket/1, head_bucket/2, head_bucket/3, get_object/2, get_object/3, get_object/4, get_object_acl/2, get_object_acl/3, get_object_acl/4, get_object_torrent/2, get_object_torrent/3, @@ -815,6 +816,24 @@ head_object(BucketName, Key, Options) -> head_object(BucketName, Key, Options, Config) -> get_or_head(head, BucketName, Key, Options, Config). +-spec head_bucket(string()) -> proplist(). + +head_bucket(BucketName) -> + head_bucket(BucketName, []). + +-spec head_bucket(string(), proplist() | aws_config()) -> proplist(). + +head_bucket(BucketName, Config) + when is_record(Config, aws_config) -> + head_bucket(BucketName, [], Config); +head_bucket(BucketName, Options) -> + head_bucket(BucketName, Options, default_config()). + +-spec head_bucket(string(), proplist(), aws_config()) -> proplist(). + +head_bucket(BucketName, Options, Config) -> + get_or_head(head, BucketName, Options, Config). + -spec get_object(string(), string()) -> proplist(). get_object(BucketName, Key) -> @@ -834,6 +853,26 @@ get_object(BucketName, Key, Options) -> get_object(BucketName, Key, Options, Config) -> get_or_head(get, BucketName, Key, Options, Config). +get_or_head(Method, BucketName, Options, Config) -> + RequestHeaders = [{"Range", proplists:get_value(range, Options)}, + {"If-Modified-Since", proplists:get_value(if_modified_since, Options)}, + {"If-Unmodified-Since", proplists:get_value(if_unmodified_since, Options)}, + {"If-Match", proplists:get_value(if_match, Options)}, + {"If-None-Match", proplists:get_value(if_none_match, Options)}, + {"x-amz-server-side-encryption-customer-algorithm", proplists:get_value(server_side_encryption_customer_algorithm, Options)}, + {"x-amz-server-side-encryption-customer-key", proplists:get_value(server_side_encryption_customer_key, Options)}, + {"x-amz-server-side-encryption-customer-key-md5", proplists:get_value(server_side_encryption_customer_key_md5, Options)}], + Subresource = case proplists:get_value(version_id, Options) of + undefined -> ""; + Version -> ["versionId=", Version] + end, + {Headers, Body} = s3_request(Config, Method, BucketName, [$/], Subresource, [], <<>>, RequestHeaders), + [{content_length, proplists:get_value("content-length", Headers)}, + {content_type, proplists:get_value("content-type", Headers)}, + {access_point_alias, proplists:get_value("x-amz-access-point-alias", Headers)}, + {bucket_region, proplists:get_value("x-amz-bucket-region", Headers)}| + extract_metadata(Headers)]. + get_or_head(Method, BucketName, Key, Options, Config) -> RequestHeaders = [{"Range", proplists:get_value(range, Options)}, {"If-Modified-Since", proplists:get_value(if_modified_since, Options)}, From 46c6e0296f96ce3927f74b65ed6e21f9800a24bb Mon Sep 17 00:00:00 2001 From: cody-friedrichsen Date: Mon, 21 Apr 2025 12:36:40 -0500 Subject: [PATCH 307/310] Address warning --- src/erlcloud_s3.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/erlcloud_s3.erl b/src/erlcloud_s3.erl index ce2f23968..41895e976 100644 --- a/src/erlcloud_s3.erl +++ b/src/erlcloud_s3.erl @@ -866,7 +866,7 @@ get_or_head(Method, BucketName, Options, Config) -> undefined -> ""; Version -> ["versionId=", Version] end, - {Headers, Body} = s3_request(Config, Method, BucketName, [$/], Subresource, [], <<>>, RequestHeaders), + {Headers, _Body} = s3_request(Config, Method, BucketName, [$/], Subresource, [], <<>>, RequestHeaders), [{content_length, proplists:get_value("content-length", Headers)}, {content_type, proplists:get_value("content-type", Headers)}, {access_point_alias, proplists:get_value("x-amz-access-point-alias", Headers)}, From 5d58397e5335fc9aa004e11e854805922d47d357 Mon Sep 17 00:00:00 2001 From: cody-friedrichsen Date: Mon, 21 Apr 2025 15:59:44 -0500 Subject: [PATCH 308/310] Add unit test for head_bucket --- test/erlcloud_s3_tests.erl | 133 ++++++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) diff --git a/test/erlcloud_s3_tests.erl b/test/erlcloud_s3_tests.erl index 5ef1a9490..c2380ca69 100755 --- a/test/erlcloud_s3_tests.erl +++ b/test/erlcloud_s3_tests.erl @@ -42,7 +42,12 @@ operation_test_() -> fun get_bucket_encryption_not_found_test/1, fun delete_bucket_encryption_test/1, fun hackney_proxy_put_validation_test/1, - fun get_bucket_and_key/1 + fun get_bucket_and_key/1, + fun head_bucket_ok/1, + fun head_bucket_redirect/1, + fun head_bucket_bad_request/1, + fun head_bucket_forbidden/1, + fun head_bucket_not_found/1 ]}. start() -> @@ -828,3 +833,129 @@ get_bucket_and_key(_) -> ErlcloudS3ExportExample = "https://s3.amazonaws.com/some_bucket/path_to_file", Result = erlcloud_s3:get_bucket_and_key(ErlcloudS3ExportExample), ?_assertEqual({"some_bucket","path_to_file"}, Result). + +head_bucket_ok(_) -> + Response = {ok, {{200, "OK"}, + [ + {"server","AmazonS3"}, + {"transfer-encoding","chunked"}, + {"content-type","application/xml"}, + {"x-amz-access-point-alias","false"}, + {"x-amz-bucket-region","us-west-2"}, + {"date","Mon, 21 Apr 2025 19:45:15 GMT"}, + {"x-amz-id-2", + "YIgyI9Lb9I/dMpDrRASSD8w5YsNAyhRlF+PDF0jlf9Hq6eVLvSkuj+ftZI2RmU5eXnOKW1Wqh20="}, + {"x-amz-request-id","FAECC30C2CD53BCA"} + ], + <<>>} + }, + meck:expect(erlcloud_httpc, request, httpc_expect(head, Response)), + Result = erlcloud_s3:head_bucket( + "bucket.name", + config()), + ?_assertEqual( + [ + {content_length,undefined}, + {content_type,"application/xml"}, + {access_point_alias,"false"}, + {bucket_region,"us-west-2"} + ], Result + ). + +head_bucket_redirect(_) -> + Response1 = {ok, {{307, "Temporary Redirect"}, + [ + {"server","AmazonS3"}, + {"transfer-encoding","chunked"}, + {"content-type","application/xml"}, + {"location", "https://bucket.name.s3-us-west-2.amazonaws.com/"}, + {"x-amz-bucket-region","us-west-2"}, + {"date","Mon, 21 Apr 2025 19:45:15 GMT"}, + {"x-amz-id-2", + "YIgyI9Lb9I/dMpDrRASSD8w5YsNAyhRlF+PDF0jlf9Hq6eVLvSkuj+ftZI2RmU5eXnOKW1Wqh20="}, + {"x-amz-request-id","FAECC30C2CD53BCA"} + ], + <<>>} + }, + Response2 = {ok, {{200, "OK"}, + [ + {"server","AmazonS3"}, + {"transfer-encoding","chunked"}, + {"content-type","application/xml"}, + {"x-amz-access-point-alias","false"}, + {"x-amz-bucket-region","us-west-2"}, + {"date","Mon, 21 Apr 2025 19:45:15 GMT"}, + {"x-amz-id-2", + "YIgyI9Lb9I/dMpDrRASSD8w5YsNAyhRlF+PDF0jlf9Hq6eVLvSkuj+ftZI2RmU5eXnOKW1Wqh20="}, + {"x-amz-request-id","FAECC30C2CD53BCA"} + ], + <<>>} + }, + meck:sequence(erlcloud_httpc, request, 6, [Response1, Response2]), + Result = erlcloud_s3:head_bucket( + "bucket.name", + config()), + ?_assertEqual( + [ + {content_length,undefined}, + {content_type,"application/xml"}, + {access_point_alias,"false"}, + {bucket_region,"us-west-2"} + ], Result + ). + +head_bucket_bad_request(_) -> + Response = {ok, {{400, "Bad Request"}, + [ + {"server","AmazonS3"}, + {"transfer-encoding","chunked"}, + {"content-type","application/xml"}, + {"date","Mon, 21 Apr 2025 19:45:15 GMT"}, + {"connection", "close"}, + {"x-amz-id-2", + "YIgyI9Lb9I/dMpDrRASSD8w5YsNAyhRlF+PDF0jlf9Hq6eVLvSkuj+ftZI2RmU5eXnOKW1Wqh20="}, + {"x-amz-request-id","FAECC30C2CD53BCA"} + ], <<>>}}, + meck:expect(erlcloud_httpc, request, httpc_expect(head, Response)), + ?_assertException( + error, + {aws_error, {http_error, 400, "Bad Request", <<>>}}, + erlcloud_s3:head_bucket("bucket.name", config()) + ). + +head_bucket_forbidden(_) -> + Response = {ok, {{403, "Forbidden"}, + [ + {"server","AmazonS3"}, + {"transfer-encoding","chunked"}, + {"content-type","application/xml"}, + {"date","Mon, 21 Apr 2025 19:45:15 GMT"}, + {"x-amz-bucket-region","us-west-2"}, + {"x-amz-id-2", + "YIgyI9Lb9I/dMpDrRASSD8w5YsNAyhRlF+PDF0jlf9Hq6eVLvSkuj+ftZI2RmU5eXnOKW1Wqh20="}, + {"x-amz-request-id","FAECC30C2CD54BCA"} + ], <<>>}}, + meck:expect(erlcloud_httpc, request, httpc_expect(head, Response)), + ?_assertException( + error, + {aws_error, {http_error, 403, "Forbidden", <<>>}}, + erlcloud_s3:head_bucket("bucket.name", config()) + ). + +head_bucket_not_found(_) -> + Response = {ok, {{404, "Not Found"}, + [ + {"server","AmazonS3"}, + {"transfer-encoding","chunked"}, + {"content-type","application/xml"}, + {"date","Mon, 21 Apr 2025 19:45:15 GMT"}, + {"x-amz-id-2", + "YIgyI9Lb9I/dMpDrRASSD8w5YsNAyhRlF+PDF0jlf9Hq6eVLvSkuj+ftZI2RmU5eXnOKW1Wqh20="}, + {"x-amz-request-id","FAECC30C2CD54CCA"} + ], <<>>}}, + meck:expect(erlcloud_httpc, request, httpc_expect(head, Response)), + ?_assertException( + error, + {aws_error, {http_error, 404, "Not Found", <<>>}}, + erlcloud_s3:head_bucket("bucket.name", config()) + ). From 4a5fcae28571f46e032eb9106dc1ecbbb8ba7958 Mon Sep 17 00:00:00 2001 From: Carl Isom Date: Wed, 22 Oct 2025 14:01:54 -0500 Subject: [PATCH 309/310] Update erlcloud_sm to add missing functionality --- src/erlcloud_sm.erl | 739 +++++++++++++++++++++++++++- test/erlcloud_sm_tests.erl | 978 ++++++++++++++++++++++++++++++++++++- 2 files changed, 1688 insertions(+), 29 deletions(-) diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl index c1189dda5..2ec3ffcb0 100644 --- a/src/erlcloud_sm.erl +++ b/src/erlcloud_sm.erl @@ -9,16 +9,33 @@ %%% API -export([ + batch_get_secret_value/2, batch_get_secret_value/3, + cancel_rotate_secret/1, cancel_rotate_secret/2, + create_secret/3, create_secret/4, create_secret/5, create_secret_binary/3, create_secret_binary/4, create_secret_binary/5, create_secret_string/3, create_secret_string/4, create_secret_string/5, delete_resource_policy/1, delete_resource_policy/2, delete_secret/1, delete_secret/2, delete_secret/3, describe_secret/1, describe_secret/2, + get_random_password/0, get_random_password/1, get_random_password/2, get_resource_policy/1, get_resource_policy/2, get_secret_value/2, get_secret_value/3, + list_secrets/0, list_secrets/1, list_secrets/2, + list_secret_version_ids/1, list_secret_version_ids/2, list_secret_version_ids/3, put_resource_policy/2, put_resource_policy/3, put_resource_policy/4, put_secret_binary/3, put_secret_binary/4, put_secret_binary/5, - put_secret_string/3, put_secret_string/4, put_secret_string/5 + put_secret_string/3, put_secret_string/4, put_secret_string/5, + put_secret_value/3, put_secret_value/4, put_secret_value/5, + remove_regions_from_replication/2, remove_regions_from_replication/3, + replicate_secret_to_regions/2, replicate_secret_to_regions/3, replicate_secret_to_regions/4, + restore_secret/1, restore_secret/2, + rotate_secret/2, rotate_secret/3, rotate_secret/4, + stop_replication_to_replica/1, stop_replication_to_replica/2, + tag_resource/2, tag_resource/3, + untag_resource/2, untag_resource/3, + update_secret/2, update_secret/3, update_secret/4, + update_secret_version_stage/2, update_secret_version_stage/3, update_secret_version_stage/4, + validate_resource_policy/1, validate_resource_policy/2, validate_resource_policy/3 ]). %%%------------------------------------------------------------------------------ @@ -35,6 +52,16 @@ -type replica_region() :: [proplist()]. -type replica_regions() :: [replica_region()]. +%% batch get secret value types +-type secret_value_filter() :: {key, binary(), values, [binary()]}. +-type secret_value_filters() :: [secret_value_filter()]. + +-type batch_get_secret_value_param() :: {filters, secret_value_filters()} | {secret_id_list, [binary()]}. + +-type batch_get_secret_value_option() :: {max_results, pos_integer()} + | {next_token, binary()}. +-type batch_get_secret_value_options() :: [batch_get_secret_value_option()]. + -type create_secret_option() :: {add_replica_regions, replica_regions()} | {client_request_token, binary()} | {description, binary()} @@ -49,15 +76,68 @@ | {recovery_window_in_days, pos_integer()}. %% If none of these two options are specified then SM defaults to 30 day recovery window -type delete_secret_options() :: [delete_secret_option()]. +%% get random password options types +-type get_random_password_option() :: {exclude_characters, binary()} + | {exclude_lowercase, boolean()} + | {exclude_numbers, boolean()} + | {exclude_punctuation, boolean()} + | {exclude_uppercase, boolean()} + | {include_space, boolean()} + | {password_length, pos_integer()} + | {require_each_included_type, boolean()}. +-type get_random_password_options() :: [get_random_password_option()]. + +%% list secrets options types +-type list_secrets_option() :: {filters, secret_value_filters()} + | {max_results, pos_integer()} + | {include_planned_deletion, boolean()} + | {next_token, binary()} + | {sort_order, binary()}. +-type list_secrets_options() :: [list_secrets_option()]. + +% list secret version ids options types +-type list_secret_version_ids_option() :: {include_deprecated, boolean()} + | {max_results, pos_integer()} + | {next_token, binary()}. +-type list_secret_version_ids_options() :: [list_secret_version_ids_option()]. + -type put_resource_policy_option() :: {block_public_policy, boolean()}. -type put_resource_policy_options() :: [put_resource_policy_option()]. +-type secret_value() :: {secret_binary, binary()} | {secret_string, binary()}. -type put_secret_value_option() :: {client_request_token, binary()} | {secret_binary, binary()} | {secret_string, binary()} | {version_stages, [binary()]}. -type put_secret_value_options() :: [put_secret_value_option()]. +-type rotation_rules() :: [rotate_rule()]. +-type rotate_rule() :: {automatically_after_days, pos_integer()} + | {duration, binary()} + | {schedule_expression, binary()}. + +-type rotate_secret_options() :: [rotate_secret_option()]. +-type rotate_secret_option() :: {rotate_immediately, boolean()} + | {rotation_lambda_arn, binary()} + | {rotation_rules, proplist()}. + +-type replicate_secret_to_regions_option() :: {force_overwrite_replica_secret, boolean()}. +-type replicate_secret_to_regions_options() :: [replicate_secret_to_regions_option()]. + +-type update_secret_options() :: [update_secret_option()]. +-type update_secret_option() :: {description, binary()} + | {kms_key_id, binary()} + | {secret_binary, binary()} + | {secret_string, binary()}. + +-type update_secret_version_stage_option() :: {move_to_version_id, binary()} + | {remove_from_version_id, binary()}. +-type update_secret_version_stage_options() :: [update_secret_version_stage_option()]. + +-type validate_resource_policy_option() :: {secret_id, binary()}. +-type validate_resource_policy_options() :: [validate_resource_policy_option()]. + + %%%------------------------------------------------------------------------------ %%% Library initialization. %%%------------------------------------------------------------------------------ @@ -89,16 +169,120 @@ new(AccessKeyID, SecretAccessKey, Host, Port) -> }. +%%------------------------------------------------------------------------------ +%% BatchGetSecretValue +%%------------------------------------------------------------------------------ +%% @doc +%% Retrieves the contents of the encrypted fields SecretString or SecretBinary for up to 20 secrets. +%% To retrieve a single secret, call GetSecretValue. You must include Filters or SecretIdList, but not both. +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_BatchGetSecretValue.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec batch_get_secret_value(FiltersOrSecretIdList :: batch_get_secret_value_param(), + Opts :: batch_get_secret_value_options()) -> sm_response(). +batch_get_secret_value(FiltersOrSecretIdList, Opts) -> + batch_get_secret_value(FiltersOrSecretIdList, Opts, erlcloud_aws:default_config()). + +-spec batch_get_secret_value(FiltersOrSecretIdList :: batch_get_secret_value_param(), + Opts :: batch_get_secret_value_options(), + Config :: aws_config()) -> sm_response(). +batch_get_secret_value(FiltersOrSecretIdList, Opts, Config) -> + BatchGetSecretParam = case FiltersOrSecretIdList of + {filters, Filter} -> {<<"Filters">>, Filter}; + {secret_id_list, IdList} -> {<<"SecretIdList">>, IdList} + end, + Json = lists:map( + fun + ({max_results, Val}) -> {<<"MaxResults">>, Val}; + ({next_token, Val}) -> {<<"NextToken">>, Val}; + (Other) -> Other + end, + [BatchGetSecretParam | Opts]), + sm_request(Config, "secretsmanager.BatchGetSecretValue", Json). + +%%------------------------------------------------------------------------------ +%% CancelRotateSecret +%%------------------------------------------------------------------------------ +%% @doc +%% Turns off automatic rotation, and if a rotation is currently in progress, cancels the rotation. +%% +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_CancelRotateSecret.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec get_random_password() -> sm_response(). +get_random_password() -> + get_random_password([]). + +-spec cancel_rotate_secret(SecretId :: binary()) -> sm_response(). +cancel_rotate_secret(SecretId) -> + cancel_rotate_secret(SecretId, erlcloud_aws:default_config()). + +-spec cancel_rotate_secret(SecretId :: binary(), Config :: aws_config()) -> sm_response(). +cancel_rotate_secret(SecretId, Config) -> + Json = [{<<"SecretId">>, SecretId}], + sm_request(Config, "secretsmanager.CancelRotateSecret", Json). + +%%------------------------------------------------------------------------------ +%% CreateSecret - CreateSecret +%%------------------------------------------------------------------------------ +%% @doc +%% Creates a new secret.A secret can be a password, a set of credentials such as +%% a user name and password, an OAuth token, or other secret information that you +%% store in an encrypted form in Secrets Manager. The secret also includes the connection +%% information to access a database or other service, which Secrets Manager doesn't encrypt. +%% A secret in Secrets Manager consists of both the protected secret data and the important +%% information needed to manage the secret. +%% +%% ClientRequestToken is used by AWS for secret versioning purposes. +%% It is recommended to be a UUID type value, and is required to be between +%% 32 and 64 characters. +%% +%% Note: Use create_secret, as create_secret_binary and create_secret_string functions are kept for backward compatibility. +%% +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_CreateSecret.html] +%% +%% @end +%%------------------------------------------------------------------------------ + +-spec create_secret(Name :: binary(), ClientRequestToken :: binary(), + Secret :: secret_value()) -> sm_response(). +create_secret(Name, ClientRequestToken, Secret) -> + create_secret(Name, ClientRequestToken, Secret, []). + +-spec create_secret(Name :: binary(), ClientRequestToken :: binary(), + Secret :: secret_value(), + Opts :: create_secret_options()) -> sm_response(). +create_secret(Name, ClientRequestToken, Secret, Opts) -> + create_secret(Name, ClientRequestToken, Secret, Opts, erlcloud_aws:default_config()). + +-spec create_secret(Name :: binary(), ClientRequestToken :: binary(), + Secret :: secret_value(), Opts :: create_secret_options(), + Config :: aws_config()) -> sm_response(). +create_secret(Name, ClientRequestToken, Secret, Opts, Config) -> + SecretValue = case Secret of + {secret_binary, Val} -> {secret_binary, base64:encode(Val)}; + {secret_string, Val} -> {secret_string, Val} + end, + create_secret_call(Name, ClientRequestToken, [SecretValue | Opts], Config). + %%------------------------------------------------------------------------------ %% CreateSecret - SecretBinary %%------------------------------------------------------------------------------ %% @doc +%% +%% Note: Use create_secret, as create_secret_binary and create_secret_string functions are kept for backward compatibility. +%% %% Creates a new secret binary. The function internally base64-encodes the binary %% as it is expected by the AWS SecretManager API, so raw blob is expected %% to be passed as an attribute. %% %% ClientRequestToken is used by AWS for secret versioning purposes. -%% It is recommended to be a UUID type value, and is requred to be between +%% It is recommended to be a UUID type value, and is required to be between %% 32 and 64 characters. %% %% To store a text secret use CreateSecret - SecretString version of the function @@ -132,18 +316,21 @@ create_secret_binary(Name, ClientRequestToken, SecretBinary, Opts) -> Config :: aws_config()) -> sm_response(). create_secret_binary(Name, ClientRequestToken, SecretBinary, Opts, Config) -> Secret = {secret_binary, base64:encode(SecretBinary)}, - create_secret(Name, ClientRequestToken, [Secret | Opts], Config). + create_secret_call(Name, ClientRequestToken, [Secret | Opts], Config). %%------------------------------------------------------------------------------ %% CreateSecret - SecretString %%------------------------------------------------------------------------------ %% @doc +%% +%% Note: Use create_secret, as create_secret_binary and create_secret_string functions are kept for backward compatibility. +%% %% Creates a new secret string. The API expects SecretString is a text data to %% encrypt and store in the SecretManager. It is recommended a JSON structure %% of key/value pairs is used for the secret value. %% %% ClientRequestToken is used by AWS for secret versioning purposes. -%% It is recommended to be a UUID type value, and is requred to be between +%% It is recommended to be a UUID type value, and is required to be between %% 32 and 64 characters. %% %% To store a binary (which will be base64 encoded by the library, as it is @@ -179,7 +366,7 @@ create_secret_string(Name, ClientRequestToken, SecretString, Opts) -> Config :: aws_config()) -> sm_response(). create_secret_string(Name, ClientRequestToken, SecretString, Opts, Config) -> Secret = {secret_string, SecretString}, - create_secret(Name, ClientRequestToken, [Secret | Opts], Config). + create_secret_call(Name, ClientRequestToken, [Secret | Opts], Config). %%------------------------------------------------------------------------------ %% DeleteResourcePolicy @@ -246,6 +433,41 @@ describe_secret(SecretId, Config) -> Json = [{<<"SecretId">>, SecretId}], sm_request(Config, "secretsmanager.DescribeSecret", Json). +%%------------------------------------------------------------------------------ +%% GetRandomPassword +%%------------------------------------------------------------------------------ +%% @doc +%% Generates a random password. We recommend that you specify the maximum length and include +%% every character type that the system you are generating a password for can support. +%% By default, Secrets Manager uses uppercase and lowercase letters, numbers, and the +%% following characters in passwords: !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ +%% +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_GetRandomPassword.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec get_random_password(Opts :: get_random_password_options()) -> sm_response(). +get_random_password(Opts) -> + get_random_password(Opts, erlcloud_aws:default_config()). + +-spec get_random_password(Opts :: get_random_password_options(), Config :: aws_config()) -> sm_response(). +get_random_password(Opts, Config) -> + Json = lists:map( + fun + ({exclude_characters, Val}) -> {<<"ExcludeCharacters">>, Val}; + ({exclude_lowercase, Val}) -> {<<"ExcludeLowercase">>, Val}; + ({exclude_numbers, Val}) -> {<<"ExcludeNumbers">>, Val}; + ({exclude_punctuation, Val}) -> {<<"ExcludePunctuation">>, Val}; + ({exclude_uppercase, Val}) -> {<<"ExcludeUppercase">>, Val}; + ({include_space, Val}) -> {<<"IncludeSpace">>, Val}; + ({password_length, Val}) -> {<<"PasswordLength">>, Val}; + ({require_each_included_type, Val}) -> {<<"RequireEachIncludedType">>, Val}; + (Other) -> Other + end, + Opts), + sm_request(Config, "secretsmanager.GetRandomPassword", Json). + %%------------------------------------------------------------------------------ %% GetResourcePolicy %%------------------------------------------------------------------------------ @@ -289,6 +511,76 @@ get_secret_value(SecretId, Opts, Config) -> [{<<"SecretId">>, SecretId} | Opts]), sm_request(Config, "secretsmanager.GetSecretValue", Json). +%%------------------------------------------------------------------------------ +%% ListSecrets +%%------------------------------------------------------------------------------ +%% @doc +%% Lists the secrets that are stored by Secrets Manager in the AWS account, not including +%% secrets that are marked for deletion. To see secrets marked for deletion, use the Secrets Manager console. +%% +%% All Secrets Manager operations are eventually consistent. ListSecrets might not reflect changes +%% from the last five minutes. You can get more recent information for a specific secret by calling DescribeSecret. +%% +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_ListSecrets.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec list_secrets() -> sm_response(). +list_secrets() -> + list_secrets([]). + +-spec list_secrets(Opts :: list_secrets_options()) -> sm_response(). +list_secrets(Opts) -> + list_secrets(Opts, erlcloud_aws:default_config()). + +-spec list_secrets(Opts :: list_secrets_options(), + Config :: aws_config()) -> sm_response(). +list_secrets(Opts, Config) -> + Json = lists:map( + fun + ({filters, Val}) -> {<<"Filters">>, Val}; + ({include_planned_deletion, Val}) -> {<<"IncludePlannedDeletion">>, Val}; + ({max_results, Val}) -> {<<"MaxResults">>, Val}; + ({next_token, Val}) -> {<<"NextToken">>, Val}; + ({sort_order, Val}) -> {<<"SortOrder">>, Val}; + (Other) -> Other + end, + Opts), + sm_request(Config, "secretsmanager.ListSecrets", Json). + +%%------------------------------------------------------------------------------ +%% ListSecretVersionIds +%%------------------------------------------------------------------------------ +%% @doc +%% Lists the versions of a secret. Secrets Manager uses staging labels to indicate the different versions of a secret. +%% +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_ListSecretVersionIds.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec list_secret_version_ids(SecretId :: binary()) -> sm_response(). +list_secret_version_ids(SecretId) -> + list_secret_version_ids(SecretId, []). + +-spec list_secret_version_ids(SecretId :: binary(), Opts :: list_secret_version_ids_options()) -> sm_response(). +list_secret_version_ids(SecretId, Opts) -> + list_secret_version_ids(SecretId, Opts, erlcloud_aws:default_config()). + +-spec list_secret_version_ids(SecretId :: binary(), Opts :: list_secret_version_ids_options(), + Config :: aws_config()) -> sm_response(). +list_secret_version_ids(SecretId, Opts, Config) -> + Json = lists:map( + fun + ({include_deprecated, Val}) -> {<<"IncludeDeprecated">>, Val}; + ({max_results, Val}) -> {<<"MaxResults">>, Val}; + ({next_token, Val}) -> {<<"NextToken">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId} | Opts]), + sm_request(Config, "secretsmanager.ListSecretVersionIds", Json). + %%------------------------------------------------------------------------------ %% PutResourcePolicy %%------------------------------------------------------------------------------ @@ -319,10 +611,64 @@ put_resource_policy(SecretId, ResourcePolicy, Opts, Config) -> [{<<"SecretId">>, SecretId}, {<<"ResourcePolicy">>, ResourcePolicy} | Opts]), sm_request(Config, "secretsmanager.PutResourcePolicy", Json). +%%------------------------------------------------------------------------------ +%% PutSecretValue - put_secret_value +%%------------------------------------------------------------------------------ +%% @doc +%% +%% Note: Use put_secret_value as put_secret_string and put_secret_binary functions are kept for backward compatibility. +%% +%% Creates a new version of your secret by creating a new encrypted value and attaching it to the secret. +%% Version can contain a new SecretString value or a new SecretBinary value. +%% +%% Do not call PutSecretValue at a sustained rate of more than once every 10 minutes. +%% When you update the secret value, Secrets Manager creates a new version of the secret. +%% Secrets Manager keeps 100 of the most recent versions, but it keeps all secret versions created in the last 24 hours. +%% If you call PutSecretValue more than once every 10 minutes, you will create more versions than Secrets Manager removes, +%% and you will reach the quota for secret versions. +%% +%% You can specify the staging labels to attach to the new version in VersionStages. +%% If you don't include VersionStages, then Secrets Manager automatically moves the staging label AWSCURRENT to this version. +%% If this operation creates the first version for the secret, then Secrets Manager automatically attaches the staging label AWSCURRENT to it. +%% If this operation moves the staging label AWSCURRENT from another version to this version, then Secrets Manager also automatically moves +%% the staging label AWSPREVIOUS to the version that AWSCURRENT was removed from. +%% +%% This operation is idempotent. If you call this operation with a ClientRequestToken that matches an existing version's VersionId, +%% and you specify the same secret data, the operation succeeds but does nothing. However, if the secret data is different, then +%% the operation fails because you can't modify an existing version; you can only create new ones. +%% +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_PutSecretValue.html] +%% @end +%%------------------------------------------------------------------------------ +-spec put_secret_value(SecretId :: binary(), ClientRequestToken :: binary(), + Secret :: secret_value()) -> sm_response(). +put_secret_value(SecretId, ClientRequestToken, Secret) -> + put_secret_value(SecretId, ClientRequestToken, Secret, []). + +-spec put_secret_value(SecretId :: binary(), ClientRequestToken :: binary(), + Secret :: secret_value(), + Opts :: put_secret_value_options()) -> sm_response(). +put_secret_value(SecretId, ClientRequestToken, Secret, Opts) -> + put_secret_value(SecretId, ClientRequestToken, Secret, Opts, erlcloud_aws:default_config()). + +-spec put_secret_value(SecretId :: binary(), ClientRequestToken :: binary(), + Secret :: secret_value(), + Opts :: put_secret_value_options(), + Config :: aws_config()) -> sm_response(). +put_secret_value(SecretId, ClientRequestToken, Secret, Opts, Config) -> + SecretValue = case Secret of + {secret_binary, Val} -> {secret_binary, base64:encode(Val)}; + {secret_string, Val} -> {secret_string, Val} + end, + put_secret_call(SecretId, ClientRequestToken, [SecretValue | Opts], Config). + + %%------------------------------------------------------------------------------ %% PutSecretValue %%------------------------------------------------------------------------------ %% @doc +%% Note: Use put_secret_value as put_secret_string and put_secret_binary functions are kept for backward compatibility. %% SM API: %% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_PutSecretValue.html] %% @end @@ -342,7 +688,7 @@ put_secret_string(SecretId, ClientRequestToken, SecretString, Opts) -> Config :: aws_config()) -> sm_response(). put_secret_string(SecretId, ClientRequestToken, SecretString, Opts, Config) -> Secret = {secret_string, SecretString}, - put_secret(SecretId, ClientRequestToken, [Secret | Opts], Config). + put_secret_call(SecretId, ClientRequestToken, [Secret | Opts], Config). -spec put_secret_binary(SecretId :: binary(), ClientRequestToken :: binary(), SecretBinary :: binary()) -> sm_response(). @@ -359,20 +705,324 @@ put_secret_binary(SecretId, ClientRequestToken, SecretBinary, Opts) -> Config :: aws_config()) -> sm_response(). put_secret_binary(SecretId, ClientRequestToken, SecretBinary, Opts, Config) -> Secret = {secret_binary, base64:encode(SecretBinary)}, - put_secret(SecretId, ClientRequestToken, [Secret | Opts], Config). + put_secret_call(SecretId, ClientRequestToken, [Secret | Opts], Config). + +%%------------------------------------------------------------------------------ +%% RemoveRegionsFromReplication +%%------------------------------------------------------------------------------ +%% @doc +%% For a secret that is replicated to other Regions, deletes the secret replicas from the Regions you specify. +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_RemoveRegionsFromReplication.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec remove_regions_from_replication(SecretId :: binary(), + Regions :: list(binary())) -> sm_response(). +remove_regions_from_replication(SecretId, Regions) -> + remove_regions_from_replication(SecretId, Regions, erlcloud_aws:default_config()). + +-spec remove_regions_from_replication(SecretId :: binary(), + Regions :: list(binary()), + Config :: aws_config()) -> sm_response(). +remove_regions_from_replication(SecretId, Regions, Config) -> + Json = [{<<"SecretId">>, SecretId}, {<<"RemoveReplicaRegions">>, Regions}], + sm_request(Config, "secretsmanager.RemoveRegionsFromReplication", Json). + +%%------------------------------------------------------------------------------ +%% ReplicateSecretToRegions +%%------------------------------------------------------------------------------ +%% @doc +%% Replicates the secret to a new Regions. +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_ReplicateSecretToRegions.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec replicate_secret_to_regions(SecretId :: binary(), + Regions :: replica_regions()) -> sm_response(). +replicate_secret_to_regions(SecretId, Regions) -> + replicate_secret_to_regions(SecretId, Regions, []). + +-spec replicate_secret_to_regions(SecretId :: binary(), + Regions :: replica_regions(), + Opts :: replicate_secret_to_regions_options()) -> sm_response(). +replicate_secret_to_regions(SecretId, Regions, Opts) -> + replicate_secret_to_regions(SecretId, Regions, Opts, erlcloud_aws:default_config()). + +-spec replicate_secret_to_regions(SecretId :: binary(), + Regions :: replica_regions(), + Opts :: replicate_secret_to_regions_options(), + Config :: aws_config()) -> sm_response(). +replicate_secret_to_regions(SecretId, Regions, Opts, Config) -> + Json = lists:map( + fun + ({force_overwrite_replica_secret, Val}) -> {<<"ForceOverwriteReplicaSecret">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId}, {<<"ReplicaRegions">>, format_replica_regions(Regions)} | Opts]), + sm_request(Config, "secretsmanager.ReplicateSecretToRegions", Json). + +%%------------------------------------------------------------------------------ +%% RestoreSecret +%%------------------------------------------------------------------------------ +%% @doc +%% Cancels the scheduled deletion of a secret by removing the DeletedDate time stamp. +%% You can access a secret again after it has been restored. +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_RestoreSecret.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec restore_secret(SecretId :: binary()) -> sm_response(). +restore_secret(SecretId) -> + restore_secret(SecretId, erlcloud_aws:default_config()). + +-spec restore_secret(SecretId :: binary(), Config :: aws_config()) -> sm_response(). +restore_secret(SecretId, Config) -> + Json = [{<<"SecretId">>, SecretId}], + sm_request(Config, "secretsmanager.RestoreSecret", Json). + +%%------------------------------------------------------------------------------ +%% RotateSecret +%%------------------------------------------------------------------------------ +%% @doc +%% Configures and starts the asynchronous process of rotating the secret. +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_RotateSecret.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec rotate_secret(SecretId :: binary(), ClientRequestToken :: binary()) -> sm_response(). +rotate_secret(SecretId, ClientRequestToken) -> + rotate_secret(SecretId, ClientRequestToken, []). +-spec rotate_secret(SecretId :: binary(), ClientRequestToken :: binary(), + Opts :: rotate_secret_options()) -> sm_response(). +rotate_secret(SecretId, ClientRequestToken, Opts) -> + rotate_secret(SecretId, ClientRequestToken, Opts, erlcloud_aws:default_config()). +-spec rotate_secret(SecretId :: binary(), ClientRequestToken :: binary(), + Opts :: rotate_secret_options(), + Config :: aws_config()) -> sm_response(). +rotate_secret(SecretId, ClientRequestToken, Opts, Config) -> + Json = lists:map( + fun + ({rotate_immediately, Val}) -> {<<"RotateImmediately">>, Val}; + ({rotation_lambda_arn, Val}) -> {<<"RotationLambdaARN">>, Val}; + ({rotation_rules, Val}) -> {<<"RotationRules">>, format_rotation_rules(Val)}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId}, {<<"ClientRequestToken">>, ClientRequestToken} | Opts]), + sm_request(Config, "secretsmanager.RotateSecret", Json). + +%%------------------------------------------------------------------------------ +%% StopReplicationToReplicate +%%------------------------------------------------------------------------------ +%% @doc +%% Removes the link between the replica secret and the primary secret and promotes the replica to a primary secret in the replica Region. +%% +%% You must call this operation from the Region in which you want to promote the replica to a primary secret. +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_StopReplicationToReplica.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec stop_replication_to_replica(SecretId :: binary()) -> sm_response(). +stop_replication_to_replica(SecretId) -> + stop_replication_to_replica(SecretId, erlcloud_aws:default_config()). +-spec stop_replication_to_replica(SecretId :: binary(), Config :: aws_config()) -> sm_response(). +stop_replication_to_replica(SecretId, Config) -> + Json = [{<<"SecretId">>, SecretId}], + sm_request(Config, "secretsmanager.StopReplicationToReplica", Json). + +%%------------------------------------------------------------------------------ +%% TagResource +%%------------------------------------------------------------------------------ +%% @doc +%% Attaches tags to a secret. Tags consist of a key name and a value. +%% Tags are part of the secret's metadata. They are not associated with specific versions +%% of the secret. This operation appends tags to the existing list of tags. +%% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_TagResource.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec tag_resource(SecretId :: binary(), Tags :: list()) -> sm_response(). +tag_resource(SecretId, Tags) -> + tag_resource(SecretId, Tags, erlcloud_aws:default_config()). +-spec tag_resource(SecretId :: binary(), Tags :: list(), Config :: aws_config()) -> sm_response(). +tag_resource(SecretId, Tags, Config) -> + Json = [{<<"SecretId">>, SecretId}, {<<"Tags">>, format_tags(Tags)}], + sm_request(Config, "secretsmanager.TagResource", Json). + +%%------------------------------------------------------------------------------ +%% UntagResource +%%------------------------------------------------------------------------------ +%% @doc +%% Removes tags from a secret. +%% This operation is idempotent. If a requested tag is not attached to the secret, no error is returned and the secret metadata is unchanged. +%% %% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_UntagResource.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec untag_resource(SecretId :: binary(), TagKeys :: list(binary())) -> sm_response(). +untag_resource(SecretId, TagKeys) -> + untag_resource(SecretId, TagKeys, erlcloud_aws:default_config()). +-spec untag_resource(SecretId :: binary(), TagKeys :: list(binary()), + Config :: aws_config()) -> sm_response(). +untag_resource(SecretId, TagKeys, Config) -> + Json = [{<<"SecretId">>, SecretId}, {<<"TagKeys">>, TagKeys}], + sm_request(Config, "secretsmanager.UntagResource", Json). + +%%------------------------------------------------------------------------------ +%% UpdateSecret +%%------------------------------------------------------------------------------ +%% @doc +%% Modifies the details of a secret, including metadata and the secret value. To change the secret value, +%% you can also use PutSecretValue. +%% +%% To change the rotation configuration of a secret, use RotateSecret instead. +%% +%% It is recommended to avoid calling UpdateSecret at a sustained rate of more than once every 10 minutes. +%% When you call UpdateSecret to update the secret value, Secrets Manager creates a new version of the secret. +%% Secrets Manager removes outdated versions when there are more than 100, but it does not remove versions +%% created less than 24 hours ago. If you update the secret value more than once every 10 minutes, you create +%% more versions than Secrets Manager removes, and you will reach the quota for secret versions. +%% +%% If you include SecretString or SecretBinary to create a new secret version, Secrets Manager automatically +%% moves the staging label AWSCURRENT to the new version. Then it attaches the label AWSPREVIOUS to the version +%% that AWSCURRENT was removed from. +%% +%% If you call this operation with a ClientRequestToken that matches an existing version's VersionId, +%% the operation results in an error. You can't modify an existing version, you can only create a new version. +%% To remove a version, remove all staging labels from it. See UpdateSecretVersionStage. +%% +%% %% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_UpdateSecret.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec update_secret(SecretId :: binary(), ClientRequestToken :: binary()) -> sm_response(). +update_secret(SecretId, ClientRequestToken) -> + update_secret(SecretId, ClientRequestToken, []). +-spec update_secret(SecretId :: binary(), + ClientRequestToken :: binary(), + Opts :: update_secret_options()) -> sm_response(). +update_secret(SecretId, ClientRequestToken, Opts) -> + update_secret(SecretId, ClientRequestToken, Opts, erlcloud_aws:default_config()). +-spec update_secret(SecretId :: binary(), + ClientRequestToken :: binary(), + Opts :: update_secret_options(), + Config :: aws_config()) -> sm_response(). +update_secret(SecretId, ClientRequestToken, Opts, Config) -> + Json = lists:map( + fun + ({secret_binary, Val}) -> {<<"SecretBinary">>, base64:encode(Val)}; + ({secret_string, Val}) -> {<<"SecretString">>, Val}; + ({description, Val}) -> {<<"Description">>, Val}; + ({kms_key_id, Val}) -> {<<"KmsKeyId">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId}, {<<"ClientRequestToken">>, ClientRequestToken} | Opts]), + sm_request(Config, "secretsmanager.UpdateSecret", Json). + +%%------------------------------------------------------------------------------ +%% UpdateSecretVersionStage +%%------------------------------------------------------------------------------ +%% @doc +%% Modifies the staging labels attached to a version of a secret. Secrets Manager uses +%% staging labels to track a version as it progresses through the secret rotation process. +%% Each staging label can be attached to only one version at a time. To add a staging +%% label to a version when it is already attached to another version, Secrets Manager +%% first removes it from the other version first and then attaches it to this one. +%% +%% The staging labels that you specify in the VersionStage parameter are added to the +%% existing list of staging labels for the version. +%% +%% %% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_UpdateSecretVersionStage.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec update_secret_version_stage(SecretId :: binary(), + VersionStage :: binary()) -> sm_response(). +update_secret_version_stage(SecretId, VersionStage) -> + update_secret_version_stage(SecretId, VersionStage, []). +-spec update_secret_version_stage(SecretId :: binary(), + VersionStage :: binary(), + Opts :: update_secret_version_stage_options()) -> sm_response(). +update_secret_version_stage(SecretId, VersionStage, Opts) -> + update_secret_version_stage(SecretId, VersionStage, Opts, erlcloud_aws:default_config()). +-spec update_secret_version_stage(SecretId :: binary(), + VersionStage :: binary(), + Opts :: update_secret_version_stage_options(), + Config :: aws_config()) -> sm_response(). +update_secret_version_stage(SecretId, VersionStage, Opts, Config) -> + Json = lists:map( + fun + ({remove_from_version_id, Val}) -> {<<"RemoveFromVersionId">>, Val}; + ({move_to_version_id, Val}) -> {<<"MoveToVersionId">>, Val}; + (Other) -> Other + end, + [{<<"SecretId">>, SecretId}, {<<"VersionStage">>, VersionStage} | Opts]), + sm_request(Config, "secretsmanager.UpdateSecretVersionStage", Json). + +%%------------------------------------------------------------------------------ +%% ValidateResourcePolicy +%%------------------------------------------------------------------------------ +%% @doc +%% Modifies the resource policy attached to a secret. Secrets Manager uses resource +%% policies to manage access to secrets. This operation allows you to add or remove +%% permissions for specific principals (IAM users, roles, etc.) on a secret. +%% +%% %% SM API: +%% [https://docs.aws.amazon.com/secretsmanager/latest/apireference/API_ValidateResourcePolicy.html] +%% @end +%%------------------------------------------------------------------------------ + +-spec validate_resource_policy(ResourcePolicy :: binary()) -> sm_response(). +validate_resource_policy(ResourcePolicy) -> + validate_resource_policy(ResourcePolicy, []). +-spec validate_resource_policy(ResourcePolicy :: binary(), + Opts :: validate_resource_policy_options()) -> sm_response(). +validate_resource_policy(ResourcePolicy, Opts) -> + validate_resource_policy(ResourcePolicy, Opts, erlcloud_aws:default_config()). +-spec validate_resource_policy(ResourcePolicy :: binary(), + Opts :: validate_resource_policy_options(), + Config :: aws_config()) -> sm_response(). +validate_resource_policy(ResourcePolicy, Opts, Config) -> + Json = lists:map( + fun + ({secret_id, Val}) -> {<<"SecretId">>, Val}; + (Other) -> Other + end, + [{<<"ResourcePolicy">>, ResourcePolicy} | Opts]), + sm_request(Config, "secretsmanager.ValidateResourcePolicy", Json). + %%%------------------------------------------------------------------------------ %%% Internal Functions %%%------------------------------------------------------------------------------ -create_secret(SecretName, ClientRequestToken, Opts, Config) -> - Opts1 = [{client_request_token, ClientRequestToken} | Opts], - Json = create_secret_payload(SecretName, Opts1), +create_secret_call(SecretName, ClientRequestToken, Opts, Config) -> + Json = lists:map( + fun + ({add_replica_regions, Val}) -> {<<"AddReplicaRegions">>, format_replica_regions(Val)}; + ({description, Val}) -> {<<"Description">>, Val}; + ({force_overwrite_replica_secret, Val}) -> {<<"ForceOverwriteReplicaSecret">>, Val}; + ({kms_key_id, Val}) -> {<<"KmsKeyId">>, Val}; + ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; + ({secret_string, Val}) -> {<<"SecretString">>, Val}; + ({tags, Val}) -> {<<"Tags">>, format_tags(Val)}; + (Other) -> Other + end, + [{<<"Name">>, SecretName}, {<<"ClientRequestToken">>, ClientRequestToken} | Opts]), sm_request(Config, "secretsmanager.CreateSecret", Json). -put_secret(SecretId, ClientRequestToken, Opts, Config) -> +put_secret_call(SecretId, ClientRequestToken, Opts, Config) -> Json = lists:map( fun + ({rotation_token, Val}) -> {<<"RotationToken">>, Val}; ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; ({secret_string, Val}) -> {<<"SecretString">>, Val}; ({version_stages, Val}) -> {<<"VersionStages">>, Val}; @@ -382,6 +1032,57 @@ put_secret(SecretId, ClientRequestToken, Opts, Config) -> sm_request(Config, "secretsmanager.PutSecretValue", Json). +-spec format_replica_regions(ReplicaRegions :: replica_regions()) -> list(). +format_replica_regions(ReplicaRegions) -> + lists:map(fun format_replica_region/1, ReplicaRegions). + +-spec format_replica_region(ReplicaRegion :: replica_region()) -> list(). +format_replica_region(ReplicaRegion) -> + format_replica_region(ReplicaRegion, []). + +-spec format_replica_region(ReplicaRegion :: replica_region(), + Acc :: list()) -> list(). +format_replica_region([{region, Region} | Rest], Acc) -> + format_replica_region(Rest, [{<<"Region">>, Region} | Acc]); +format_replica_region([{kms_key_id, KmsKeyId} | Rest], Acc) -> + format_replica_region(Rest, [{<<"KmsKeyId">>, KmsKeyId} | Acc]); +format_replica_region([Val | Rest], Acc) -> + format_replica_region(Rest, [Val | Acc]); +format_replica_region([], Acc) -> + Acc. + + +-spec format_tags(Tags :: list(proplists:proplist())) -> list(proplists:proplist()). +format_tags(Tags) -> + lists:map(fun format_tag/1, Tags). + +-spec format_tag(Tags :: proplists:proplist()) -> list(). +format_tag(Tags) -> + format_tag(Tags, []). + +-spec format_tag(Tags :: proplists:proplist(), + Acc :: list()) -> list(). +format_tag([{key, Key} | Rest], Acc) -> + format_tag(Rest, [{<<"Key">>, Key} | Acc]); +format_tag([{value, Value} | Rest], Acc) -> + format_tag(Rest, [{<<"Value">>, Value} | Acc]); +format_tag([Val | Rest], Acc) -> + format_tag(Rest, [Val | Acc]); +format_tag([], Acc) -> + Acc. + +-spec format_rotation_rules(RotationRules :: rotation_rules()) -> list(). +format_rotation_rules(RotationRules) -> + lists:map( + fun + ({automatically_after_days, Val}) -> {<<"AutomaticallyAfterDays">>, Val}; + ({duration, Val}) -> {<<"Duration">>, Val}; + ({schedule_expression, Val}) -> {<<"ScheduleExpression">>, Val}; + (Other) -> Other + end, + RotationRules). + + sm_request(Config, Operation, Body) -> case erlcloud_aws:update_config(Config) of {ok, Config1} -> @@ -432,19 +1133,3 @@ sm_result_fun(#aws_request{response_type = error, Request#aws_request{should_retry = true}; sm_result_fun(#aws_request{response_type = error, error_type = aws} = Request) -> Request#aws_request{should_retry = false}. - -create_secret_payload(SecretName, Opts) -> - Json = lists:map( - fun - ({add_replica_regions, Val}) -> {<<"AddReplicaRegions">>, Val}; - ({client_request_token, Val}) -> {<<"ClientRequestToken">>, Val}; - ({description, Val}) -> {<<"Description">>, Val}; - ({force_overwrite_replica_secret, Val}) -> {<<"ForceOverwriteReplicaSecret">>, Val}; - ({kms_key_id, Val}) -> {<<"KmsKeyId">>, Val}; - ({secret_binary, Val}) -> {<<"SecretBinary">>, Val}; - ({secret_string, Val}) -> {<<"SecretString">>, Val}; - ({tags, Val}) -> {<<"Tags">>, Val}; - (Other) -> Other - end, - [{<<"Name">>, SecretName} | Opts]), - Json. diff --git a/test/erlcloud_sm_tests.erl b/test/erlcloud_sm_tests.erl index 7c88ef7a5..83c1b3664 100644 --- a/test/erlcloud_sm_tests.erl +++ b/test/erlcloud_sm_tests.erl @@ -23,10 +23,14 @@ %%%=================================================================== -define(SECRET_ID, <<"MyTestDatabaseSecret">>). +-define(SECRET_ID2, <<"MyTestDatabaseSecret2">>). -define(SECRET_STRING, <<"{\"username\":\"david\",\"password\":\"SECRET-PASSWORD\"}">>). -define(SECRET_BINARY, base64:encode(?SECRET_STRING)). -define(CLIENT_REQUEST_TOKEN, <<"EXAMPLE2-90ab-cdef-fedc-ba987EXAMPLE">>). +-define(MAX_RESULTS, 10). +-define(NEXT_TOKEN, <<"TOKEN">>). -define(VERSION_ID, <<"EXAMPLE1-90ab-cdef-fedc-ba987SECRET1">>). +-define(VERSION_ID2, <<"EXAMPLE1-90ab-cdef-fedc-ba987SECRET2">>). -define(VERSION_STAGE, <<"AWSPREVIOUS">>). %%%=================================================================== @@ -38,10 +42,42 @@ operation_test_() -> fun start/0, fun stop/1, [ + fun batch_get_secret_value_input_tests/1, + fun batch_get_secret_value_output_tests/1, + fun cancel_rotate_secret_value_input_tests/1, + fun cancel_rotate_secret_value_output_tests/1, + fun create_secret_value_input_tests/1, + fun create_secret_value_output_tests/1, + fun get_random_password_value_input_tests/1, + fun get_random_password_value_output_tests/1, fun get_secret_value_input_tests/1, fun get_secret_value_output_tests/1, + fun list_secrets_value_input_tests/1, + fun list_secrets_value_output_tests/1, + fun list_secret_version_ids_value_input_tests/1, + fun list_secret_version_ids_value_output_tests/1, fun put_secret_value_input_tests/1, - fun put_secret_value_output_tests/1 + fun put_secret_value_output_tests/1, + fun remove_regions_from_replication_value_input_tests/1, + fun remove_regions_from_replication_value_output_tests/1, + fun replicate_secret_to_regions_value_input_tests/1, + fun replicate_secret_to_regions_value_output_tests/1, + fun restore_secret_value_input_tests/1, + fun restore_secret_value_output_tests/1, + fun rotate_secret_value_input_tests/1, + fun rotate_secret_value_output_tests/1, + fun stop_replication_to_replica_value_input_tests/1, + fun stop_replication_to_replica_value_output_tests/1, + fun tag_resource_value_input_tests/1, + fun tag_resource_value_output_tests/1, + fun untag_resource_value_input_tests/1, + fun untag_resource_value_output_tests/1, + fun update_secret_value_input_tests/1, + fun update_secret_value_output_tests/1, + fun update_secret_version_stage_value_input_tests/1, + fun update_secret_version_stage_value_output_tests/1, + fun validate_resource_policy_value_input_tests/1, + fun validate_resource_policy_value_output_tests/1 ]}. start() -> @@ -144,6 +180,336 @@ output_tests(Fun, Tests) -> %%%=================================================================== %%% Tests %%%=================================================================== +batch_get_secret_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"batch_get_secret_value2id input test", + ?_f( + erlcloud_sm:batch_get_secret_value( + {secret_id_list, [?SECRET_ID, ?SECRET_ID2]}, + [{max_results, ?MAX_RESULTS}, {next_token, ?NEXT_TOKEN}] + ) + ), + jsx:encode([ + {<<"SecretIdList">>, [?SECRET_ID, ?SECRET_ID2]}, + {<<"MaxResults">>, ?MAX_RESULTS}, + {<<"NextToken">>, ?NEXT_TOKEN} + ]) + }), + ?_sm_test( + {"batch_get_secret_value2Filter input test", + ?_f( + erlcloud_sm:batch_get_secret_value( + {filters, [[{<<"Key">>,<<"name">>}, {<<"Values">>, [<<"value1">>, <<"value2">>]}]]}, + [{max_results, ?MAX_RESULTS}, {next_token, ?NEXT_TOKEN}] + ) + ), + jsx:encode([ + {<<"Filters">>, [[{<<"Key">>,<<"name">>}, {<<"Values">>, [<<"value1">>, <<"value2">>]}]]}, + {<<"MaxResults">>, ?MAX_RESULTS}, + {<<"NextToken">>, ?NEXT_TOKEN} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + +-define(BATCH_GET_SECRET_VALUE_RESP,[ + {<<"Errors">>, [ + {<<"ErrorCode">>, <<"ResourceNotFoundException">>}, + {<<"ErrorMessage">>, <<"Secret with id 'NonExistentSecret' not found">>}, + {<<"SecretId">>, <<"NonExistentSecret">>}]}, + {<<"NextToken">>, ?NEXT_TOKEN}, + {<<"SecretValues">>, [ + [ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"CreatedDate">>, 1.523477145713E9}, + {<<"Name">>, ?SECRET_ID}, + {<<"SecretString">>, + <<"{\n \"username\":\"david\",\n \"password\":\"BnQw&XDWgaEeT9XGTT29\"\n}\n">>}, + {<<"VersionId">>, ?VERSION_ID}, + {<<"VersionStages">>, [?VERSION_STAGE]} + ], + [ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret2-b2c3d4">>}, + {<<"CreatedDate">>, 1.523477145713E9}, + {<<"Name">>, ?SECRET_ID2}, + {<<"SecretString">>, + <<"{\n \"username\":\"alice\",\n \"password\":\"XyZ1234567890\"\n}\n">>}, + {<<"VersionId">>, ?VERSION_ID2}, + {<<"VersionStages">>, [?VERSION_STAGE]} + ] + ]} +]). + + +batch_get_secret_value_output_tests(_) -> + Tests = [?_sm_test( + {"batch_get_secret_value output test", + jsx:encode(?BATCH_GET_SECRET_VALUE_RESP), + {ok, ?BATCH_GET_SECRET_VALUE_RESP}} + )], + + output_tests(?_f( + erlcloud_sm:batch_get_secret_value( + {secret_id_list, [?SECRET_ID, ?SECRET_ID2]}, + [{max_results, ?MAX_RESULTS}, {next_token, ?NEXT_TOKEN}]) + ), + Tests), + output_tests(?_f( + erlcloud_sm:batch_get_secret_value( + {filters, [[{<<"Key">>,<<"name">>}, {<<"Values">>, [?SECRET_ID, ?SECRET_ID2]}]]}, + [{max_results, ?MAX_RESULTS}, {next_token, ?NEXT_TOKEN}]) + ), + Tests). + +cancel_rotate_secret_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"cancel_rotate_secret input test", + ?_f(erlcloud_sm:cancel_rotate_secret(?SECRET_ID)), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + +-define(CANCEL_ROTATE_SECRET_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"Name">>, ?SECRET_ID}, + {<<"VersionId">>, ?VERSION_ID} +]). + +cancel_rotate_secret_value_output_tests(_) -> + Tests = [?_sm_test( + {"cancel_rotate_secret output test", + jsx:encode(?CANCEL_ROTATE_SECRET_RESP), + {ok, ?CANCEL_ROTATE_SECRET_RESP}} + )], + + output_tests(?_f(erlcloud_sm:cancel_rotate_secret(?SECRET_ID)), Tests). + + +create_secret_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"create_secret_string input test", + ?_f(erlcloud_sm:create_secret_string(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"Name">>,?SECRET_ID}, + {<<"SecretString">>,?SECRET_STRING}]) + }), + ?_sm_test( + {"create_secret_binary input test", + ?_f(erlcloud_sm:create_secret_binary(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"Name">>,?SECRET_ID}, + {<<"SecretBinary">>,?SECRET_BINARY}]) + }), + ?_sm_test( + {"create_secret string input test", + ?_f(erlcloud_sm:create_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, {secret_string, ?SECRET_STRING})), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"Name">>,?SECRET_ID}, + {<<"SecretString">>,?SECRET_STRING}]) + }), + ?_sm_test( + {"create_secret binary input test", + ?_f(erlcloud_sm:create_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, {secret_binary, ?SECRET_STRING})), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"Name">>,?SECRET_ID}, + {<<"SecretBinary">>,?SECRET_BINARY}]) + }), + ?_sm_test( + {"create_secret binary with options input test", + ?_f(erlcloud_sm:create_secret( + ?SECRET_ID, + ?CLIENT_REQUEST_TOKEN, + {secret_binary, ?SECRET_STRING}, + [ + {add_replica_regions, [ + [ + {<<"Region">>, <<"us-east-1">>}, + {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ] + ]}, + {description, <<"My test database secret">>}, + {force_overwrite_replica_secret, true}, + {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {tags, [ + [ + {<<"Key">>, <<"Environment">>}, + {<<"Value">>, <<"Production">>} + ] + ]} + ] + )), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"Name">>,?SECRET_ID}, + {<<"SecretBinary">>,?SECRET_BINARY}, + {<<"AddReplicaRegions">>, [ + [ + {<<"Region">>, <<"us-east-1">>}, + {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ] + ]}, + {<<"Description">>, <<"My test database secret">>}, + {<<"ForceOverwriteReplicaSecret">>, true}, + {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {<<"Tags">>, [ + [ + {<<"Key">>, <<"Environment">>}, + {<<"Value">>, <<"Production">>} + ] + ]} + ]) + }), + ?_sm_test( + {"create_secret binary with options input test", + ?_f(erlcloud_sm:create_secret( + ?SECRET_ID, + ?CLIENT_REQUEST_TOKEN, + {secret_binary, ?SECRET_STRING}, + [ + {add_replica_regions, [ + [ + {region, <<"us-east-1">>}, + {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ], + [ + {region, <<"us-west-2">>}, + {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ] + ]}, + {description, <<"My test database secret">>}, + {force_overwrite_replica_secret, true}, + {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {tags, [ + [ + {key, <<"Environment">>}, + {value, <<"Production">>} + ] + ]} + ] + )), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"Name">>,?SECRET_ID}, + {<<"SecretBinary">>,?SECRET_BINARY}, + {<<"AddReplicaRegions">>, [ + [ + {<<"Region">>, <<"us-east-1">>}, + {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ], + [ + {<<"Region">>, <<"us-west-2">>}, + {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ] + ]}, + {<<"Description">>, <<"My test database secret">>}, + {<<"ForceOverwriteReplicaSecret">>, true}, + {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {<<"Tags">>, [ + [ + {<<"Key">>, <<"Environment">>}, + {<<"Value">>, <<"Production">>} + ] + ]} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + +-define(CREATE_SECRET_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"Name">>, ?SECRET_ID}, + {<<"ReplicationStatus">>, [ + [ + {<<"KmsKeyId">>, + <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {<<"LastAccessedDate">>, 1.523477145813E9}, + {<<"Region">>, <<"us-east-1">>}, + {<<"Status">>, <<"InSync">>}, + {<<"StatusMessage">>, <<"Replication succeeded">>} + ] + ]}, + {<<"VersionId">>, ?VERSION_ID} +]). + + +create_secret_value_output_tests(_) -> + Tests = [?_sm_test( + {"create_secret output test", + jsx:encode(?CREATE_SECRET_RESP), + {ok, ?CREATE_SECRET_RESP}} + )], + + output_tests(?_f(erlcloud_sm:create_secret_string(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), Tests), + output_tests(?_f(erlcloud_sm:create_secret_binary(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), Tests), + output_tests(?_f(erlcloud_sm:create_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, {secret_string, ?SECRET_STRING})), Tests), + output_tests(?_f(erlcloud_sm:create_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, {secret_binary, ?SECRET_BINARY})), Tests). + +get_random_password_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"get_random_password value input test", + ?_f(erlcloud_sm:get_random_password()), + jsx:encode([]) + }), + ?_sm_test( + {"get_random_password input test", + ?_f(erlcloud_sm:get_random_password( + [ + {<<"ExcludeCharacters">>, <<"!@#$%^&*()_+">>}, + {<<"ExcludeLowercase">>, true}, + {<<"ExcludeNumbers">>, true}, + {<<"ExcludePunctuation">>, true}, + {<<"ExcludeUppercase">>, true}, + {<<"IncludeSpace">>, true}, + {<<"PasswordLength">>, 16}, + {<<"RequireEachIncludedType">>, true} + ])), + jsx:encode( + [ + {<<"ExcludeCharacters">>, <<"!@#$%^&*()_+">>}, + {<<"ExcludeLowercase">>, true}, + {<<"ExcludeNumbers">>, true}, + {<<"ExcludePunctuation">>, true}, + {<<"ExcludeUppercase">>, true}, + {<<"IncludeSpace">>, true}, + {<<"PasswordLength">>, 16}, + {<<"RequireEachIncludedType">>, true} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + +-define(GET_RANDOM_PASSWORD_RESP,[ + {<<"RandomPassword">>, <<"BnQw&XDWgaEeT9XGTT29">>} +]). + +get_random_password_value_output_tests(_) -> + Tests = [?_sm_test( + {"get_random_password value output test", + jsx:encode(?GET_RANDOM_PASSWORD_RESP), + {ok, ?GET_RANDOM_PASSWORD_RESP}} + )], + + output_tests(?_f(erlcloud_sm:get_random_password()), Tests). get_secret_value_input_tests(_) -> Tests = [ @@ -190,6 +556,139 @@ get_secret_value_output_tests(_) -> output_tests(?_f(erlcloud_sm:get_secret_value(?SECRET_ID, [{version_id, ?VERSION_ID}])), Tests), output_tests(?_f(erlcloud_sm:get_secret_value(?SECRET_ID, [{version_stage, ?VERSION_STAGE}])), Tests). +list_secrets_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"list_secrets value input test", + ?_f(erlcloud_sm:list_secrets()), + jsx:encode([]) + }), + ?_sm_test( + {"list_secrets input test", + ?_f(erlcloud_sm:list_secrets( + [ + {filters, [[{<<"Key">>,<<"name">>}, {<<"Values">>, [<<"value1">>, <<"value2">>]}]]}, + {max_results, ?MAX_RESULTS}, + {include_planned_deletion, true}, + {next_token, ?NEXT_TOKEN}, + {sort_order, <<"asc">>} + ])), + jsx:encode( + [ + {<<"Filters">>, [[{<<"Key">>,<<"name">>}, {<<"Values">>, [<<"value1">>, <<"value2">>]}]]}, + {<<"MaxResults">>, ?MAX_RESULTS}, + {<<"IncludePlannedDeletion">>, true}, + {<<"NextToken">>, ?NEXT_TOKEN}, + {<<"SortOrder">>, <<"asc">>} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + +-define(LIST_SECRETS_VALUE_RESP,[ + {<<"NextToken">>, ?NEXT_TOKEN}, + {<<"SecretList">>, [ + [ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"CreatedDate">>, 1.523477145713E9}, + {<<"DeletedDate">>, 1.523477145813E9}, + {<<"Description">>, <<"My test database secret">>}, + {<<"KmsKeyId">>, + <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {<<"LastAccessedDate">>, 1.523477145913E9}, + {<<"LastChangedDate">>, 1.523477146013E9}, + {<<"LastRotatedDate">>, 1.523477146113E9}, + {<<"Name">>, ?SECRET_ID}, + {<<"NextRotationDate">>, 1.523477146213E9}, + {<<"OwningService">>, <<"secretsmanager">>}, + {<<"PrimaryRegion">>, <<"us-west-2">>}, + {<<"RotationEnabled">>, true}, + {<<"RotationLambdaARN">>, + <<"arn:aws:lambda:us-west-2:123456789012:function:MySecretRotationFunction">>}, + {<<"RotationRules">>, [ + {<<"AutomaticallyAfterDays">>,90}, + {<<"Duration">>,<<"3h">>}, + {<<"ScheduleExpression">>,<<"rate(30 days)">>} + ]}, + {<<"SecretVersionsToStages">>, [ + { ?VERSION_ID, [?VERSION_STAGE]}, + { ?VERSION_ID2, [<<"AWSCURRENT">>]} + ]}, + {<<"Tags">>, [ + [ + {<<"Key">>, <<"Environment">>}, + {<<"Value">>, <<"Production">>} + ] + ]} + ] + ]} +]). + +list_secrets_value_output_tests(_) -> + Tests = [?_sm_test( + {"list_secrets value output test", + jsx:encode(?LIST_SECRETS_VALUE_RESP), + {ok, ?LIST_SECRETS_VALUE_RESP}} + )], + + output_tests(?_f(erlcloud_sm:list_secrets()), Tests). + +list_secret_version_ids_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"list_secret_version_ids value input test", + ?_f(erlcloud_sm:list_secret_version_ids(?SECRET_ID)), + jsx:encode([{<<"SecretId">>, ?SECRET_ID}]) + }), + ?_sm_test( + {"list_secret_version_ids input test", + ?_f(erlcloud_sm:list_secret_version_ids( + ?SECRET_ID, + [ + {include_deprecated, true}, + {max_results, ?MAX_RESULTS}, + {next_token, ?NEXT_TOKEN} + ])), + jsx:encode( + [ + {<<"SecretId">>, ?SECRET_ID}, + {<<"IncludeDeprecated">>, true}, + {<<"MaxResults">>, ?MAX_RESULTS}, + {<<"NextToken">>, ?NEXT_TOKEN} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + +-define(LIST_SECRET_VERSION_IDS_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"Name">>, ?SECRET_ID}, + {<<"NextToken">>, ?NEXT_TOKEN}, + {<<"Versions">>, [ + [ + {<<"CreatedDate">>, 1.523477145713E9}, + {<<"KmsKeyId">>, + <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {<<"LastAccessedDate">>, 1.523477145813E9}, + {<<"VersionId">>, ?VERSION_ID}, + {<<"VersionStages">>, [?VERSION_STAGE]} + ] + ]} +]). + +list_secret_version_ids_value_output_tests(_) -> + Tests = [?_sm_test( + {"list_secret_version_ids value output test", + jsx:encode(?LIST_SECRET_VERSION_IDS_VALUE_RESP), + {ok, ?LIST_SECRET_VERSION_IDS_VALUE_RESP}} + )], + + output_tests(?_f(erlcloud_sm:list_secret_version_ids(?SECRET_ID)), Tests). + put_secret_value_input_tests(_) -> Tests = [ @@ -208,6 +707,40 @@ put_secret_value_input_tests(_) -> {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, {<<"SecretId">>,?SECRET_ID}, {<<"SecretBinary">>,?SECRET_BINARY}]) + }), + ?_sm_test( + {"put_secret_value string input test", + ?_f(erlcloud_sm:put_secret_value(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, {secret_string, ?SECRET_STRING})), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}, + {<<"SecretString">>,?SECRET_STRING}]) + }), + ?_sm_test( + {"put_secret_value binary input test", + ?_f(erlcloud_sm:put_secret_value(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, {secret_binary, ?SECRET_STRING})), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}, + {<<"SecretBinary">>,?SECRET_BINARY}]) + }), + ?_sm_test( + {"put_secret_value binary input test", + ?_f(erlcloud_sm:put_secret_value( + ?SECRET_ID, + ?CLIENT_REQUEST_TOKEN, + {secret_binary, ?SECRET_STRING}, + [ + {version_stages, [?VERSION_STAGE]}, + {rotation_token, <<"ROTATION-TOKEN-EXAMPLE-1234">>} + ])), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}, + {<<"SecretBinary">>,?SECRET_BINARY}, + {<<"VersionStages">>, [?VERSION_STAGE]}, + {<<"RotationToken">>, <<"ROTATION-TOKEN-EXAMPLE-1234">>} + ]) }) ], Response = <<>>, @@ -231,4 +764,445 @@ put_secret_value_output_tests(_) -> )], output_tests(?_f(erlcloud_sm:put_secret_string(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), Tests), - output_tests(?_f(erlcloud_sm:put_secret_binary(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), Tests). + output_tests(?_f(erlcloud_sm:put_secret_binary(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, ?SECRET_STRING)), Tests), + output_tests(?_f(erlcloud_sm:put_secret_value(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, {secret_string, ?SECRET_STRING})), Tests), + output_tests(?_f(erlcloud_sm:put_secret_value(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, {secret_binary, ?SECRET_BINARY})), Tests). + +remove_regions_from_replication_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"remove_regions_from_replication value input test", + ?_f(erlcloud_sm:remove_regions_from_replication(?SECRET_ID, [<<"us-east-1">>])), + jsx:encode([{<<"SecretId">>, ?SECRET_ID}, {<<"RemoveReplicaRegions">>, [<<"us-east-1">>]}]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + +-define(REMOVE_REGIONS_FROM_REPLICATION_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"ReplicationStatus">>, [ + [ + {<<"KmsKeyId">>, + <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {<<"LastAccessedDate">>, 1.523477145813E9}, + {<<"Region">>, <<"us-west-2">>}, + {<<"Status">>, <<"InSync">>}, + {<<"StatusMessage">>, <<"Replication succeeded">>} + ] + ]} +]). + +remove_regions_from_replication_value_output_tests(_) -> + Tests = [?_sm_test( + {"remove_regions_from_replication value output test", + jsx:encode(?REMOVE_REGIONS_FROM_REPLICATION_VALUE_RESP), + {ok, ?REMOVE_REGIONS_FROM_REPLICATION_VALUE_RESP}} + )], + + output_tests(?_f(erlcloud_sm:remove_regions_from_replication(?SECRET_ID, [<<"us-east-1">>])), Tests). + +replicate_secret_to_regions_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"replicate_secret_to_regions value input test", + ?_f(erlcloud_sm:replicate_secret_to_regions( + ?SECRET_ID, + [[{<<"Region">>, <<"us-east-1">>}, {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}]] + )), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"ReplicaRegions">>, [[{<<"Region">>, <<"us-east-1">>}, {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}]]} + ]) + }), + ?_sm_test( + {"replicate_secret_to_regions value input test", + ?_f(erlcloud_sm:replicate_secret_to_regions( + ?SECRET_ID, + [[{region, <<"us-east-1">>}, {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}]] + )), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"ReplicaRegions">>, [[{<<"Region">>, <<"us-east-1">>}, {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}]]} + ]) + }), + ?_sm_test( + {"replicate_secret_to_regions value input test", + ?_f(erlcloud_sm:replicate_secret_to_regions( + ?SECRET_ID, + [[{<<"Region">>, <<"us-east-1">>}, {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}]], + [{force_overwrite_replica_secret, true}] + )), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"ReplicaRegions">>, [[{<<"Region">>, <<"us-east-1">>}, {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}]]}, + {<<"ForceOverwriteReplicaSecret">>, true} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + +-define(REPLICATE_SECRET_TO_REGIONS_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"ReplicationStatus">>, [ + [ + {<<"KmsKeyId">>, + <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {<<"LastAccessedDate">>, 1.523477145813E9}, + {<<"Region">>, <<"us-west-2">>}, + {<<"Status">>, <<"InSync">>}, + {<<"StatusMessage">>, <<"Replication succeeded">>} + ], + [ + {<<"KmsKeyId">>, + <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}, + {<<"LastAccessedDate">>, 1.523477145813E9}, + {<<"Region">>, <<"us-east-1">>}, + {<<"Status">>, <<"InSync">>}, + {<<"StatusMessage">>, <<"Replication succeeded">>} + ] + ]} +]). + +replicate_secret_to_regions_value_output_tests(_) -> + Tests = [?_sm_test( + {"replicate_secret_to_regions value output test", + jsx:encode(?REPLICATE_SECRET_TO_REGIONS_VALUE_RESP), + {ok, ?REPLICATE_SECRET_TO_REGIONS_VALUE_RESP}} + )], + + output_tests(?_f(erlcloud_sm:replicate_secret_to_regions(?SECRET_ID, [[{region, <<"us-east-1">>}, {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>}]])), Tests). + +restore_secret_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"restore_secret input test", + ?_f(erlcloud_sm:restore_secret(?SECRET_ID)), + jsx:encode([{<<"SecretId">>, ?SECRET_ID}]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + +-define(RESTORE_SECRET_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"Name">>, ?SECRET_ID}]). + +restore_secret_value_output_tests(_) -> + Tests = [?_sm_test( + {"restore_secret value output test", + jsx:encode(?RESTORE_SECRET_VALUE_RESP), + {ok, ?RESTORE_SECRET_VALUE_RESP}} + )], + + output_tests(?_f(erlcloud_sm:restore_secret(?SECRET_ID)), Tests). + +rotate_secret_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"rotate_secret input test", + ?_f(erlcloud_sm:rotate_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN)), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}]) + }), + ?_sm_test( + {"rotate_secret with options input test", + ?_f(erlcloud_sm:rotate_secret( + ?SECRET_ID, + ?CLIENT_REQUEST_TOKEN, + [ + {rotation_lambda_arn, <<"arn:aws:lambda:us-west-2:123456789012:function:MySecretRotationFunction">>}, + {rotate_immediately, false}, + {rotation_rules, [ + {automatically_after_days, 30}, + {duration, <<"3h">>}, + {schedule_expression, <<"rate(30 days)">>} + ]} + ] + )), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}, + {<<"RotationLambdaARN">>, <<"arn:aws:lambda:us-west-2:123456789012:function:MySecretRotationFunction">>}, + {<<"RotateImmediately">>, false}, + {<<"RotationRules">>, [ + {<<"AutomaticallyAfterDays">>,30}, + {<<"Duration">>,<<"3h">>}, + {<<"ScheduleExpression">>,<<"rate(30 days)">>} + ]} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + -define(ROTATE_SECRET_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"Name">>, ?SECRET_ID}, + {<<"VersionId">>, ?VERSION_ID} + ]). + + rotate_secret_value_output_tests(_) -> + Tests = [?_sm_test( + {"rotate_secret value output test", + jsx:encode(?ROTATE_SECRET_VALUE_RESP), + {ok, ?ROTATE_SECRET_VALUE_RESP}} + )], + + output_tests(?_f(erlcloud_sm:rotate_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN)), Tests), + output_tests(?_f(erlcloud_sm:rotate_secret( + ?SECRET_ID, + ?CLIENT_REQUEST_TOKEN, + [ + {rotation_lambda_arn, <<"arn:aws:lambda:us-west-2:123456789012:function:MySecretRotationFunction">>}, + {rotate_immediately, false}, + {rotation_rules, [ + {automatically_after_days, 30}, + {duration, <<"3h">>}, + {schedule_expression, <<"rate(30 days)">>} + ]} + ] + )), Tests). + + stop_replication_to_replica_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"stop_replication_to_replica input test", + ?_f(erlcloud_sm:stop_replication_to_replica(?SECRET_ID)), + jsx:encode([{<<"SecretId">>, ?SECRET_ID}]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + -define(STOP_REPLICATION_TO_REPLICA_VALUE_RESP,[ + [{<<"ARN">>, <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}] + ]). + + stop_replication_to_replica_value_output_tests(_) -> + Tests = [?_sm_test( + {"stop_replication_to_replica value output test", + jsx:encode(?STOP_REPLICATION_TO_REPLICA_VALUE_RESP), + {ok, ?STOP_REPLICATION_TO_REPLICA_VALUE_RESP}} + )], + output_tests(?_f(erlcloud_sm:stop_replication_to_replica(?SECRET_ID)), Tests). + + tag_resource_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"tag_resource input test", + ?_f(erlcloud_sm:tag_resource(?SECRET_ID, [ + [ {<<"Key">>, <<"Environment">>}, {<<"Value">>, <<"Production">>} ] + ])), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"Tags">>, [ + [ {<<"Key">>, <<"Environment">>}, {<<"Value">>, <<"Production">>} ] + ]} + ]) + }), + ?_sm_test( + {"tag_resource input test", + ?_f(erlcloud_sm:tag_resource(?SECRET_ID, [ + [ {key, <<"Environment">>}, {value, <<"Production">>} ] + ])), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"Tags">>, [ + [ {<<"Key">>, <<"Environment">>}, {<<"Value">>, <<"Production">>} ] + ]} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + tag_resource_value_output_tests(_) -> + Tests = [?_sm_test( + {"tag_resource value output test", + <<>>, + {ok, []}} + )], + output_tests(?_f(erlcloud_sm:tag_resource(?SECRET_ID, [ + [ {<<"Key">>, <<"Environment">>}, {<<"Value">>, <<"Production">>} ] + ])), Tests), + output_tests(?_f(erlcloud_sm:tag_resource(?SECRET_ID, [ + [ {key, <<"Environment">>}, {value, <<"Production">>} ] + ])), Tests). + + untag_resource_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"untag_resource input test", + ?_f(erlcloud_sm:untag_resource(?SECRET_ID, ["Environment"])), + jsx:encode([{<<"SecretId">>, ?SECRET_ID}, {<<"TagKeys">>, ["Environment"]}]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + untag_resource_value_output_tests(_) -> + Tests = [?_sm_test( + {"untag_resource value output test", + <<>>, + {ok, []}} + )], + output_tests(?_f(erlcloud_sm:untag_resource(?SECRET_ID, ["Environment"])), Tests). + + update_secret_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"update_secret input test", + ?_f(erlcloud_sm:update_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, + [ + {secret_string, ?SECRET_STRING}, + {description, <<"Updated secret description">>}, + {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ])), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}, + {<<"SecretString">>,?SECRET_STRING}, + {<<"Description">>, <<"Updated secret description">>}, + {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ]) + }), + ?_sm_test( + {"update_secret input test", + ?_f(erlcloud_sm:update_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, + [ + {secret_binary, ?SECRET_STRING}, + {description, <<"Updated secret description">>}, + {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ])), + jsx:encode([ + {<<"ClientRequestToken">>,?CLIENT_REQUEST_TOKEN}, + {<<"SecretId">>,?SECRET_ID}, + {<<"SecretBinary">>,?SECRET_BINARY}, + {<<"Description">>, <<"Updated secret description">>}, + {<<"KmsKeyId">>, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + -define(UPDATE_SECRET_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"Name">>, ?SECRET_ID}, + {<<"VersionId">>, ?VERSION_ID} + ]). + + update_secret_value_output_tests(_) -> + Tests = [?_sm_test( + {"update_secret value output test", + jsx:encode(?UPDATE_SECRET_VALUE_RESP), + {ok, ?UPDATE_SECRET_VALUE_RESP}} + )], + output_tests(?_f(erlcloud_sm:update_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, + [ + {secret_string, ?SECRET_STRING}, + {description, <<"Updated secret description">>}, + {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ])), Tests), + output_tests(?_f(erlcloud_sm:update_secret(?SECRET_ID, ?CLIENT_REQUEST_TOKEN, + [ + {secret_binary, ?SECRET_STRING}, + {description, <<"Updated secret description">>}, + {kms_key_id, <<"alias/aws/abcd1234-a123-456a-a12b-a123b4cd56ef">>} + ])), Tests). + + update_secret_version_stage_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"update_secret_version_stage input test", + ?_f(erlcloud_sm:update_secret_version_stage( + ?SECRET_ID, + ?VERSION_STAGE + )), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"VersionStage">>, ?VERSION_STAGE} + ]) + }), + ?_sm_test( + {"update_secret_version_stage input test", + ?_f(erlcloud_sm:update_secret_version_stage( + ?SECRET_ID, + ?VERSION_STAGE, + [{remove_from_version_id, ?VERSION_ID2}, {move_to_version_id, ?VERSION_ID}] + )), + jsx:encode([ + {<<"SecretId">>, ?SECRET_ID}, + {<<"VersionStage">>, ?VERSION_STAGE}, + {<<"MoveToVersionId">>, ?VERSION_ID}, + {<<"RemoveFromVersionId">>, ?VERSION_ID2} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + -define(UPDATE_SECRET_VERSION_STAGE_VALUE_RESP,[ + {<<"ARN">>, + <<"arn:aws:secretsmanager:us-west-2:123456789012:secret:MyTestDatabaseSecret-a1b2c3">>}, + {<<"Name">>, ?SECRET_ID} + ]). + + update_secret_version_stage_value_output_tests(_) -> + Tests = [?_sm_test( + {"update_secret_version_stage value output test", + jsx:encode(?UPDATE_SECRET_VERSION_STAGE_VALUE_RESP), + {ok, ?UPDATE_SECRET_VERSION_STAGE_VALUE_RESP}} + )], + output_tests(?_f(erlcloud_sm:update_secret_version_stage( + ?SECRET_ID, + ?VERSION_STAGE + )), Tests), + output_tests(?_f(erlcloud_sm:update_secret_version_stage( + ?SECRET_ID, + ?VERSION_STAGE, + [{remove_from_version_id, ?VERSION_ID2}, {move_to_version_id, ?VERSION_ID}] + )), Tests). + + validate_resource_policy_value_input_tests(_) -> + Tests = [ + ?_sm_test( + {"validate_resource_policy input test", + ?_f(erlcloud_sm:validate_resource_policy(<<"policy-document">>)), + jsx:encode([ + {<<"ResourcePolicy">>, <<"policy-document">>} + ]) + }), + ?_sm_test( + {"validate_resource_policy input test", + ?_f(erlcloud_sm:validate_resource_policy(<<"policy-document">>, [{secret_id, ?SECRET_ID}])), + jsx:encode([ + {<<"ResourcePolicy">>, <<"policy-document">>}, + {<<"SecretId">>, ?SECRET_ID} + ]) + }) + ], + Response = <<>>, + input_tests(Response, Tests). + + -define(VALIDATE_RESOURCE_POLICY_VALUE_RESP,[ + {<<"PolicyValidationPassed">>, true}, + {<<"ValidationErrors">>, []} + ]). + + validate_resource_policy_value_output_tests(_) -> + Tests = [?_sm_test( + {"validate_resource_policy value output test", + jsx:encode(?VALIDATE_RESOURCE_POLICY_VALUE_RESP), + {ok, ?VALIDATE_RESOURCE_POLICY_VALUE_RESP}} + )], + output_tests(?_f(erlcloud_sm:validate_resource_policy(<<"policy-document">>)), Tests), + output_tests(?_f(erlcloud_sm:validate_resource_policy(<<"policy-document">>, [{secret_id, ?SECRET_ID}])), Tests). \ No newline at end of file From 460b15cbd9ac5c1885b60526763fbd372b9d9a8f Mon Sep 17 00:00:00 2001 From: cody-friedrichsen Date: Tue, 28 Oct 2025 09:42:48 -0500 Subject: [PATCH 310/310] Move get_random_password/0 to be near the same named higher arity functions --- src/erlcloud_sm.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/erlcloud_sm.erl b/src/erlcloud_sm.erl index 2ec3ffcb0..bf0150fb0 100644 --- a/src/erlcloud_sm.erl +++ b/src/erlcloud_sm.erl @@ -213,10 +213,6 @@ batch_get_secret_value(FiltersOrSecretIdList, Opts, Config) -> %% @end %%------------------------------------------------------------------------------ --spec get_random_password() -> sm_response(). -get_random_password() -> - get_random_password([]). - -spec cancel_rotate_secret(SecretId :: binary()) -> sm_response(). cancel_rotate_secret(SecretId) -> cancel_rotate_secret(SecretId, erlcloud_aws:default_config()). @@ -447,6 +443,10 @@ describe_secret(SecretId, Config) -> %% @end %%------------------------------------------------------------------------------ +-spec get_random_password() -> sm_response(). +get_random_password() -> + get_random_password([]). + -spec get_random_password(Opts :: get_random_password_options()) -> sm_response(). get_random_password(Opts) -> get_random_password(Opts, erlcloud_aws:default_config()).