From 067aafa7b667d063c898647f0ce164a2b459fc06 Mon Sep 17 00:00:00 2001 From: Micah Warren Date: Tue, 30 Dec 2014 16:37:23 -0600 Subject: [PATCH 01/89] Added test for realtime helper process leak. --- tests/repl_process_leak.erl | 90 +++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tests/repl_process_leak.erl diff --git a/tests/repl_process_leak.erl b/tests/repl_process_leak.erl new file mode 100644 index 000000000..e1a980493 --- /dev/null +++ b/tests/repl_process_leak.erl @@ -0,0 +1,90 @@ +%% @doc The purpose of thie test is to ensure the realtime helpers on both +%% the source and sink sides properly exit when a connection is flakey; ie +%% then there are errors and not out-right closes of the connection. + +-module(repl_process_leak). +-behavior(riak_test). +-export([confirm/0]). +-include_lib("eunit/include/eunit.hrl"). + +-define(SEND_ERROR_INTERVAL, 500). + +confirm() -> + Conf = [ + {riak_repl, [ + {fullsync_on_connect, false}, + {fullsync_interval, disabled} + ]} + ], + + lager:info("deploying 2 nodes"), + Nodes = rt:deploy_nodes(2, Conf, [riak_kv, riak_repl]), + + [SourceNode, SinkNode] = Nodes, + + lager:info("nameing clusters"), + repl_util:name_cluster(SourceNode, "source"), + repl_util:name_cluster(SinkNode, "sink"), + + {ok, {_IP, Port}} = rpc:call(SinkNode, application, get_env, [riak_core, cluster_mgr]), + + lager:info("connecting clusters using port ~p", [Port]), + repl_util:connect_cluster(SourceNode, "127.0.0.1", Port), + repl_util:wait_for_connection(SourceNode, "sink"), + + lager:info("enabling and starting realtime"), + repl_util:enable_realtime(SourceNode, "sink"), + repl_util:start_realtime(SourceNode, "sink"), + + flakey_sink(SourceNode, SinkNode), + + flakey_source(SourceNode, SinkNode), + + fail. + +flakey_sink(_SourceNode, SinkNode) -> + InitialProcCount = rpc:call(SinkNode, erlang, system_info, [process_count]), + send_sink_tcp_errors(SinkNode, 10), + + PostProcCount = rpc:call(SinkNode, erlang, system_info, [process_count]), + + ?assertEqual(InitialProcCount, PostProcCount). + +send_sink_tcp_errors(_SinkNode, 0) -> + ok; + +send_sink_tcp_errors(SinkNode, N) -> + case rpc:call(SinkNode, riak_repl2_rtsink_conn_sup, started, []) of + [] -> + timer:sleep(?SEND_ERROR_INTERVAL), + send_sink_tcp_errors(SinkNode, N); + [P | _] -> + P ! {tcp_error, <<>>, test}, + timer:sleep(?SEND_ERROR_INTERVAL), + send_sink_tcp_errors(SinkNode, N - 1) + end. + +flakey_source(SourceNode, _SinkNode) -> + InitialProcCount = rpc:call(SourceNode, erlang, system_info, [process_count]), + send_source_tcp_errors(SourceNode, 10), + + PostProcCount = rpc:call(SourceNode, erlang, system_info, [process_count]), + + lager:info("initial: ~p; post: ~p", [InitialProcCount, PostProcCount]), + ?assertEqual(InitialProcCount, PostProcCount). + +send_source_tcp_errors(_SourceNode, 0) -> + ok; + +send_source_tcp_errors(SourceNode, N) -> + List = rpc:call(SourceNode, riak_repl2_rtsource_conn_sup, enabled, []), + case proplists:get_value("sink", List) of + undefined -> + timer:sleep(?SEND_ERROR_INTERVAL), + send_source_tcp_errors(SourceNode, N); + Pid -> + Pid ! {tcp_error, <<>>, test}, + timer:sleep(?SEND_ERROR_INTERVAL), + send_source_tcp_errors(SourceNode, N - 1) + end. + From 7539ad53c8a3a4d2670b0ebc179d35955def2f28 Mon Sep 17 00:00:00 2001 From: Micah Warren Date: Thu, 8 Jan 2015 14:36:32 -0600 Subject: [PATCH 02/89] Made test more reliable to check for leaks. rpc or other processes will cause one or two persistent processes to start up during the test, but the number shoould not increase after that. Altered the test to check for massive increase in processes over a larger number of faults rather than an equality of process_count over a relatively small number of faults. --- tests/repl_process_leak.erl | 95 +++++++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 21 deletions(-) diff --git a/tests/repl_process_leak.erl b/tests/repl_process_leak.erl index e1a980493..fed599142 100644 --- a/tests/repl_process_leak.erl +++ b/tests/repl_process_leak.erl @@ -36,55 +36,108 @@ confirm() -> repl_util:enable_realtime(SourceNode, "sink"), repl_util:start_realtime(SourceNode, "sink"), + lager:info("testing for leaks on flakey sink"), flakey_sink(SourceNode, SinkNode), + lager:info("testing for leaks on flakey source"), flakey_source(SourceNode, SinkNode), - fail. + pass. flakey_sink(_SourceNode, SinkNode) -> - InitialProcCount = rpc:call(SinkNode, erlang, system_info, [process_count]), - send_sink_tcp_errors(SinkNode, 10), + InitialCount = rpc:call(SinkNode, erlang, system_info, [process_count]), + ProcCounts = send_sink_tcp_errors(SinkNode, 20, [InitialCount]), - PostProcCount = rpc:call(SinkNode, erlang, system_info, [process_count]), + Smallest = lists:min(ProcCounts), + Biggest = lists:max(ProcCounts), + ?assert(2 =< Biggest - Smallest), + %?assertEqual(InitialProcCount, PostProcCount), + % the process count is increasing, but the helper did die + true. - ?assertEqual(InitialProcCount, PostProcCount). +send_sink_tcp_errors(_SinkNode, 0, Acc) -> + Acc; -send_sink_tcp_errors(_SinkNode, 0) -> - ok; - -send_sink_tcp_errors(SinkNode, N) -> +send_sink_tcp_errors(SinkNode, N, Acc) -> case rpc:call(SinkNode, riak_repl2_rtsink_conn_sup, started, []) of [] -> timer:sleep(?SEND_ERROR_INTERVAL), - send_sink_tcp_errors(SinkNode, N); + send_sink_tcp_errors(SinkNode, N, Acc); [P | _] -> + SysStatus = sys:get_status(P), + {status, P, _Modul, [_PDict, _Status, _, _, Data]} = SysStatus, + [_Header, _Data1, Data2] = Data, + {data, [{"State", StateRec}]} = Data2, + [Helper | _] = lists:filter(fun(E) -> + is_pid(E) + end, tuple_to_list(StateRec)), + HelpMon = erlang:monitor(process, Helper), P ! {tcp_error, <<>>, test}, + Mon = erlang:monitor(process, P), + receive {'DOWN', Mon, process, P, _} -> ok end, + receive + {'DOWN', HelpMon, process, Helper, _} -> + ok + after 10000 -> + throw("helper didn't die") + end, timer:sleep(?SEND_ERROR_INTERVAL), - send_sink_tcp_errors(SinkNode, N - 1) + Procs = rpc:call(SinkNode, erlang, system_info, [process_count]), + send_sink_tcp_errors(SinkNode, N - 1, [Procs | Acc]) end. flakey_source(SourceNode, _SinkNode) -> InitialProcCount = rpc:call(SourceNode, erlang, system_info, [process_count]), - send_source_tcp_errors(SourceNode, 10), - - PostProcCount = rpc:call(SourceNode, erlang, system_info, [process_count]), + ProcCounts = send_source_tcp_errors(SourceNode, 20, [InitialProcCount]), - lager:info("initial: ~p; post: ~p", [InitialProcCount, PostProcCount]), - ?assertEqual(InitialProcCount, PostProcCount). + Biggest = lists:max(ProcCounts), + Smallest = lists:min(ProcCounts), + %lager:info("initial: ~p; post: ~p", [InitialProcCount, PostProcCount]), + %?assertEqual(InitialProcCount, PostProcCount). + ?assert(2 =< Biggest - Smallest), + true. -send_source_tcp_errors(_SourceNode, 0) -> - ok; +send_source_tcp_errors(_SourceNode, 0, Acc) -> + Acc; -send_source_tcp_errors(SourceNode, N) -> +send_source_tcp_errors(SourceNode, N, Acc) -> List = rpc:call(SourceNode, riak_repl2_rtsource_conn_sup, enabled, []), case proplists:get_value("sink", List) of undefined -> timer:sleep(?SEND_ERROR_INTERVAL), - send_source_tcp_errors(SourceNode, N); + send_source_tcp_errors(SourceNode, N, Acc); Pid -> + lager:debug("Get the status"), + SysStatus = try sys:get_status(Pid) of + S -> S + catch + W:Y -> + lager:info("Sys failed due to ~p:~p", [W,Y]), + {status, Pid, undefined, [undefined, undefined, undefined, undefined, [undefined, undefined, {data, [{"State", {Pid}}]}]]} + end, + {status, Pid, _Module, [_PDict, _Status, _, _, Data]} = SysStatus, + [_Header, _Data1, Data2] = Data, + {data, [{"State", StateRec}]} = Data2, + [Helper | _] = lists:filter(fun(E) -> + is_pid(E) + end, tuple_to_list(StateRec)), + lager:debug("mon the hlepr"), + HelperMon = erlang:monitor(process, Helper), + lager:debug("Send the murder"), Pid ! {tcp_error, <<>>, test}, + Mon = erlang:monitor(process, Pid), + lager:debug("Wait for deaths"), + receive + {'DOWN', Mon, process, Pid, _} -> ok + end, + receive + {'DOWN', HelperMon, process, Helper, _} -> + ok + after 10000 -> + throw("Helper didn't die") + end, timer:sleep(?SEND_ERROR_INTERVAL), - send_source_tcp_errors(SourceNode, N - 1) + Count = rpc:call(SourceNode, erlang, system_info, [process_count]), + send_source_tcp_errors(SourceNode, N - 1, [Count | Acc]) end. From 712ea2c4ec6289ee7f834b2b7037c741dc4117dd Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Tue, 14 Apr 2015 15:50:24 +0100 Subject: [PATCH 03/89] Add failing test for github issue riak-727 --- priv/1.4-default-app.config | 345 ++++++++++++++++++++++++++++++++++++ tests/riak727.erl | 96 ++++++++++ 2 files changed, 441 insertions(+) create mode 100644 priv/1.4-default-app.config create mode 100644 tests/riak727.erl diff --git a/priv/1.4-default-app.config b/priv/1.4-default-app.config new file mode 100644 index 000000000..d4e1e050e --- /dev/null +++ b/priv/1.4-default-app.config @@ -0,0 +1,345 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et +[ + %% Riak Client APIs config + {riak_api, [ + %% pb_backlog is the maximum length to which the queue of pending + %% connections may grow. If set, it must be an integer >= 0. + %% By default the value is 5. If you anticipate a huge number of + %% connections being initialised *simultaneously*, set this number + %% higher. + %% {pb_backlog, 64}, + + %% pb is a list of IP addresses and TCP ports that the Riak + %% Protocol Buffers interface will bind. + {pb, [ {"127.0.0.1", 10017 } ]} + ]}, + + %% Riak Core config + {riak_core, [ + %% Default location of ringstate + {ring_state_dir, "./data/ring"}, + + %% Default ring creation size. Make sure it is a power of 2, + %% e.g. 16, 32, 64, 128, 256, 512 etc + %{ring_creation_size, 64}, + + %% http is a list of IP addresses and TCP ports that the Riak + %% HTTP interface will bind. + {http, [ {"127.0.0.1", 10018 } ]}, + + %% https is a list of IP addresses and TCP ports that the Riak + %% HTTPS interface will bind. + %{https, [{ "127.0.0.1", 10018 }]}, + + %% Default cert and key locations for https can be overridden + %% with the ssl config variable, for example: + %{ssl, [ + % {certfile, "./etc/cert.pem"}, + % {keyfile, "./etc/key.pem"} + % ]}, + + %% riak_handoff_port is the TCP port that Riak uses for + %% intra-cluster data handoff. + {handoff_port, 10019 }, + + %% To encrypt riak_core intra-cluster data handoff traffic, + %% uncomment the following line and edit its path to an + %% appropriate certfile and keyfile. (This example uses a + %% single file with both items concatenated together.) + %{handoff_ssl_options, [{certfile, "/tmp/erlserver.pem"}]}, + + %% DTrace support + %% Do not enable 'dtrace_support' unless your Erlang/OTP + %% runtime is compiled to support DTrace. DTrace is + %% available in R15B01 (supported by the Erlang/OTP + %% official source package) and in R14B04 via a custom + %% source repository & branch. + {dtrace_support, false}, + + %% Health Checks + %% If disabled, health checks registered by an application will + %% be ignored. NOTE: this option cannot be changed at runtime. + %% To re-enable, the setting must be changed and the node restarted. + %% NOTE: As of Riak 1.3.2, health checks are deprecated as they + %% may interfere with the new overload protection mechanisms. + %% If there is a good reason to re-enable them, you must uncomment + %% this line and also add an entry in the riak_kv section: + %% {riak_kv, [ ..., {enable_health_checks, true}, ...]} + %% {enable_health_checks, true}, + + %% Platform-specific installation paths (substituted by rebar) + {platform_bin_dir, "./bin"}, + {platform_data_dir, "./data"}, + {platform_etc_dir, "./etc"}, + {platform_lib_dir, "./lib"}, + {platform_log_dir, "./log"} + ]}, + + %% Riak KV config + {riak_kv, [ + %% Storage_backend specifies the Erlang module defining the storage + %% mechanism that will be used on this node. + {storage_backend, riak_kv_bitcask_backend}, + + %% raw_name is the first part of all URLS used by the Riak raw HTTP + %% interface. See riak_web.erl and raw_http_resource.erl for + %% details. + %{raw_name, "riak"}, + + %% Enable active anti-entropy subsystem + optional debug messages: + %% {anti_entropy, {on|off, []}}, + %% {anti_entropy, {on|off, [debug]}}, + {anti_entropy, {on, []}}, + + %% Restrict how fast AAE can build hash trees. Building the tree + %% for a given partition requires a full scan over that partition's + %% data. Once built, trees stay built until they are expired. + %% Config is of the form: + %% {num-builds, per-timespan-in-milliseconds} + %% Default is 1 build per hour. + {anti_entropy_build_limit, {1, 3600000}}, + + %% Determine how often hash trees are expired after being built. + %% Periodically expiring a hash tree ensures the on-disk hash tree + %% data stays consistent with the actual k/v backend data. It also + %% helps Riak identify silent disk failures and bit rot. However, + %% expiration is not needed for normal AAE operation and should be + %% infrequent for performance reasons. The time is specified in + %% milliseconds. The default is 1 week. + {anti_entropy_expire, 604800000}, + + %% Limit how many AAE exchanges/builds can happen concurrently. + {anti_entropy_concurrency, 2}, + + %% The tick determines how often the AAE manager looks for work + %% to do (building/expiring trees, triggering exchanges, etc). + %% The default is every 15 seconds. Lowering this value will + %% speedup the rate that all replicas are synced across the cluster. + %% Increasing the value is not recommended. + {anti_entropy_tick, 15000}, + + %% The directory where AAE hash trees are stored. + {anti_entropy_data_dir, "./data/anti_entropy"}, + + %% The LevelDB options used by AAE to generate the LevelDB-backed + %% on-disk hashtrees. + {anti_entropy_leveldb_opts, [{write_buffer_size, 4194304}, + {max_open_files, 20}]}, + + %% mapred_name is URL used to submit map/reduce requests to Riak. + {mapred_name, "mapred"}, + + %% mapred_2i_pipe indicates whether secondary-index + %% MapReduce inputs are queued in parallel via their own + %% pipe ('true'), or serially via a helper process + %% ('false' or undefined). Set to 'false' or leave + %% undefined during a rolling upgrade from 1.0. + {mapred_2i_pipe, true}, + + %% Each of the following entries control how many Javascript + %% virtual machines are available for executing map, reduce, + %% pre- and post-commit hook functions. + {map_js_vm_count, 8 }, + {reduce_js_vm_count, 6 }, + {hook_js_vm_count, 2 }, + + %% js_max_vm_mem is the maximum amount of memory, in megabytes, + %% allocated to the Javascript VMs. If unset, the default is + %% 8MB. + {js_max_vm_mem, 8}, + + %% js_thread_stack is the maximum amount of thread stack, in megabyes, + %% allocate to the Javascript VMs. If unset, the default is 16MB. + %% NOTE: This is not the same as the C thread stack. + {js_thread_stack, 16}, + + %% js_source_dir should point to a directory containing Javascript + %% source files which will be loaded by Riak when it initializes + %% Javascript VMs. + %{js_source_dir, "/tmp/js_source"}, + + %% http_url_encoding determines how Riak treats URL encoded + %% buckets, keys, and links over the REST API. When set to 'on' + %% Riak always decodes encoded values sent as URLs and Headers. + %% Otherwise, Riak defaults to compatibility mode where links + %% are decoded, but buckets and keys are not. The compatibility + %% mode will be removed in a future release. + {http_url_encoding, on}, + + %% Switch to vnode-based vclocks rather than client ids. This + %% significantly reduces the number of vclock entries. + %% Only set true if *all* nodes in the cluster are upgraded to 1.0 + {vnode_vclocks, true}, + + %% This option toggles compatibility of keylisting with 1.0 + %% and earlier versions. Once a rolling upgrade to a version + %% > 1.0 is completed for a cluster, this should be set to + %% true for better control of memory usage during key listing + %% operations + {listkeys_backpressure, true}, + + %% This option specifies how many of each type of fsm may exist + %% concurrently. This is for overload protection and is a new + %% mechanism that obsoletes 1.3's health checks. Note that this number + %% represents two potential processes, so +P in vm.args should be at + %% least 3X the fsm_limit. + {fsm_limit, 50000}, + + %% Uncomment to make non-paginated results be sorted the + %% same way paginated results are: by term, then key. + %% In Riak 1.4.* before 1.4.4, all results were sorted this way + %% by default, which can adversely affect performance in some cases. + %% Setting this to true emulates that behavior. + %% {secondary_index_sort_default, true}, + + %% object_format controls which binary representation of a riak_object + %% is stored on disk. + %% Current options are: v0, v1. + %% v0: Original erlang:term_to_binary format. Higher space overhead. + %% v1: New format for more compact storage of small values. + {object_format, v1} + ]}, + + %% Riak Search Config + {riak_search, [ + %% To enable Search functionality set this 'true'. + {enabled, false} + ]}, + + %% Merge Index Config + {merge_index, [ + %% The root dir to store search merge_index data + {data_root, "./data/merge_index"}, + + %% Size, in bytes, of the in-memory buffer. When this + %% threshold has been reached the data is transformed + %% into a segment file which resides on disk. + {buffer_rollover_size, 1048576}, + + %% Overtime the segment files need to be compacted. + %% This is the maximum number of segments that will be + %% compacted at once. A lower value will lead to + %% quicker but more frequent compactions. + {max_compact_segments, 20} + ]}, + + %% Bitcask Config + {bitcask, [ + %% Configure how Bitcask writes data to disk. + %% erlang: Erlang's built-in file API + %% nif: Direct calls to the POSIX C API + %% + %% The NIF mode provides higher throughput for certain + %% workloads, but has the potential to negatively impact + %% the Erlang VM, leading to higher worst-case latencies + %% and possible throughput collapse. + {io_mode, erlang}, + + {data_root, "./data/bitcask"} + ]}, + + %% eLevelDB Config + {eleveldb, [ + {data_root, "./data/leveldb"} + ]}, + + %% Lager Config + {lager, [ + %% What handlers to install with what arguments + %% The defaults for the logfiles are to rotate the files when + %% they reach 10Mb or at midnight, whichever comes first, and keep + %% the last 5 rotations. See the lager README for a description of + %% the time rotation format: + %% https://github.com/basho/lager/blob/master/README.org + %% + %% If you wish to disable rotation, you can either set the size to 0 + %% and the rotation time to "", or instead specify a 2-tuple that only + %% consists of {Logfile, Level}. + %% + %% If you wish to have riak log messages to syslog, you can use a handler + %% like this: + %% {lager_syslog_backend, ["riak", daemon, info]}, + %% + {handlers, [ + {lager_console_backend, info}, + {lager_file_backend, [ + {"./log/error.log", error, 10485760, "$D0", 5}, + {"./log/console.log", info, 10485760, "$D0", 5} + ]} + ] }, + + %% Whether to write a crash log, and where. + %% Commented/omitted/undefined means no crash logger. + {crash_log, "./log/crash.log"}, + + %% Maximum size in bytes of events in the crash log - defaults to 65536 + {crash_log_msg_size, 65536}, + + %% Maximum size of the crash log in bytes, before its rotated, set + %% to 0 to disable rotation - default is 0 + {crash_log_size, 10485760}, + + %% What time to rotate the crash log - default is no time + %% rotation. See the lager README for a description of this format: + %% https://github.com/basho/lager/blob/master/README.org + {crash_log_date, "$D0"}, + + %% Number of rotated crash logs to keep, 0 means keep only the + %% current one - default is 0 + {crash_log_count, 5}, + + %% Whether to redirect error_logger messages into lager - defaults to true + {error_logger_redirect, true}, + + %% maximum number of error_logger messages to handle in a second + %% lager 2.0.0 shipped with a limit of 50, which is a little low for riak's startup + {error_logger_hwm, 100} + ]}, + + %% riak_sysmon config + {riak_sysmon, [ + %% To disable forwarding events of a particular type, use a + %% limit of 0. + {process_limit, 30}, + {port_limit, 2}, + + %% Finding reasonable limits for a given workload is a matter + %% of experimentation. + %% NOTE: Enabling the 'gc_ms_limit' monitor (by setting non-zero) + %% can cause performance problems on multi-CPU systems. + {gc_ms_limit, 0}, + {heap_word_limit, 40111000}, + + %% Configure the following items to 'false' to disable logging + %% of that event type. + {busy_port, true}, + {busy_dist_port, true} + ]}, + + %% SASL config + {sasl, [ + {sasl_error_logger, false} + ]}, + + %% riak_control config + {riak_control, [ + %% Set to false to disable the admin panel. + {enabled, false}, + + %% Authentication style used for access to the admin + %% panel. Valid styles are 'userlist' . + {auth, userlist}, + + %% If auth is set to 'userlist' then this is the + %% list of usernames and passwords for access to the + %% admin panel. + {userlist, [{"user", "pass"} + ]}, + + %% The admin panel is broken up into multiple + %% components, each of which is enabled or disabled + %% by one of these settings. + {admin, true} + ]} +]. diff --git a/tests/riak727.erl b/tests/riak727.erl new file mode 100644 index 000000000..679ee2ec9 --- /dev/null +++ b/tests/riak727.erl @@ -0,0 +1,96 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%%% @doc +%%% riak_test for github issue riak 727. +%%% +%%% Running a riak-2.0.x node with a default 1.4.x app.config causes +%%% the default bucket properties for allow_mult=true and +%%% dvv_enabled=true when they should be false. +%%% +%%% @end + +-module(riak727). +-behavior(riak_test). +-export([confirm/0]). +-include_lib("eunit/include/eunit.hrl"). + +-define(HARNESS, (rt_config:get(rt_harness))). +-define(TYPE1, <<"bob">>). +-define(TYPE2, <<"fred">>). +-define(DEF_APP_CONF, "1.4-default-app.config"). +-define(APP_CONF, "app.config"). + +confirm() -> + [Node] = rt:build_cluster([current]), + + verify_default_bucket_props(Node, ?TYPE1), + add_one_four_config(Node), + verify_default_bucket_props(Node, ?TYPE2), + + pass. + +%% @private check that the default bucket props for both <<"default">> +%% typed, and custome typed buckets are as expected. +-spec verify_default_bucket_props(node(), binary()) -> ok | no_return(). +verify_default_bucket_props(Node, Type) -> + rt:create_and_activate_bucket_type(Node, Type, [{nonsense, <<"value">>}]), + + DefProps = get_props(Node, <<"default">>), + TypeProps = get_props(Node, Type), + + lager:info("~p", [DefProps]), + + ?assertEqual(false, proplists:get_value(allow_mult, DefProps)), + ?assertEqual(false, proplists:get_value(dvv_enabled, DefProps)), + ?assertEqual(true, proplists:get_value(allow_mult, TypeProps)), + ?assertEqual(true, proplists:get_value(dvv_enabled, TypeProps)). + +%% @private copy the default 1.4 app.config file to the node under +%% test. +-spec add_one_four_config(node()) -> ok. +add_one_four_config(Node) -> + rt:stop_and_wait(Node), + ok = copy(app_config_file(), node_etc_dir(Node)), + rt:start_and_wait(Node). + +-spec copy(file:filename(), file:filename()) -> ok. +copy(File, DestDir) -> + lager:info("Copying ~p to ~p~n", [File, DestDir]), + {ok, _} = file:copy(File, DestDir), + ok. + +-spec app_config_file() -> file:filename(). +app_config_file() -> + filename:join(rt:priv_dir(), ?DEF_APP_CONF). + +%% @private get the etc config directory for the given node. +-spec node_etc_dir(node()) -> file:filename(). +node_etc_dir(Node) -> + ConfPath = ?HARNESS:get_riak_conf(Node), + filename:join(filename:dirname(ConfPath), ?APP_CONF). + +-spec get_props(node(), binary()) -> proplists:proplist() | no_return(). +get_props(Node, Type) -> + case rpc:call(Node, riak_core_bucket_type, get, [Type]) of + {badrpc, Reason} -> + throw({badrpc, Reason}); + Props -> + Props + end. From c8af6c60561349e9d4b984bfd5de42102bd46fa5 Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Tue, 14 Apr 2015 16:03:20 +0100 Subject: [PATCH 04/89] Clean up comment typo and poorly named function --- tests/riak727.erl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/riak727.erl b/tests/riak727.erl index 679ee2ec9..1b124921e 100644 --- a/tests/riak727.erl +++ b/tests/riak727.erl @@ -47,7 +47,7 @@ confirm() -> pass. %% @private check that the default bucket props for both <<"default">> -%% typed, and custome typed buckets are as expected. +%% typed, and custom typed buckets are as expected. -spec verify_default_bucket_props(node(), binary()) -> ok | no_return(). verify_default_bucket_props(Node, Type) -> rt:create_and_activate_bucket_type(Node, Type, [{nonsense, <<"value">>}]), @@ -55,8 +55,6 @@ verify_default_bucket_props(Node, Type) -> DefProps = get_props(Node, <<"default">>), TypeProps = get_props(Node, Type), - lager:info("~p", [DefProps]), - ?assertEqual(false, proplists:get_value(allow_mult, DefProps)), ?assertEqual(false, proplists:get_value(dvv_enabled, DefProps)), ?assertEqual(true, proplists:get_value(allow_mult, TypeProps)), @@ -67,7 +65,7 @@ verify_default_bucket_props(Node, Type) -> -spec add_one_four_config(node()) -> ok. add_one_four_config(Node) -> rt:stop_and_wait(Node), - ok = copy(app_config_file(), node_etc_dir(Node)), + ok = copy(app_config_file(), node_config_file(Node)), rt:start_and_wait(Node). -spec copy(file:filename(), file:filename()) -> ok. @@ -81,8 +79,8 @@ app_config_file() -> filename:join(rt:priv_dir(), ?DEF_APP_CONF). %% @private get the etc config directory for the given node. --spec node_etc_dir(node()) -> file:filename(). -node_etc_dir(Node) -> +-spec node_config_file(node()) -> file:filename(). +node_config_file(Node) -> ConfPath = ?HARNESS:get_riak_conf(Node), filename:join(filename:dirname(ConfPath), ?APP_CONF). From 88bbcca9ce15fb652c79c15ea71a2e3275dd7aad Mon Sep 17 00:00:00 2001 From: Russell Brown Date: Tue, 14 Apr 2015 16:04:23 +0100 Subject: [PATCH 05/89] Remove extraneous and inaccurate comment --- tests/riak727.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/riak727.erl b/tests/riak727.erl index 1b124921e..3718c449b 100644 --- a/tests/riak727.erl +++ b/tests/riak727.erl @@ -78,7 +78,6 @@ copy(File, DestDir) -> app_config_file() -> filename:join(rt:priv_dir(), ?DEF_APP_CONF). -%% @private get the etc config directory for the given node. -spec node_config_file(node()) -> file:filename(). node_config_file(Node) -> ConfPath = ?HARNESS:get_riak_conf(Node), From 2e7f093532f30b87b290cd2d062ddfb4a4c74c5d Mon Sep 17 00:00:00 2001 From: Sean Cribbs Date: Thu, 7 May 2015 11:42:48 -0500 Subject: [PATCH 06/89] Hopefully fix the disconnection issue in the riak667_mixed test --- tests/riak667_mixed.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/riak667_mixed.erl b/tests/riak667_mixed.erl index 3c8728c95..ca6e57987 100644 --- a/tests/riak667_mixed.erl +++ b/tests/riak667_mixed.erl @@ -83,7 +83,7 @@ confirm() -> %% Create PB connection. Pid2 = rt:pbc(Node2), - riakc_pb_socket:set_options(Pid2, [queue_if_disconnected]), + riakc_pb_socket:set_options(Pid2, [queue_if_disconnected, auto_reconnect]), %% Read value. ?assertMatch({error, <<"Error processing incoming message: error:{badrecord,dict}", _/binary>>}, From 8223c7d961a8824b9c34f2cc13ddc25a21acd0ac Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 1 Jun 2015 12:49:52 -0600 Subject: [PATCH 07/89] Update build versions to 2.1.1 --- bin/rtdev-all.sh | 2 +- bin/rtdev-build-releases.sh | 54 ++++--------------------------------- 2 files changed, 6 insertions(+), 50 deletions(-) diff --git a/bin/rtdev-all.sh b/bin/rtdev-all.sh index 32d9595b7..174d086c4 100755 --- a/bin/rtdev-all.sh +++ b/bin/rtdev-all.sh @@ -4,7 +4,7 @@ ORIGDIR=`pwd` pushd `dirname $0` > /dev/null SCRIPT_DIR=`pwd` popd > /dev/null -: ${CURRENT_OTP:=$HOME/erlang-R16B02} +: ${CURRENT_OTP:="$HOME/erlang-R16B02"} : ${RT_CURRENT_TAG:=""} diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index e170a31d1..61514c6f5 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -109,7 +109,6 @@ build() RUN="env PATH=$ERLROOT/bin:$ERLROOT/lib/erlang/bin:$PATH \ C_INCLUDE_PATH=$ERLROOT/usr/include \ LD_LIBRARY_PATH=$ERLROOT/usr/lib" - fix_riak_1_3 $SRCDIR $TAG "$RUN" echo " - Building stagedevrel in $SRCDIR (this could take a while)" cd $SRCDIR @@ -124,53 +123,10 @@ build() echo " - $SRCDIR built." } -# Riak 1.3 has a few artifacts which need to be updated in order to build -# properly -fix_riak_1_3() -{ - SRCDIR=$1 - TAG="$2" - RUN="$3" - - if [ "`echo $TAG | cut -d . -f1-2`" != "1.3" ]; then - return 0 - fi - echo "- Patching Riak 1.3.x" - cd $SRCDIR - cat < - #include - #include -+#include - #include "leveldb/perf_count.h" - #include "leveldb/status.h" -EOF - cd ../../../../../../.. -} - -build "riak-1.4.10" $R15B01 http://s3.amazonaws.com/downloads.basho.com/riak/1.4/1.4.10/riak-1.4.10.tar.gz -echo -if [ -z "$RT_USE_EE" ]; then - build "riak-1.3.2" $R15B01 1.3.2 -else - build "riak-1.3.4" $R15B01 1.3.4 -fi +build "riak-1.4.12" $R15B01 1.4.12 +build "riak-2.0.2" $R16B02 2.0.2 +build "riak-2.0.4" $R16B02 2.0.4 +build "riak-2.0.5" $R16B02 2.0.5 +build "riak-2.1.1" $R16B02 2.1.1 echo From c7dad9cb93fb03835d3d32996c18a31f2f9591dd Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 1 Jun 2015 14:31:16 -0600 Subject: [PATCH 08/89] Switch to using devrels instead of stagedevrels --- bin/rtdev-build-releases.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index 61514c6f5..4037f644c 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# You need to use this script once to build a set of stagedevrels for prior +# You need to use this script once to build a set of devrels for prior # releases of Riak (for mixed version / upgrade testing). You should # create a directory and then run this script from within that directory. # I have ~/test-releases that I created once, and then re-use for testing. @@ -110,13 +110,13 @@ build() C_INCLUDE_PATH=$ERLROOT/usr/include \ LD_LIBRARY_PATH=$ERLROOT/usr/lib" - echo " - Building stagedevrel in $SRCDIR (this could take a while)" + echo " - Building devrel in $SRCDIR (this could take a while)" cd $SRCDIR - $RUN make all stagedevrel + $RUN make all devrel RES=$? if [ "$RES" -ne 0 ]; then - echo "[ERROR] make stagedevrel failed" + echo "[ERROR] make devrel failed" exit 1 fi cd .. From d868cb5df3795bcfb61922b5df928816213565c0 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 1 Jun 2015 23:23:07 -0600 Subject: [PATCH 09/89] Add in the VERSION file for each version of the built devrel --- bin/rtdev-setup-releases.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index a692e5a21..95d7763ea 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -24,6 +24,7 @@ then echo " - Initializing $RT_DEST_DIR/$vsn" mkdir -p "$RT_DEST_DIR/$vsn" cp -p -P -R "$rel" "$RT_DEST_DIR/$vsn" + (cd "$rel"; VERSION=`git describe --tags`; echo $VERSION > $RT_DEST_DIR/$vsn/VERSION) done else # This is useful when only testing with 'current' From d61060a89f2950c12280d4145e0624ddc833f644 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 2 Jun 2015 09:37:07 -0600 Subject: [PATCH 10/89] Change the VERSION file to have only the dotted version, not the product --- bin/rtdev-setup-releases.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index 95d7763ea..02869d884 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -24,7 +24,8 @@ then echo " - Initializing $RT_DEST_DIR/$vsn" mkdir -p "$RT_DEST_DIR/$vsn" cp -p -P -R "$rel" "$RT_DEST_DIR/$vsn" - (cd "$rel"; VERSION=`git describe --tags`; echo $VERSION > $RT_DEST_DIR/$vsn/VERSION) + # Route out the version (not product) from Git + (cd "$rel"; VERSION=`git describe --tags | cut -d- -f2`; echo $VERSION > $RT_DEST_DIR/$vsn/VERSION) done else # This is useful when only testing with 'current' @@ -41,6 +42,6 @@ git init git config user.name "Riak Test" git config user.email "dev@basho.com" -git add . +git add --ignore-removal . git commit -a -m "riak_test init" > /dev/null echo " - Successfully completed initial git commit of $RT_DEST_DIR" From 5b82111747ab7199c0b4513eaefaec56464afa91 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 2 Jun 2015 10:17:19 -0600 Subject: [PATCH 11/89] Strip the newline at the end of the VERSION file --- bin/rtdev-setup-releases.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index 02869d884..111bf9d7f 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -25,7 +25,7 @@ then mkdir -p "$RT_DEST_DIR/$vsn" cp -p -P -R "$rel" "$RT_DEST_DIR/$vsn" # Route out the version (not product) from Git - (cd "$rel"; VERSION=`git describe --tags | cut -d- -f2`; echo $VERSION > $RT_DEST_DIR/$vsn/VERSION) + (cd "$rel"; VERSION="$(git describe --tags | cut -d- -f2)"; echo -n $VERSION > $RT_DEST_DIR/$vsn/VERSION) done else # This is useful when only testing with 'current' From 06836392cd0085a3b9784c04b5db4e767897c91b Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 2 Jul 2015 10:59:01 -0400 Subject: [PATCH 12/89] Re-write of handle_handoff_write_once to test handle_handoff_command. Still needs some work! --- .../riak_core_handoff_sender_intercepts.erl | 4 + tests/verify_handoff_write_once.erl | 205 ++++++++++++------ 2 files changed, 138 insertions(+), 71 deletions(-) diff --git a/intercepts/riak_core_handoff_sender_intercepts.erl b/intercepts/riak_core_handoff_sender_intercepts.erl index f5fb62ad5..115c96a7b 100644 --- a/intercepts/riak_core_handoff_sender_intercepts.erl +++ b/intercepts/riak_core_handoff_sender_intercepts.erl @@ -6,3 +6,7 @@ delayed_visit_item_3(K, V, Acc) -> timer:sleep(100), ?M:visit_item_orig(K, V, Acc). + +slightly_delayed_visit_item_3(K, V, Acc) -> + timer:sleep(1), + ?M:visit_item_orig(K, V, Acc). diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index c2f52aa5a..8032b607f 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -22,87 +22,150 @@ -export([confirm/0]). -include_lib("eunit/include/eunit.hrl"). -define(BUCKET_TYPE, <<"write_once">>). +-define(BUCKET, {?BUCKET_TYPE, <<"write_once">>}). +-define(RING_SIZE, 8). +-define(NVAL, 1). +-define(CONFIG, + [ + {riak_core, [ + {default_bucket_props, [{n_val, ?NVAL}]}, + {vnode_management_timer, 1000}, + {ring_creation_size, ?RING_SIZE}] + }, + {riak_kv, [ + {anti_entropy_build_limit, {100, 1000}}, + {anti_entropy_concurrency, 100}, + {anti_entropy_tick, 100}, + {anti_entropy, {on, []}}, + {anti_entropy_timeout, 5000} + ]} + ] +). + +-record(state, { + sender, node, k, total = 0 +}). -%% We've got a separate test for capability negotiation and other mechanisms, so the test here is fairly -%% straightforward: get a list of different versions of nodes and join them into a cluster, making sure that -%% each time our data has been replicated: +%% +%% @doc This test is going to set up a 2-node cluster, with a write-once bucket with an nval +%% of 1. We also set up an intercept on the pref-list generation to ensure that all +%% puts will go to exactly one of the nodes in the cluster. (Note: there is some sensitivity +%% in the intercept to the node name, which is not something we parameterize to the intercept. +%% If something fundamental changes in the generation of node names (e.g., via make stagedevrel +%% in riak/_ee, then this test may break). +%% Once the cluster is set up, we write NTestItems to the cluster. +%% confirm() -> - NTestItems = 1000, %% How many test items to write/verify? - NTestNodes = 2, %% How many nodes to spin up for tests? + NTestItems = 1000, + NTestNodes = 2, run_test(NTestItems, NTestNodes), - lager:info("Test verify_handoff passed."), + lager:info("Test verify_handoff_write_once passed."), pass. + run_test(NTestItems, NTestNodes) -> lager:info("Testing handoff (items ~p, encoding: default)", [NTestItems]), + %% + %% Build a 2-node cluster + %% + Cluster = rt:build_cluster(NTestNodes, ?CONFIG), + lager:info("Set up cluster: ~p", [Cluster]), + %% + %% Use the first node to create an immutable bucket (type) + %% + [Node1, Node2] = Cluster, + rt:wait_for_service(Node1, riak_kv), + rt:create_and_activate_bucket_type(Node1, ?BUCKET_TYPE, [{write_once, true}, {n_val, 1}]), + rt:wait_until_bucket_type_status(?BUCKET_TYPE, active, Cluster), + %% + %% Add intercepts: + %% - slow down the handoff mechanism to trigger handloffs while removing nodes + %% - count of the number of handoffs, and + %% - force the preflist to send all puts to partition 0. + %% + make_intercepts_tab(Node1), + %% Insert delay into handoff folding + rt_intercept:add(Node1, {riak_core_handoff_sender, [{{visit_item, 3}, slightly_delayed_visit_item_3}]}), + + rt_intercept:add(Node1, {riak_kv_vnode, [{{handle_handoff_command, 3}, count_handoff_w1c_puts}]}), + rt_intercept:add(Node1, {riak_core_apl, [{{get_apl_ann, 3}, force_dev1}]}), + rt_intercept:add(Node2, {riak_core_apl, [{{get_apl_ann, 3}, force_dev2}]}), + %% + %% Write NTestItems entries to a write_once bucket + %% + [] = rt:systest_write(Node1, 1, NTestItems, ?BUCKET, 1), + lager:info("Wrote ~p entries.", [NTestItems]), + %% + %% Evict Node1, and wait until handoff completes + %% Run some puts asynchronously to trigger handoff command + %% in the vnode. + %% + lager:info("Evicting Node1..."), + Pid = start_proc(Node1, NTestItems), + rt:leave(Node1), + timer:sleep(1001), % let the proc send some data + rt:wait_until_transfers_complete(Cluster), + lager:info("Transfers complete."), + %% + %% + %% + TotalSent = stop_proc(Pid), + lager:info("TotalSent: ~p", [TotalSent]), + [{_, Handoffs}] = rpc:call(Node1, ets, lookup, [intercepts_tab, w1c_put_counter]), + ?assert(Handoffs > 0), + lager:info("Looking Good. We handled ~p handoffs.", [Handoffs]), + %% + %% verify NumTestItems hits through handoff, and that NumTestItems are in the remaining cluster + %% + %rt:stop(Node1), + Results = rt:systest_read(Node2, 1, TotalSent, ?BUCKET, 1), + ?assertEqual(0, length(Results)), + lager:info("Data looks ok; we can retrieve ~p entries from the cluster.", [TotalSent]), + ok. - lager:info("Spinning up test nodes"), - [RootNode | TestNodes] = Nodes = deploy_test_nodes(NTestNodes), - - rt:wait_for_service(RootNode, riak_kv), - - make_intercepts_tab(RootNode), - - %% Insert delay into handoff folding to test the efficacy of the - %% handoff heartbeat addition - [rt_intercept:add(N, {riak_core_handoff_sender, - [{{visit_item, 3}, delayed_visit_item_3}]}) - || N <- Nodes], - - %% Count everytime riak_kv_vnode:handle_overload_command/3 is called with a - %% ts_puts tuple - [rt_intercept:add(N, {riak_kv_vnode, - [{{handle_handoff_command, 3}, count_handoff_w1c_puts}]}) - || N <- Nodes], - - lager:info("Populating root node."), - %% write one object with a bucket type - rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}]), - %% allow cluster metadata some time to propogate - rt:systest_write(RootNode, 1, NTestItems, {?BUCKET_TYPE, <<"bucket">>}, 1), - - %% Test handoff on each node: - lager:info("Testing handoff for cluster."), - lists:foreach(fun(TestNode) -> test_handoff(RootNode, TestNode, NTestItems) end, TestNodes). - -%% See if we get the same data back from our new nodes as we put into the root node: -test_handoff(RootNode, NewNode, NTestItems) -> - - lager:info("Waiting for service on new node."), - rt:wait_for_service(NewNode, riak_kv), - - %% Set the w1c_put counter to 0 - true = rpc:call(RootNode, ets, insert, [intercepts_tab, {w1c_put_counter, 0}]), - - lager:info("Joining new node with cluster."), - rt:join(NewNode, RootNode), - ?assertEqual(ok, rt:wait_until_nodes_ready([RootNode, NewNode])), - spawn(fun() -> - rt:systest_write(RootNode, 1001, 2000, {?BUCKET_TYPE, <<"bucket">>}, 1) - end), - rt:wait_until_no_pending_changes([RootNode, NewNode]), - - %% See if we get the same data back from the joined node that we added to the root node. - %% Note: systest_read() returns /non-matching/ items, so getting nothing back is good: - lager:info("Validating data after handoff:"), - Results2 = rt:systest_read(NewNode, 1, NTestItems, {?BUCKET_TYPE, <<"bucket">>}, 1), - ?assertEqual(0, length(Results2)), - lager:info("Data looks ok."), - [{_, Count}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_put_counter]), - ?assert(Count > 0), - lager:info("Looking Good. We handled ~p write_once puts during handoff.", [Count]). - -deploy_test_nodes(N) -> - Config = [{riak_core, [{default_bucket_props, [{n_val, 1}]}, - {ring_creation_size, 8}, - {handoff_acksync_threshold, 20}, - {handoff_concurrency, 2}, - {handoff_receive_timeout, 2000}]}], - rt:deploy_nodes(N, Config). make_intercepts_tab(Node) -> SupPid = rpc:call(Node, erlang, whereis, [sasl_safe_sup]), - intercepts_tab = rpc:call(Node, ets, new, [intercepts_tab, [named_table, - public, set, {heir, SupPid, {}}]]). + intercepts_tab = rpc:call( + Node, ets, new, + [intercepts_tab, [named_table, public, set, {heir, SupPid, {}}]] + ), + rpc:call(Node, ets, insert, [intercepts_tab, {w1c_put_counter, 0}]). + + + +start_proc(Node, NTestItems) -> + Self = self(), + spawn_link(fun() -> loop(#state{sender=Self, node=Node, k=NTestItems}) end). + + +loop(#state{sender=Sender, node=Node, k=K, total=Total} = State) -> + {Done, NewState} = receive + done -> + {true, Sender ! K} + after 2 -> + { + false, + if Total < 500 -> + case rt:systest_write(Node, K, K+1, ?BUCKET, 1) of + [] -> State#state{k=K+1, total=Total+1}; + _ -> State + end; + true -> + State + end + } + end, + case Done of + true -> lager:info("Asynchronously added ~p entries.", [Total]), ok; + false -> loop(NewState) + end. + + +stop_proc(Pid) -> + Pid ! done, + receive + K -> K + end. \ No newline at end of file From f330f867b2fc206fb46ea62309d79bab27b9f404 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 2 Jul 2015 11:48:20 -0400 Subject: [PATCH 13/89] Reverted to original verify_handoff_write_once test with modifications * Backed out the change to riak_core_handoff_sender_intercepts which had a modified intercept * Modified the handle_command intercept to ensure we don't count the vnode crashes that were happening before the fix * Reverted the test to the original, but check to make sure we can read all the entries, including the ones we added during handoff --- .../riak_core_handoff_sender_intercepts.erl | 4 - intercepts/riak_kv_vnode_intercepts.erl | 3 +- tests/verify_handoff_write_once.erl | 205 ++++++------------ 3 files changed, 73 insertions(+), 139 deletions(-) diff --git a/intercepts/riak_core_handoff_sender_intercepts.erl b/intercepts/riak_core_handoff_sender_intercepts.erl index 115c96a7b..f5fb62ad5 100644 --- a/intercepts/riak_core_handoff_sender_intercepts.erl +++ b/intercepts/riak_core_handoff_sender_intercepts.erl @@ -6,7 +6,3 @@ delayed_visit_item_3(K, V, Acc) -> timer:sleep(100), ?M:visit_item_orig(K, V, Acc). - -slightly_delayed_visit_item_3(K, V, Acc) -> - timer:sleep(1), - ?M:visit_item_orig(K, V, Acc). diff --git a/intercepts/riak_kv_vnode_intercepts.erl b/intercepts/riak_kv_vnode_intercepts.erl index 0075fd10c..a8615339d 100644 --- a/intercepts/riak_kv_vnode_intercepts.erl +++ b/intercepts/riak_kv_vnode_intercepts.erl @@ -39,8 +39,9 @@ slow_handle_coverage(Req, Filter, Sender, State) -> ?M:handle_coverage_orig(Req, Filter, Sender, State). count_handoff_w1c_puts(#riak_kv_w1c_put_req_v1{}=Req, Sender, State) -> + Val = ?M:handle_handoff_command_orig(Req, Sender, State), ets:update_counter(intercepts_tab, w1c_put_counter, 1), - ?M:handle_handoff_command_orig(Req, Sender, State); + Val; count_handoff_w1c_puts(Req, Sender, State) -> ?M:handle_handoff_command_orig(Req, Sender, State). diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 8032b607f..b4c0bbd36 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -22,150 +22,87 @@ -export([confirm/0]). -include_lib("eunit/include/eunit.hrl"). -define(BUCKET_TYPE, <<"write_once">>). --define(BUCKET, {?BUCKET_TYPE, <<"write_once">>}). --define(RING_SIZE, 8). --define(NVAL, 1). --define(CONFIG, - [ - {riak_core, [ - {default_bucket_props, [{n_val, ?NVAL}]}, - {vnode_management_timer, 1000}, - {ring_creation_size, ?RING_SIZE}] - }, - {riak_kv, [ - {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 100}, - {anti_entropy_tick, 100}, - {anti_entropy, {on, []}}, - {anti_entropy_timeout, 5000} - ]} - ] -). - --record(state, { - sender, node, k, total = 0 -}). -%% -%% @doc This test is going to set up a 2-node cluster, with a write-once bucket with an nval -%% of 1. We also set up an intercept on the pref-list generation to ensure that all -%% puts will go to exactly one of the nodes in the cluster. (Note: there is some sensitivity -%% in the intercept to the node name, which is not something we parameterize to the intercept. -%% If something fundamental changes in the generation of node names (e.g., via make stagedevrel -%% in riak/_ee, then this test may break). -%% Once the cluster is set up, we write NTestItems to the cluster. -%% +%% We've got a separate test for capability negotiation and other mechanisms, so the test here is fairly +%% straightforward: get a list of different versions of nodes and join them into a cluster, making sure that +%% each time our data has been replicated: confirm() -> - NTestItems = 1000, - NTestNodes = 2, + NTestItems = 1000, %% How many test items to write/verify? + NTestNodes = 2, %% How many nodes to spin up for tests? run_test(NTestItems, NTestNodes), - lager:info("Test verify_handoff_write_once passed."), + lager:info("Test verify_handoff passed."), pass. - run_test(NTestItems, NTestNodes) -> lager:info("Testing handoff (items ~p, encoding: default)", [NTestItems]), - %% - %% Build a 2-node cluster - %% - Cluster = rt:build_cluster(NTestNodes, ?CONFIG), - lager:info("Set up cluster: ~p", [Cluster]), - %% - %% Use the first node to create an immutable bucket (type) - %% - [Node1, Node2] = Cluster, - rt:wait_for_service(Node1, riak_kv), - rt:create_and_activate_bucket_type(Node1, ?BUCKET_TYPE, [{write_once, true}, {n_val, 1}]), - rt:wait_until_bucket_type_status(?BUCKET_TYPE, active, Cluster), - %% - %% Add intercepts: - %% - slow down the handoff mechanism to trigger handloffs while removing nodes - %% - count of the number of handoffs, and - %% - force the preflist to send all puts to partition 0. - %% - make_intercepts_tab(Node1), - %% Insert delay into handoff folding - rt_intercept:add(Node1, {riak_core_handoff_sender, [{{visit_item, 3}, slightly_delayed_visit_item_3}]}), - - rt_intercept:add(Node1, {riak_kv_vnode, [{{handle_handoff_command, 3}, count_handoff_w1c_puts}]}), - rt_intercept:add(Node1, {riak_core_apl, [{{get_apl_ann, 3}, force_dev1}]}), - rt_intercept:add(Node2, {riak_core_apl, [{{get_apl_ann, 3}, force_dev2}]}), - %% - %% Write NTestItems entries to a write_once bucket - %% - [] = rt:systest_write(Node1, 1, NTestItems, ?BUCKET, 1), - lager:info("Wrote ~p entries.", [NTestItems]), - %% - %% Evict Node1, and wait until handoff completes - %% Run some puts asynchronously to trigger handoff command - %% in the vnode. - %% - lager:info("Evicting Node1..."), - Pid = start_proc(Node1, NTestItems), - rt:leave(Node1), - timer:sleep(1001), % let the proc send some data - rt:wait_until_transfers_complete(Cluster), - lager:info("Transfers complete."), - %% - %% - %% - TotalSent = stop_proc(Pid), - lager:info("TotalSent: ~p", [TotalSent]), - [{_, Handoffs}] = rpc:call(Node1, ets, lookup, [intercepts_tab, w1c_put_counter]), - ?assert(Handoffs > 0), - lager:info("Looking Good. We handled ~p handoffs.", [Handoffs]), - %% - %% verify NumTestItems hits through handoff, and that NumTestItems are in the remaining cluster - %% - %rt:stop(Node1), - Results = rt:systest_read(Node2, 1, TotalSent, ?BUCKET, 1), - ?assertEqual(0, length(Results)), - lager:info("Data looks ok; we can retrieve ~p entries from the cluster.", [TotalSent]), - ok. + lager:info("Spinning up test nodes"), + [RootNode | TestNodes] = Nodes = deploy_test_nodes(NTestNodes), + + rt:wait_for_service(RootNode, riak_kv), + + make_intercepts_tab(RootNode), + + %% Insert delay into handoff folding to test the efficacy of the + %% handoff heartbeat addition + [rt_intercept:add(N, {riak_core_handoff_sender, + [{{visit_item, 3}, delayed_visit_item_3}]}) + || N <- Nodes], + + %% Count everytime riak_kv_vnode:handle_overload_command/3 is called with a + %% ts_puts tuple + [rt_intercept:add(N, {riak_kv_vnode, + [{{handle_handoff_command, 3}, count_handoff_w1c_puts}]}) + || N <- Nodes], + + lager:info("Populating root node."), + %% write one object with a bucket type + rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}]), + %% allow cluster metadata some time to propogate + rt:systest_write(RootNode, 1, NTestItems, {?BUCKET_TYPE, <<"bucket">>}, 1), + + %% Test handoff on each node: + lager:info("Testing handoff for cluster."), + lists:foreach(fun(TestNode) -> test_handoff(RootNode, TestNode, NTestItems) end, TestNodes). + +%% See if we get the same data back from our new nodes as we put into the root node: +test_handoff(RootNode, NewNode, NTestItems) -> + + lager:info("Waiting for service on new node."), + rt:wait_for_service(NewNode, riak_kv), + + %% Set the w1c_put counter to 0 + true = rpc:call(RootNode, ets, insert, [intercepts_tab, {w1c_put_counter, 0}]), + + lager:info("Joining new node with cluster."), + rt:join(NewNode, RootNode), + ?assertEqual(ok, rt:wait_until_nodes_ready([RootNode, NewNode])), + spawn(fun() -> + rt:systest_write(RootNode, NTestItems + 1, NTestItems + 1000, {?BUCKET_TYPE, <<"bucket">>}, 1) + end), + rt:wait_until_no_pending_changes([RootNode, NewNode]), + + %% See if we get the same data back from the joined node that we added to the root node. + %% Note: systest_read() returns /non-matching/ items, so getting nothing back is good: + lager:info("Validating data after handoff:"), + Results2 = rt:systest_read(NewNode, 1, NTestItems + 1000, {?BUCKET_TYPE, <<"bucket">>}, 1), + ?assertEqual(0, length(Results2)), + lager:info("Data looks ok."), + [{_, Count}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_put_counter]), + ?assert(Count > 0), + lager:info("Looking Good. We handled ~p write_once puts during handoff.", [Count]). + +deploy_test_nodes(N) -> + Config = [{riak_core, [{default_bucket_props, [{n_val, 1}]}, + {ring_creation_size, 8}, + {handoff_acksync_threshold, 20}, + {handoff_concurrency, 2}, + {handoff_receive_timeout, 2000}]}], + rt:deploy_nodes(N, Config). make_intercepts_tab(Node) -> SupPid = rpc:call(Node, erlang, whereis, [sasl_safe_sup]), - intercepts_tab = rpc:call( - Node, ets, new, - [intercepts_tab, [named_table, public, set, {heir, SupPid, {}}]] - ), - rpc:call(Node, ets, insert, [intercepts_tab, {w1c_put_counter, 0}]). - - - -start_proc(Node, NTestItems) -> - Self = self(), - spawn_link(fun() -> loop(#state{sender=Self, node=Node, k=NTestItems}) end). - - -loop(#state{sender=Sender, node=Node, k=K, total=Total} = State) -> - {Done, NewState} = receive - done -> - {true, Sender ! K} - after 2 -> - { - false, - if Total < 500 -> - case rt:systest_write(Node, K, K+1, ?BUCKET, 1) of - [] -> State#state{k=K+1, total=Total+1}; - _ -> State - end; - true -> - State - end - } - end, - case Done of - true -> lager:info("Asynchronously added ~p entries.", [Total]), ok; - false -> loop(NewState) - end. - - -stop_proc(Pid) -> - Pid ! done, - receive - K -> K - end. \ No newline at end of file + intercepts_tab = rpc:call(Node, ets, new, [intercepts_tab, [named_table, + public, set, {heir, SupPid, {}}]]). From b68ffbc9bed19f0135938c391ee8ed02c947218a Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 7 Jul 2015 13:01:42 -0400 Subject: [PATCH 14/89] update some yokozuna tests from work done in riak/2.0 branch for 2.0.6 release --- tests/yz_default_bucket_type_upgrade.erl | 12 ++++++++++-- tests/yz_extractors.erl | 9 +++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/yz_default_bucket_type_upgrade.erl b/tests/yz_default_bucket_type_upgrade.erl index 84dc7378d..20d197779 100644 --- a/tests/yz_default_bucket_type_upgrade.erl +++ b/tests/yz_default_bucket_type_upgrade.erl @@ -17,6 +17,13 @@ %% under the License. %% %%-------------------------------------------------------------------- + +%% @doc Test that checks to make sure that default bucket_types +%% do not lose data when expiring/clearing AAE trees when +%% trees are rebuilt for comparison. +%% @end + + -module(yz_default_bucket_type_upgrade). -compile(export_all). -include_lib("eunit/include/eunit.hrl"). @@ -43,8 +50,9 @@ ]). confirm() -> - TestMetaData = riak_test_runner:metadata(), - OldVsn = proplists:get_value(upgrade_version, TestMetaData, previous), + %% This test explicitly requires an upgrade from 2.0.5 to test a + %% new capability + OldVsn = "2.0.5", [_, Node|_] = Cluster = rt:build_cluster(lists:duplicate(4, {OldVsn, ?CFG})), rt:wait_for_cluster_service(Cluster, yokozuna), diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl index 29739eba9..0e1582e86 100644 --- a/tests/yz_extractors.erl +++ b/tests/yz_extractors.erl @@ -18,6 +18,10 @@ %% %%------------------------------------------------------------------- +%% @doc Test that checks if we're caching the extractor map and that +%% creating custom extractors is doable via protobufs. +%% @end + -module(yz_extractors). -compile(export_all). -include_lib("eunit/include/eunit.hrl"). @@ -54,8 +58,9 @@ ]). confirm() -> - TestMetaData = riak_test_runner:metadata(), - OldVsn = proplists:get_value(upgrade_version, TestMetaData, previous), + %% This test explicitly requires an upgrade from 2.0.5 to test a + %% new capability + OldVsn = "2.0.5", [_, Node|_] = Cluster = rt:build_cluster(lists:duplicate(4, {OldVsn, ?CFG})), rt:wait_for_cluster_service(Cluster, yokozuna), From fbedd4cd3392943056cd5d39af03e596f6fe8fda Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Tue, 7 Jul 2015 16:45:05 -0400 Subject: [PATCH 15/89] Tried to make the verify_handoff_write_once test more predictable --- .../riak_core_handoff_sender_intercepts.erl | 9 ++ tests/verify_handoff_write_once.erl | 97 ++++++++++++++----- 2 files changed, 82 insertions(+), 24 deletions(-) diff --git a/intercepts/riak_core_handoff_sender_intercepts.erl b/intercepts/riak_core_handoff_sender_intercepts.erl index f5fb62ad5..ee90f87ab 100644 --- a/intercepts/riak_core_handoff_sender_intercepts.erl +++ b/intercepts/riak_core_handoff_sender_intercepts.erl @@ -6,3 +6,12 @@ delayed_visit_item_3(K, V, Acc) -> timer:sleep(100), ?M:visit_item_orig(K, V, Acc). + +delayed_visit_item_3_1ms(K, V, Acc) -> + timer:sleep(1), + ?M:visit_item_orig(K, V, Acc). + +start_fold_global_send(TargetNode, Module, {Type, Opts}, ParentPid, SslOpts) -> + Return = ?M:start_fold_orig(TargetNode, Module, {Type, Opts}, ParentPid, SslOpts), + catch global:send(start_fold_started_proc, start), + Return. \ No newline at end of file diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index b4c0bbd36..5cabf34ca 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -22,6 +22,7 @@ -export([confirm/0]). -include_lib("eunit/include/eunit.hrl"). -define(BUCKET_TYPE, <<"write_once">>). +-define(BUCKET, {?BUCKET_TYPE, <<"write_once">>}). %% We've got a separate test for capability negotiation and other mechanisms, so the test here is fairly %% straightforward: get a list of different versions of nodes and join them into a cluster, making sure that @@ -39,31 +40,34 @@ run_test(NTestItems, NTestNodes) -> lager:info("Testing handoff (items ~p, encoding: default)", [NTestItems]), lager:info("Spinning up test nodes"), - [RootNode | TestNodes] = Nodes = deploy_test_nodes(NTestNodes), + [RootNode | TestNodes] = deploy_test_nodes(NTestNodes), rt:wait_for_service(RootNode, riak_kv), make_intercepts_tab(RootNode), - %% Insert delay into handoff folding to test the efficacy of the - %% handoff heartbeat addition - [rt_intercept:add(N, {riak_core_handoff_sender, - [{{visit_item, 3}, delayed_visit_item_3}]}) - || N <- Nodes], - - %% Count everytime riak_kv_vnode:handle_overload_command/3 is called with a - %% ts_puts tuple - [rt_intercept:add(N, {riak_kv_vnode, - [{{handle_handoff_command, 3}, count_handoff_w1c_puts}]}) - || N <- Nodes], + %% Insert delay into handoff folding to test the efficacy of the handle_handoff command + rt_intercept:add( + RootNode, {riak_core_handoff_sender, [{{visit_item, 3}, delayed_visit_item_3_1ms}]} + ), + % signal when a fold starts + rt_intercept:add( + RootNode, {riak_core_handoff_sender, [{{start_fold, 5}, start_fold_global_send}]} + ), + %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message + rt_intercept:add( + RootNode, {riak_kv_vnode, [{{handle_handoff_command, 3}, count_handoff_w1c_puts}]} + ), + + %% Force handoff to always continue on the receiver + %[rt_intercept:add( + % Node, {riak_core_vnode, [{{mark_handoff_complete, 5}, mark_handoff_complete_always_continue}]} + %) || Node <- TestNodes], lager:info("Populating root node."), - %% write one object with a bucket type rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}]), - %% allow cluster metadata some time to propogate - rt:systest_write(RootNode, 1, NTestItems, {?BUCKET_TYPE, <<"bucket">>}, 1), + rt:systest_write(RootNode, 1, NTestItems, ?BUCKET, 1), - %% Test handoff on each node: lager:info("Testing handoff for cluster."), lists:foreach(fun(TestNode) -> test_handoff(RootNode, TestNode, NTestItems) end, TestNodes). @@ -77,19 +81,20 @@ test_handoff(RootNode, NewNode, NTestItems) -> true = rpc:call(RootNode, ets, insert, [intercepts_tab, {w1c_put_counter, 0}]), lager:info("Joining new node with cluster."), + start_proc(RootNode, NTestItems), + timer:sleep(1000), rt:join(NewNode, RootNode), + %timer:sleep(1000), ?assertEqual(ok, rt:wait_until_nodes_ready([RootNode, NewNode])), - spawn(fun() -> - rt:systest_write(RootNode, NTestItems + 1, NTestItems + 1000, {?BUCKET_TYPE, <<"bucket">>}, 1) - end), - rt:wait_until_no_pending_changes([RootNode, NewNode]), + rt:wait_until_transfers_complete([RootNode, NewNode]), + TotalSent = stop_proc(), %% See if we get the same data back from the joined node that we added to the root node. %% Note: systest_read() returns /non-matching/ items, so getting nothing back is good: - lager:info("Validating data after handoff:"), - Results2 = rt:systest_read(NewNode, 1, NTestItems + 1000, {?BUCKET_TYPE, <<"bucket">>}, 1), - ?assertEqual(0, length(Results2)), - lager:info("Data looks ok."), + lager:info("Validating data after handoff..."), + Results2 = rt:systest_read(NewNode, 1, TotalSent, ?BUCKET, 1), + ?assertEqual([], Results2), + lager:info("Data looks good. Read ~p entries.", [TotalSent]), [{_, Count}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_put_counter]), ?assert(Count > 0), lager:info("Looking Good. We handled ~p write_once puts during handoff.", [Count]). @@ -106,3 +111,47 @@ make_intercepts_tab(Node) -> SupPid = rpc:call(Node, erlang, whereis, [sasl_safe_sup]), intercepts_tab = rpc:call(Node, ets, new, [intercepts_tab, [named_table, public, set, {heir, SupPid, {}}]]). + + +-record(state, { + state = waiting, node, sender, k, n +}). + +start_proc(Node, NTestItems) -> + Self = self(), + Pid = spawn_link(fun() -> loop(#state{state=running, node=Node, sender=Self, k=NTestItems+1, n=1}) end), + global:register_name(start_fold_started_proc, Pid). + +loop(#state{node=Node, sender=Sender, state=RunningState, k=K, n=N} = State) -> + {Done, NewState} = + receive + start -> + case RunningState of + waiting -> + {false, State#state{state=running}}; + _ -> {false, State} + end; + stop -> + {true, State}; + _ -> {false, State} + after 10 -> + case RunningState of + running -> + %lager:info("Asynchronously sending ~p entries.", [N]), + rt:systest_write(Node, K, K + N, ?BUCKET, 1), + {false, State#state{k=K + N}}; + _ -> + {false, State} + end + end, + case Done of + true -> Sender ! K; + false -> loop(NewState) + end. + + +stop_proc() -> + global:send(start_fold_started_proc, stop), + receive + K -> K + end. From 3fa57b40252cc6e11dbf62521d9947f69cfe7dd7 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Tue, 7 Jul 2015 17:25:49 -0400 Subject: [PATCH 16/89] Added a cap to the number of messages to be send asynchronously, to give handoff a chance to complete. --- tests/verify_handoff_write_once.erl | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 5cabf34ca..de6adc153 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -135,23 +135,25 @@ loop(#state{node=Node, sender=Sender, state=RunningState, k=K, n=N} = State) -> {true, State}; _ -> {false, State} after 10 -> - case RunningState of - running -> - %lager:info("Asynchronously sending ~p entries.", [N]), - rt:systest_write(Node, K, K + N, ?BUCKET, 1), - {false, State#state{k=K + N}}; - _ -> - {false, State} + if K < 10000 -> + case RunningState of + running -> + rt:systest_write(Node, K, K + N, ?BUCKET, 1), + {false, State#state{k=K + N}}; + _ -> + {false, State} + end; + true -> {true, State} end end, case Done of - true -> Sender ! K; + true -> #state{k=K2} = NewState, Sender ! K2; false -> loop(NewState) end. stop_proc() -> - global:send(start_fold_started_proc, stop), + catch global:send(start_fold_started_proc, stop), receive K -> K end. From c2d610a6d83adaa438356db7493dec777d0b3acf Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 16 Jul 2015 17:03:37 -0400 Subject: [PATCH 17/89] Force calls to riak_kv_vnode:handle_handoff_command during a handoff. --- intercepts/riak_kv_worker_intercepts.erl | 26 ++++++++++++++++++++++++ tests/verify_handoff_write_once.erl | 25 ++++++++++------------- 2 files changed, 37 insertions(+), 14 deletions(-) create mode 100644 intercepts/riak_kv_worker_intercepts.erl diff --git a/intercepts/riak_kv_worker_intercepts.erl b/intercepts/riak_kv_worker_intercepts.erl new file mode 100644 index 000000000..306cc280f --- /dev/null +++ b/intercepts/riak_kv_worker_intercepts.erl @@ -0,0 +1,26 @@ +-module(riak_kv_worker_intercepts). +-compile(export_all). +-include("intercept.hrl"). +-define(M, riak_kv_worker_orig). + +% +% Okay, this is an interesting intercept. The intention here is to insert some +% code into the point where the vnode has completed its fold, but before it invokes the finish +% command, which will inform the riak_core_vnode that handoff has completed for this node. +% This is a magic time, when handoff is running, and the fold has completed. +% In this case, we send a message to a process that is running in riak test +% (see verify_handoff_write_once, for an example), which will do a write during this +% magic time. We wait for said process to return us an ok. +% +% The objective is to force the vnode to trigger a handle_handoff_command, +% thus exercising the runtime/forwarding handoff logic in the vnode. +% +handle_work_intercept({fold, FoldFun, FinishFun}, Sender, State) -> + FinishWrapperFun = fun(X) -> + catch global:send(start_fold_started_proc, {write, self()}), + receive + ok -> ok + end, + FinishFun(X) + end, + ?M:handle_work_orig({fold, FoldFun, FinishWrapperFun}, Sender, State). \ No newline at end of file diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index de6adc153..073a6e4c8 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -46,24 +46,16 @@ run_test(NTestItems, NTestNodes) -> make_intercepts_tab(RootNode), - %% Insert delay into handoff folding to test the efficacy of the handle_handoff command + % This intercept will tell the backround process (below) to send an event for each + % vnode that is being handed off (there will be 4 such vnodes, in this test case) rt_intercept:add( - RootNode, {riak_core_handoff_sender, [{{visit_item, 3}, delayed_visit_item_3_1ms}]} - ), - % signal when a fold starts - rt_intercept:add( - RootNode, {riak_core_handoff_sender, [{{start_fold, 5}, start_fold_global_send}]} + RootNode, {riak_kv_worker, [{{handle_work, 3}, handle_work_intercept}]} ), %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message rt_intercept:add( RootNode, {riak_kv_vnode, [{{handle_handoff_command, 3}, count_handoff_w1c_puts}]} ), - %% Force handoff to always continue on the receiver - %[rt_intercept:add( - % Node, {riak_core_vnode, [{{mark_handoff_complete, 5}, mark_handoff_complete_always_continue}]} - %) || Node <- TestNodes], - lager:info("Populating root node."), rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}]), rt:systest_write(RootNode, 1, NTestItems, ?BUCKET, 1), @@ -103,8 +95,9 @@ deploy_test_nodes(N) -> Config = [{riak_core, [{default_bucket_props, [{n_val, 1}]}, {ring_creation_size, 8}, {handoff_acksync_threshold, 20}, - {handoff_concurrency, 2}, - {handoff_receive_timeout, 2000}]}], + {handoff_concurrency, 4}, + {handoff_receive_timeout, 2000}, + {vnode_management_timer, 1000}]}], rt:deploy_nodes(N, Config). make_intercepts_tab(Node) -> @@ -119,7 +112,7 @@ make_intercepts_tab(Node) -> start_proc(Node, NTestItems) -> Self = self(), - Pid = spawn_link(fun() -> loop(#state{state=running, node=Node, sender=Self, k=NTestItems+1, n=1}) end), + Pid = spawn_link(fun() -> loop(#state{state=waiting, node=Node, sender=Self, k=NTestItems+1, n=1}) end), global:register_name(start_fold_started_proc, Pid). loop(#state{node=Node, sender=Sender, state=RunningState, k=K, n=N} = State) -> @@ -133,6 +126,10 @@ loop(#state{node=Node, sender=Sender, state=RunningState, k=K, n=N} = State) -> end; stop -> {true, State}; + {write, Pid} -> + rt:systest_write(Node, K, K + 1, ?BUCKET, 1), + Pid ! ok, + {false, State#state{k=K + 1}}; _ -> {false, State} after 10 -> if K < 10000 -> From 85c79f48af31e6b1287c83737ef45b7ff7b92546 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 16 Jul 2015 17:21:10 -0400 Subject: [PATCH 18/89] Cleaned up the verify_handoff_write_once to run faster (by not waiting for handoffs, just pending changes, as the old test did) --- tests/verify_handoff_write_once.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 073a6e4c8..4ad333a35 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -76,9 +76,8 @@ test_handoff(RootNode, NewNode, NTestItems) -> start_proc(RootNode, NTestItems), timer:sleep(1000), rt:join(NewNode, RootNode), - %timer:sleep(1000), ?assertEqual(ok, rt:wait_until_nodes_ready([RootNode, NewNode])), - rt:wait_until_transfers_complete([RootNode, NewNode]), + rt:wait_until_no_pending_changes([RootNode, NewNode]), TotalSent = stop_proc(), %% See if we get the same data back from the joined node that we added to the root node. @@ -128,6 +127,7 @@ loop(#state{node=Node, sender=Sender, state=RunningState, k=K, n=N} = State) -> {true, State}; {write, Pid} -> rt:systest_write(Node, K, K + 1, ?BUCKET, 1), + lager:info("Asynchronously wrote event ~p during handoff.", [K + 1]), Pid ! ok, {false, State#state{k=K + 1}}; _ -> {false, State} From 90c18d4a63b611d0ed568a1b4b22d2916b322dbc Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 16 Jul 2015 17:25:21 -0400 Subject: [PATCH 19/89] Backed out a false start in the riak_core_handoff_sender intercept --- intercepts/riak_core_handoff_sender_intercepts.erl | 9 --------- 1 file changed, 9 deletions(-) diff --git a/intercepts/riak_core_handoff_sender_intercepts.erl b/intercepts/riak_core_handoff_sender_intercepts.erl index ee90f87ab..f5fb62ad5 100644 --- a/intercepts/riak_core_handoff_sender_intercepts.erl +++ b/intercepts/riak_core_handoff_sender_intercepts.erl @@ -6,12 +6,3 @@ delayed_visit_item_3(K, V, Acc) -> timer:sleep(100), ?M:visit_item_orig(K, V, Acc). - -delayed_visit_item_3_1ms(K, V, Acc) -> - timer:sleep(1), - ?M:visit_item_orig(K, V, Acc). - -start_fold_global_send(TargetNode, Module, {Type, Opts}, ParentPid, SslOpts) -> - Return = ?M:start_fold_orig(TargetNode, Module, {Type, Opts}, ParentPid, SslOpts), - catch global:send(start_fold_started_proc, start), - Return. \ No newline at end of file From e38c0f2533ddcad37ed4fced3f665f6f17734122 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Fri, 17 Jul 2015 09:04:17 -0400 Subject: [PATCH 20/89] Cleaned up async process to remove code that is not exercised by this test. --- tests/verify_handoff_write_once.erl | 49 +++++++++-------------------- 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 4ad333a35..a00e69d9f 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -106,49 +106,28 @@ make_intercepts_tab(Node) -> -record(state, { - state = waiting, node, sender, k, n + node, sender, k }). start_proc(Node, NTestItems) -> Self = self(), - Pid = spawn_link(fun() -> loop(#state{state=waiting, node=Node, sender=Self, k=NTestItems+1, n=1}) end), + Pid = spawn_link(fun() -> loop(#state{node=Node, sender=Self, k=NTestItems}) end), global:register_name(start_fold_started_proc, Pid). -loop(#state{node=Node, sender=Sender, state=RunningState, k=K, n=N} = State) -> - {Done, NewState} = - receive - start -> - case RunningState of - waiting -> - {false, State#state{state=running}}; - _ -> {false, State} - end; - stop -> - {true, State}; - {write, Pid} -> - rt:systest_write(Node, K, K + 1, ?BUCKET, 1), - lager:info("Asynchronously wrote event ~p during handoff.", [K + 1]), - Pid ! ok, - {false, State#state{k=K + 1}}; - _ -> {false, State} - after 10 -> - if K < 10000 -> - case RunningState of - running -> - rt:systest_write(Node, K, K + N, ?BUCKET, 1), - {false, State#state{k=K + N}}; - _ -> - {false, State} - end; - true -> {true, State} - end - end, - case Done of - true -> #state{k=K2} = NewState, Sender ! K2; - false -> loop(NewState) +loop(#state{node=Node, sender=Sender, k=K} = State) -> + receive + stop -> + Sender ! K; + {write, Pid} -> + rt:systest_write(Node, K, K + 1, ?BUCKET, 1), + lager:info("Asynchronously wrote event ~p during handoff.", [K + 1]), + Pid ! ok, + loop(State#state{k=K + 1}); + Msg -> + lager:warning("~p: Unexpected Message: ~p. Ignoring...", [?MODULE, Msg]), + loop(State) end. - stop_proc() -> catch global:send(start_fold_started_proc, stop), receive From 1e39711ff36eb24e32d3b7c8bb6b4c4d0fa6fef1 Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Fri, 17 Jul 2015 16:23:06 -0400 Subject: [PATCH 21/89] Add basic cluster converge test to try to capture issues with cluster convergence outside of other tests. --- tests/verify_cluster_converge.erl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/verify_cluster_converge.erl diff --git a/tests/verify_cluster_converge.erl b/tests/verify_cluster_converge.erl new file mode 100644 index 000000000..6b5931f79 --- /dev/null +++ b/tests/verify_cluster_converge.erl @@ -0,0 +1,15 @@ +-module(verify_cluster_converge). + +-behavior(riak_test). +-export([confirm/0]). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("riakc/include/riakc.hrl"). + +-define(assertDenied(Op), ?assertMatch({error, <<"Permission",_/binary>>}, Op)). + +confirm() -> + lager:info("Deploy & cluster some nodes"), + + _Nodes = rt:build_cluster(4), + pass. \ No newline at end of file From 16ce998e3f448946e0af034c10907ed89e0f794c Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Tue, 2 Jun 2015 15:24:25 -0400 Subject: [PATCH 22/89] * clean-up yokozuna_rt for reusability * add new types for dialyzer * add test to reload schema and overwrite it on type change * test nested search (e.g. nested json objects) * test re-putting fields, include crdt map workaround, discussed in https://gist.github.com/sdebnath/36c235e042cb35db7d1f * increase aae concurrency for core_prop test --- include/yokozuna_rt.hrl | 2 + src/rt.erl | 4 + src/yokozuna_rt.erl | 232 +++++++++++++-- tests/yz_core_properties_create_unload.erl | 29 +- tests/yz_schema_change_reset.erl | 315 +++++++++++++++++++++ 5 files changed, 537 insertions(+), 45 deletions(-) create mode 100644 tests/yz_schema_change_reset.erl diff --git a/include/yokozuna_rt.hrl b/include/yokozuna_rt.hrl index 1d85fb20e..38ca739a3 100644 --- a/include/yokozuna_rt.hrl +++ b/include/yokozuna_rt.hrl @@ -18,4 +18,6 @@ %% %% ------------------------------------------------------------------- -type index_name() :: binary(). +-type schema_name() :: string(). +-type raw_schema() :: binary(). -type bucket() :: bucket() | {bucket(), bucket()}. diff --git a/src/rt.erl b/src/rt.erl index 7a090660b..09d66d796 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -173,9 +173,13 @@ wait_until_bucket_type_status/3, whats_up/0 ]). +-export_type([interfaces/0, + conn_info/0, + predicate/1]). -type strings() :: [string(),...] | []. -type capability() :: atom() | {atom(), tuple()}. +-type predicate(A) :: fun((A) -> boolean()). -define(HARNESS, (rt_config:get(rt_harness))). -define(RT_ETS, rt_ets). -define(RT_ETS_OPTS, [public, named_table, {write_concurrency, true}]). diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl index 8ab45d796..e7a17c123 100644 --- a/src/yokozuna_rt.erl +++ b/src/yokozuna_rt.erl @@ -22,13 +22,36 @@ -include_lib("eunit/include/eunit.hrl"). -include("yokozuna_rt.hrl"). --export([expire_trees/1, +-export([check_exists/2, + expire_trees/1, + host_entries/1, + remove_index_dirs/2, rolling_upgrade/2, rolling_upgrade/3, + search/4, + search/5, + search_expect/5, + search_expect/6, + search_expect/7, verify_num_found_query/3, wait_for_aae/1, wait_for_full_exchange_round/2, - write_data/5]). + wait_for_index/2, + wait_for_schema/2, + wait_for_schema/3, + write_data/5, + write_data/6]). + +-type host() :: string(). +-type portnum() :: integer(). +-type count() :: non_neg_integer(). +-type json_string() :: atom | string() | binary(). + +-define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). + +-spec host_entries(rt:conn_info()) -> [{host(), portnum()}]. +host_entries(ClusterConnInfo) -> + [riak_http(I) || {_,I} <- ClusterConnInfo]. %% @doc Write `Keys' via the PB inteface to a `Bucket' and have them %% searchable in an `Index'. @@ -36,13 +59,23 @@ write_data(Cluster, Pid, Index, Bucket, Keys) -> riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - %% Create a search index and associate with a bucket - riakc_pb_socket:create_search_index(Pid, Index), + create_and_set_index(Cluster, Pid, Index, Bucket), + timer:sleep(1000), - %% For possible legacy upgrade reasons, wrap create index in a wait - wait_for_index(Cluster, Index), + %% Write keys + lager:info("Writing ~p keys", [length(Keys)]), + [ok = rt:pbc_write(Pid, Bucket, Key, Key, "text/plain") || Key <- Keys], + ok. + +-spec write_data([node()], pid(), index_name(), {schema_name(), raw_schema()}, + bucket(), [binary()]) -> ok. +write_data(Cluster, Pid, Index, {SchemaName, SchemaData}, + Bucket, Keys) -> + riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - ok = riakc_pb_socket:set_search_index(Pid, Bucket, Index), + riakc_pb_socket:create_search_schema(Pid, SchemaName, SchemaData), + + create_and_set_index(Cluster, Pid, Bucket, Index, SchemaName), timer:sleep(1000), %% Write keys @@ -79,21 +112,6 @@ rolling_upgrade(Cluster, Vsn, YZCfgChanges) -> end || {SolrPort, Node} <- Cluster2], ok. --spec config_merge(proplists:proplist(), proplists:proplist()) -> - orddict:orddict() | proplists:proplist(). -config_merge(DefaultCfg, NewCfg) when NewCfg /= [] -> - orddict:update(yokozuna, - fun(V) -> - orddict:merge(fun(_, _X, Y) -> Y end, - orddict:from_list(V), - orddict:from_list( - orddict:fetch( - yokozuna, NewCfg))) - end, - DefaultCfg); -config_merge(DefaultCfg, _NewCfg) -> - DefaultCfg. - %% @doc Use AAE status to verify that exchange has occurred for all %% partitions since the time this function was invoked. -spec wait_for_aae([node()]) -> ok. @@ -151,6 +169,38 @@ wait_for_index(Cluster, Index) -> [?assertEqual(ok, rt:wait_until(Node, IsIndexUp)) || Node <- Cluster], ok. +%% @see wait_for_schema/3 +wait_for_schema(Cluster, Name) -> + wait_for_schema(Cluster, Name, ignore). + +%% @doc Wait for the schema `Name' to be read by all nodes in +%% `Cluster' before returning. If `Content' is binary data when +%% verify the schema bytes exactly match `Content'. +-spec wait_for_schema([node()], schema_name(), ignore | raw_schema()) -> ok. +wait_for_schema(Cluster, Name, Content) -> + F = fun(Node) -> + lager:info("Attempt to read schema ~s from node ~p", + [Name, Node]), + {Host, Port} = riak_pb(hd(rt:connection_info([Node]))), + {ok, PBConn} = riakc_pb_socket:start_link(Host, Port), + R = riakc_pb_socket:get_search_schema(PBConn, Name), + riakc_pb_socket:stop(PBConn), + case R of + {ok, PL} -> + case Content of + ignore -> + Name == proplists:get_value(name, PL); + _ -> + (Name == proplists:get_value(name, PL)) and + (Content == proplists:get_value(content, PL)) + end; + _ -> + false + end + end, + rt:wait_until(Cluster, F), + ok. + %% @doc Expire YZ trees -spec expire_trees([node()]) -> ok. expire_trees(Cluster) -> @@ -162,6 +212,27 @@ expire_trees(Cluster) -> timer:sleep(100), ok. +%% @doc Remove index directories, removing the index. +-spec remove_index_dirs([node()], index_name()) -> ok. +remove_index_dirs(Nodes, IndexName) -> + IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || + Node <- Nodes], + lager:info("Remove index dirs: ~p, on nodes: ~p~n", + [IndexDirs, Nodes]), + [rt:stop(ANode) || ANode <- Nodes], + [rt:del_dir(binary_to_list(IndexDir)) || IndexDir <- IndexDirs], + [rt:start(ANode) || ANode <- Nodes], + ok. + +%% @doc Check if index/core exists in metadata, disk via yz_index:exists. +-spec check_exists([node()], index_name()) -> ok. +check_exists(Nodes, IndexName) -> + rt:wait_until(Nodes, + fun(N) -> + rpc:call(N, yz_index, exists, [IndexName]) + end). + +-spec verify_num_found_query([node()], index_name(), count()) -> ok. verify_num_found_query(Cluster, Index, ExpectedCount) -> F = fun(Node) -> Pid = rt:pbc(Node), @@ -172,3 +243,120 @@ verify_num_found_query(Cluster, Index, ExpectedCount) -> end, rt:wait_until(Cluster, F), ok. + +search_expect(HP, Index, Name, Term, Expect) -> + search_expect(yokozuna, HP, Index, Name, Term, Expect). + +search_expect(Type, HP, Index, Name, Term, Expect) -> + {ok, "200", _, R} = search(Type, HP, Index, Name, Term), + verify_count_http(Expect, R). + +search_expect(solr, {Host, Port}, Index, Name0, Term0, Shards, Expect) + when is_list(Shards), length(Shards) > 0 -> + Name = quote_unicode(Name0), + Term = quote_unicode(Term0), + URL = internal_solr_url(Host, Port, Index, Name, Term, Shards), + lager:info("Run search ~s", [URL]), + Opts = [{response_format, binary}], + {ok, "200", _, R} = ibrowse:send_req(URL, [], get, [], Opts), + verify_count_http(Expect, R). + +search(HP, Index, Name, Term) -> + search(yokozuna, HP, Index, Name, Term). + +search(Type, {Host, Port}, Index, Name, Term) when is_integer(Port) -> + search(Type, {Host, integer_to_list(Port)}, Index, Name, Term); + +search(Type, {Host, Port}, Index, Name0, Term0) -> + Name = quote_unicode(Name0), + Term = quote_unicode(Term0), + FmtStr = case Type of + solr -> + "http://~s:~s/internal_solr/~s/select?q=~s:~s&wt=json"; + yokozuna -> + "http://~s:~s/search/query/~s?q=~s:~s&wt=json" + end, + URL = ?FMT(FmtStr, [Host, Port, Index, Name, Term]), + lager:info("Run search ~s", [URL]), + Opts = [{response_format, binary}], + ibrowse:send_req(URL, [], get, [], Opts). + +%%%=================================================================== +%%% Private +%%%=================================================================== + +-spec verify_count_http(count(), json_string()) -> boolean(). +verify_count_http(Expected, Resp) -> + Count = get_count_http(Resp), + lager:info("Expected: ~p, Actual: ~p", [Expected, Count]), + Expected == Count. + +-spec get_count_http(json_string()) -> count(). +get_count_http(Resp) -> + Struct = mochijson2:decode(Resp), + kvc:path([<<"response">>, <<"numFound">>], Struct). + +-spec riak_http({node(), rt:interfaces()} | rt:interfaces()) -> + {host(), portnum()}. +riak_http({_Node, ConnInfo}) -> + riak_http(ConnInfo); +riak_http(ConnInfo) -> + proplists:get_value(http, ConnInfo). + +-spec riak_pb({node(), rt:interfaces()} | rt:interfaces()) -> + {host(), portnum()}. +riak_pb({_Node, ConnInfo}) -> + riak_pb(ConnInfo); +riak_pb(ConnInfo) -> + proplists:get_value(pb, ConnInfo). + +-spec config_merge(proplists:proplist(), proplists:proplist()) -> + orddict:orddict() | proplists:proplist(). +config_merge(DefaultCfg, NewCfg) when NewCfg /= [] -> + orddict:update(yokozuna, + fun(V) -> + orddict:merge(fun(_, _X, Y) -> Y end, + orddict:from_list(V), + orddict:from_list( + orddict:fetch( + yokozuna, NewCfg))) + end, + DefaultCfg); +config_merge(DefaultCfg, _NewCfg) -> + DefaultCfg. + +-spec create_and_set_index([node()], pid(), bucket(), index_name()) -> ok. +create_and_set_index(Cluster, Pid, Bucket, Index) -> + %% Create a search index and associate with a bucket + lager:info("Create a search index ~s and associate it with bucket ~s", + [Index, Bucket]), + ok = riakc_pb_socket:create_search_index(Pid, Index), + %% For possible legacy upgrade reasons, wrap create index in a wait + wait_for_index(Cluster, Index), + set_index(Pid, Bucket, Index). +-spec create_and_set_index([node()], pid(), bucket(), index_name(), + schema_name()) -> ok. +create_and_set_index(Cluster, Pid, Bucket, Index, Schema) -> + %% Create a search index and associate with a bucket + lager:info("Create a search index ~s with a custom schema named ~s and + associate it with bucket ~s", [Index, Schema, Bucket]), + ok = riakc_pb_socket:create_search_index(Pid, Index, Schema, []), + %% For possible legacy upgrade reasons, wrap create index in a wait + wait_for_index(Cluster, Index), + set_index(Pid, Bucket, Index). + +-spec set_index(pid(), bucket(), index_name()) -> ok. +set_index(Pid, Bucket, Index) -> + ok = riakc_pb_socket:set_search_index(Pid, Bucket, Index). + +internal_solr_url(Host, Port, Index) -> + ?FMT("http://~s:~B/internal_solr/~s", [Host, Port, Index]). +internal_solr_url(Host, Port, Index, Name, Term, Shards) -> + Ss = [internal_solr_url(Host, ShardPort, Index) + || {_, ShardPort} <- Shards], + ?FMT("http://~s:~B/internal_solr/~s/select?wt=json&q=~s:~s&shards=~s", + [Host, Port, Index, Name, Term, string:join(Ss, ",")]). + +quote_unicode(Value) -> + mochiweb_util:quote_plus(binary_to_list( + unicode:characters_to_binary(Value))). diff --git a/tests/yz_core_properties_create_unload.erl b/tests/yz_core_properties_create_unload.erl index 50821950b..29a6180d3 100644 --- a/tests/yz_core_properties_create_unload.erl +++ b/tests/yz_core_properties_create_unload.erl @@ -25,7 +25,7 @@ [ %% allow AAE to build trees and exchange rapidly {anti_entropy_build_limit, {100, 1000}}, - {anti_entropy_concurrency, 4} + {anti_entropy_concurrency, 8} ]}, {yokozuna, [ @@ -83,7 +83,7 @@ test_core_props_removal(Cluster, RandNodes, KeyCount, Pid) -> lager:info("Remove core.properties file in each index data dir"), remove_core_props(RandNodes, ?INDEX), - check_exists(Cluster, ?INDEX), + yokozuna_rt:check_exists(Cluster, ?INDEX), lager:info("Write one more piece of data"), ok = rt:pbc_write(Pid, ?BUCKET, <<"foo">>, <<"foo">>, "text/plain"), @@ -93,9 +93,9 @@ test_core_props_removal(Cluster, RandNodes, KeyCount, Pid) -> test_remove_index_dirs(Cluster, RandNodes, KeyCount, Pid) -> lager:info("Remove index directories on each node and let them recreate/reindex"), - remove_index_dirs(RandNodes, ?INDEX), + yokozuna_rt:remove_index_dirs(RandNodes, ?INDEX), - check_exists(Cluster, ?INDEX), + yokozuna_rt:check_exists(Cluster, ?INDEX), yokozuna_rt:expire_trees(Cluster), yokozuna_rt:wait_for_aae(Cluster), @@ -112,9 +112,9 @@ test_remove_segment_infos_and_rebuild(Cluster, RandNodes, KeyCount, Pid) -> lager:info("To fix, we remove index directories on each node and let them recreate/reindex"), - remove_index_dirs(RandNodes, ?INDEX), + yokozuna_rt:remove_index_dirs(RandNodes, ?INDEX), - check_exists(Cluster, ?INDEX), + yokozuna_rt:check_exists(Cluster, ?INDEX), yokozuna_rt:expire_trees(Cluster), yokozuna_rt:wait_for_aae(Cluster), @@ -147,23 +147,6 @@ remove_core_props(Nodes, IndexName) -> [file:delete(PropsFile) || PropsFile <- PropsFiles], ok. -%% @doc Check if index/core exists in metadata, disk via yz_index:exists. -check_exists(Nodes, IndexName) -> - rt:wait_until(Nodes, - fun(N) -> - rpc:call(N, yz_index, exists, [IndexName]) - end). - -%% @doc Remove index directories, removing the index. -remove_index_dirs(Nodes, IndexName) -> - IndexDirs = [rpc:call(Node, yz_index, index_dir, [IndexName]) || - Node <- Nodes], - lager:info("Remove index dirs: ~p, on nodes: ~p~n", - [IndexDirs, Nodes]), - [rt:stop(ANode) || ANode <- Nodes], - [rt:del_dir(binary_to_list(IndexDir)) || IndexDir <- IndexDirs], - [rt:start(ANode) || ANode <- Nodes]. - %% @doc Remove lucence segment info files to check if reindexing will occur %% on re-creation/re-indexing. remove_segment_infos(Nodes, IndexName) -> diff --git a/tests/yz_schema_change_reset.erl b/tests/yz_schema_change_reset.erl new file mode 100644 index 000000000..9980ff69b --- /dev/null +++ b/tests/yz_schema_change_reset.erl @@ -0,0 +1,315 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%-------------------------------------------------------------------- +-module(yz_schema_change_reset). +-compile(export_all). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("riakc/include/riakc.hrl"). + +-define(GET(K,L), proplists:get_value(K, L)). +-define(INDEX, <<"test_schema_change_reset">>). +-define(TYPE, <<"test_schema_change">>). +-define(BUCKET1, <<"test_schema_change_reset">>). +-define(BUCKET2, {?TYPE, <<"test_schema_change_reset_2">>}). +-define(SCHEMANAME, <<"test">>). + +-define(TEST_SCHEMA, +<<" + + + + + + + + + + + + + + + + +_yz_id + + + + + + + + + + + + + + + + + + + + +">>). +-define(TEST_SCHEMA_UPDATE, +<<" + + + + + + + + + + + + + + + + + + + +_yz_id + + + + + + + + + + + + + + + + + + + +">>). + +-define(SEQMAX, 20). +-define(CFG, + [{riak_core, + [ + {ring_creation_size, 16}, + {anti_entropy_build_limit, {100, 1000}}, + {anti_entropy_concurrency, 8} + ]}, + {yokozuna, + [ + {anti_entropy_tick, 1000}, + {enabled, true} + ]} + ]). + +confirm() -> + [Node1|_RestNodes] = Cluster = rt:build_cluster(4, ?CFG), + rt:wait_for_cluster_service(Cluster, yokozuna), + + %% Generate keys, YZ only supports UTF-8 compatible keys + GenKeys = [<> || N <- lists:seq(1, ?SEQMAX), + not lists:any( + fun(E) -> E > 127 end, + binary_to_list(<>))], + KeyCount = length(GenKeys), + lager:info("KeyCount ~p", [KeyCount]), + + Pid = rt:pbc(rt:select_random(Cluster)), + + lager:info("Write initial data to index ~p with schema ~p", + [?INDEX, ?SCHEMANAME]), + + yokozuna_rt:write_data(Cluster, Pid, ?INDEX, + {?SCHEMANAME, ?TEST_SCHEMA}, + ?BUCKET1, GenKeys), + timer:sleep(1100), + + lager:info("Create and activate map-based bucket type ~s and tie it to search_index ~s", + [?TYPE, ?INDEX]), + rt:create_and_activate_bucket_type(Node1, ?TYPE, [{datatype, map}, + {search_index, ?INDEX}]), + + lager:info("Write and check age at integer per original schema"), + + NewObj1A = riakc_obj:new(?BUCKET1, <<"keyA">>, + <<"{\"age\":26}">>, + "application/json"), + + NewObj1B = riakc_obj:new(?BUCKET1, <<"keyB">>, + <<"{\"age\":99}">>, + "application/json"), + + {ok, _ObjA} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), + timer:sleep(1100), + {ok, _ObjB} = riakc_pb_socket:put(Pid, NewObj1B, [return_head]), + timer:sleep(1100), + + yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 2), + + assert_search(Pid, Cluster, <<"age:26">>, {<<"age">>, <<"26">>}, []), + assert_search(Pid, Cluster, <<"age:99">>, {<<"age">>, <<"99">>}, []), + + Map1 = riakc_map:update( + {<<"0_foo">>, register}, + fun(R) -> + riakc_register:set(<<"44ab">>, R) + end, riakc_map:new()), + ok = riakc_pb_socket:update_type( + Pid, + ?BUCKET2, + <<"keyMap1">>, + riakc_map:to_op(Map1)), + + {ok, Map2} = riakc_pb_socket:fetch_type(Pid, ?BUCKET2, <<"keyMap1">>), + Map3 = riakc_map:update( + {<<"1_baz">>, counter}, + fun(R) -> + riakc_counter:increment(10, R) + end, Map2), + ok = riakc_pb_socket:update_type( + Pid, + ?BUCKET2, + <<"keyMap1">>, + riakc_map:to_op(Map3)), + + timer:sleep(1100), + assert_search(Pid, Cluster, <<"0_foo_register:44ab">>, {<<"0_foo_register">>, + <<"44ab">>}, []), + + lager:info("Expire and re-check count before updating schema"), + + yokozuna_rt:expire_trees(Cluster), + yokozuna_rt:wait_for_aae(Cluster), + + yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 3), + + lager:info("Overwrite schema with updated schema"), + override_schema(Pid, Cluster, ?INDEX, ?SCHEMANAME, ?TEST_SCHEMA_UPDATE), + + lager:info("Write and check hello_i at integer per schema update"), + + NewObj2 = riakc_obj:new(?BUCKET1, <<"key2">>, + <<"{\"hello_i\":36}">>, + "application/json"), + + {ok, _Obj2} = riakc_pb_socket:put(Pid, NewObj2, [return_head]), + timer:sleep(1100), + + yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 4), + assert_search(Pid, Cluster, <<"hello_i:36">>, {<<"hello_i">>, <<"36">>}, []), + + lager:info("Write and check age at string per schema update"), + + NewObj3 = riakc_obj:new(?BUCKET1, <<"key3">>, + <<"{\"age\":\"3jlkjkl\"}">>, + "application/json"), + + {ok, _Obj3} = riakc_pb_socket:put(Pid, NewObj3, [return_head]), + + yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), + assert_search(Pid, Cluster, <<"age:3jlkjkl">>, + {<<"age">>, <<"3jlkjkl">>}, []), + + lager:info("Expire and re-check count to make sure we're correctly indexed + by the new schema"), + + yokozuna_rt:expire_trees(Cluster), + yokozuna_rt:wait_for_aae(Cluster), + + yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), + + HP = rt:select_random(yokozuna_rt:host_entries(rt:connection_info(Cluster))), + yokozuna_rt:search_expect(HP, ?INDEX, <<"age">>, <<"*">>, 2), + + lager:info("Re-Put because AAE won't find a diff even though the types + have changed, as it only compares based on bkey currently. + Also, this re-put will work as we have a default bucket (type) + with allow_mult=false... no siblings"), + + {ok, _Obj4} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), + timer:sleep(1100), + + assert_search(Pid, Cluster, <<"age:26">>, {<<"age">>, <<"26">>}, []), + + lager:info("Re-Put Map data by dec/inc counter to account for *change* and + allow previously unindexed counter to be searchable"), + + {ok, Map4} = riakc_pb_socket:fetch_type(Pid, ?BUCKET2, <<"keyMap1">>), + Map5 = riakc_map:update( + {<<"1_baz">>, counter}, + fun(R) -> + riakc_counter:decrement(0, R), + riakc_counter:increment(0, R) + end, Map4), + ok = riakc_pb_socket:update_type( + Pid, + ?BUCKET2, + <<"keyMap1">>, + riakc_map:to_op(Map5)), + + timer:sleep(1100), + assert_search(Pid, Cluster, <<"0_foo_register:44ab">>, {<<"0_foo_register">>, + <<"44ab">>}, []), + assert_search(Pid, Cluster, <<"1_baz_counter:10">>, {<<"1_baz_counter">>, + <<"10">>}, []), + + lager:info("Test nested json searches w/ unsearched fields ignored"), + + NewObj5 = riakc_obj:new(?BUCKET1, <<"key4">>, + <<"{\"quip\":\"blashj3\", + \"paths\":{\"quip\":\"88\"}}">>, + "application/json"), + {ok, _Obj5} = riakc_pb_socket:put(Pid, NewObj5, [return_head]), + + timer:sleep(1100), + assert_search(Pid, Cluster, <<"paths.quip:88">>, + {<<"paths.quip">>, <<"88">>}, []), + + riakc_pb_socket:stop(Pid), + + pass. + +override_schema(Pid, Cluster, Index, Schema, RawUpdate) -> + ok = riakc_pb_socket:create_search_schema(Pid, Schema, RawUpdate), + yokozuna_rt:wait_for_schema(Cluster, Schema, RawUpdate), + [Node|_] = Cluster, + {ok, _} = rpc:call(Node, yz_index, reload, [Index]). + +assert_search(Pid, Cluster, Search, SearchExpect, Params) -> + F = fun(_) -> + lager:info("Searching ~p and asserting it exists", + [SearchExpect]), + {ok,{search_results,[{_Index,Fields}], _Score, Found}} = + riakc_pb_socket:search(Pid, ?INDEX, Search, Params), + ?assert(lists:member(SearchExpect, Fields)), + case Found of + 1 -> true; + 0 -> false + end + end, + rt:wait_until(Cluster, F). From 0ffcc61b97660abd27c8d8e21970dd627ac09e7a Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Mon, 20 Jul 2015 11:10:09 -0400 Subject: [PATCH 23/89] Add verify_snmp_repl riak_test The verify_snmp_repl test reproduces the error condition (one cluster configured for realtime replication with two or more other clusters) and passes if and only if the riak_snmp_stat_poller module correctly handles this configuration. https://bashoeng.atlassian.net/browse/RIAK-1884 https://github.com/basho/riak_snmp/issues/27 --- tests/verify_snmp_repl.erl | 100 +++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 tests/verify_snmp_repl.erl diff --git a/tests/verify_snmp_repl.erl b/tests/verify_snmp_repl.erl new file mode 100644 index 000000000..a4f48d28b --- /dev/null +++ b/tests/verify_snmp_repl.erl @@ -0,0 +1,100 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2010-2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(verify_snmp_repl). +-behavior(riak_test). +-export([confirm/0]). +-include_lib("eunit/include/eunit.hrl"). +-compile({parse_transform, rt_intercept_pt}). + +confirm() -> + Clusters = make_clusters(["cluster-1", "cluster-2", "cluster-3"], 1), + [{_, Leader, _}|_] = Clusters, + intercept_riak_snmp_stat_poller(Leader), + wait_for_snmp_stat_poller(). + +make_clusters(Names, NodeCount) -> + ClusterCount = length(Names), + Config = [{riak_snmp, [{polling_interval, 100}]}], + AllNodes = make_nodes(NodeCount, ClusterCount, Config), + Clusters = lists:zip(Names, AllNodes), + lists:foreach(fun make_named_cluster/1, Clusters), + lists:foreach(fun wait_until_ring_converged/1, Clusters), + lists:foreach(fun wait_until_leader_converge/1, Clusters), + + ClustersWithLeaders = [{Name, repl_util:get_leader(hd(Nodes)), Nodes} || {Name, Nodes} <- Clusters], + enable_realtime(ClustersWithLeaders), + ClustersWithLeaders. + +intercept_riak_snmp_stat_poller(Node) -> + RiakTestProcess = self(), + rt_intercept:add( + Node, + {riak_snmp_stat_poller, + [{{set_rows, 4}, + {[RiakTestProcess], + fun(Table, Indexes, Cols, IndexCol) + when Table =:= replRealtimeStatusTable; Table =:= replFullsyncStatusTable -> + lager:log(info, self(), "JVOEGELE> set_rows(~p, ~p, ~p, ~p)", [Table, Indexes, Cols, IndexCol]), + try + riak_snmp_stat_poller_orig:set_rows_orig(Table, Indexes, Cols, IndexCol), + RiakTestProcess ! pass + catch + Exception:Reason -> + RiakTestProcess ! {fail, {Exception, Reason}}, + error({Exception, Reason}) + end + end}}]}). + +wait_for_snmp_stat_poller() -> + receive + pass -> pass; + {fail, Reason} -> {fail, Reason}; + X -> {fail, {unknown, X}} + after + 1000 -> {fail, timeout} + end. + +make_nodes(NodeCount, ClusterCount, Config) -> + Nodes = rt:deploy_nodes(NodeCount * ClusterCount, Config), + sublists(Nodes, NodeCount). + +sublists(List, Len) -> + lists:map( + fun(I) -> lists:sublist(List, I, Len) end, + lists:seq(1, length(List), Len)). + +make_named_cluster({Name, Nodes}) -> + repl_util:make_cluster(Nodes), + repl_util:name_cluster(hd(Nodes), Name). + +wait_until_ring_converged({_Name, Nodes}) -> + rt:wait_until_ring_converged(Nodes). + +wait_until_leader_converge({_Name, Nodes}) -> + repl_util:wait_until_leader_converge(Nodes). + +enable_realtime([{_, Node, _}|OtherClusters]) -> + lists:foreach( + fun({Cluster, _, _}) -> + repl_util:enable_realtime(Node, Cluster) + end, + OtherClusters). + From 3a7be664cdb9bfe3cf50a76d280e17ec7f317df4 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Mon, 20 Jul 2015 14:44:44 -0400 Subject: [PATCH 24/89] Remove extraneous lager call --- tests/verify_snmp_repl.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/verify_snmp_repl.erl b/tests/verify_snmp_repl.erl index a4f48d28b..71307fd36 100644 --- a/tests/verify_snmp_repl.erl +++ b/tests/verify_snmp_repl.erl @@ -52,7 +52,6 @@ intercept_riak_snmp_stat_poller(Node) -> {[RiakTestProcess], fun(Table, Indexes, Cols, IndexCol) when Table =:= replRealtimeStatusTable; Table =:= replFullsyncStatusTable -> - lager:log(info, self(), "JVOEGELE> set_rows(~p, ~p, ~p, ~p)", [Table, Indexes, Cols, IndexCol]), try riak_snmp_stat_poller_orig:set_rows_orig(Table, Indexes, Cols, IndexCol), RiakTestProcess ! pass From 666f0ca3756dc2ff93c59355c1f331db1f67140d Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Mon, 20 Jul 2015 21:23:22 -0400 Subject: [PATCH 25/89] Added intercepts to count the synchronous and asynchronous calls through handle_command, to ensure we are testing the case statements in riak_kv_vnode:handle_command with write_once puts. --- intercepts/riak_kv_vnode_intercepts.erl | 20 ++++ tests/verify_handoff_write_once.erl | 123 ++++++++++++++---------- 2 files changed, 93 insertions(+), 50 deletions(-) diff --git a/intercepts/riak_kv_vnode_intercepts.erl b/intercepts/riak_kv_vnode_intercepts.erl index a8615339d..ad3d6c893 100644 --- a/intercepts/riak_kv_vnode_intercepts.erl +++ b/intercepts/riak_kv_vnode_intercepts.erl @@ -9,6 +9,10 @@ type :: primary | fallback % start_time :: non_neg_integer(), Jon to add? }). +-record(riak_kv_w1c_put_reply_v1, { + reply :: ok | {error, term()}, + type :: primary | fallback +}). -define(M, riak_kv_vnode_orig). @@ -38,6 +42,7 @@ slow_handle_coverage(Req, Filter, Sender, State) -> timer:sleep(Rand), ?M:handle_coverage_orig(Req, Filter, Sender, State). +%% @doc Count how many times we call handle_handoff_command count_handoff_w1c_puts(#riak_kv_w1c_put_req_v1{}=Req, Sender, State) -> Val = ?M:handle_handoff_command_orig(Req, Sender, State), ets:update_counter(intercepts_tab, w1c_put_counter, 1), @@ -45,6 +50,21 @@ count_handoff_w1c_puts(#riak_kv_w1c_put_req_v1{}=Req, Sender, State) -> count_handoff_w1c_puts(Req, Sender, State) -> ?M:handle_handoff_command_orig(Req, Sender, State). +%% @doc Count how many times we handle syncchronous and asynchronous replies +%% in handle_command when using w1c buckets +count_w1c_handle_command(#riak_kv_w1c_put_req_v1{}=Req, Sender, State) -> + case ?M:handle_command_orig(Req, Sender, State) of + {noreply, NewState} -> + ets:update_counter(intercepts_tab, w1c_async_replies, 1), + {noreply, NewState}; + {reply, #riak_kv_w1c_put_reply_v1{reply=ok, type=Type}, NewState} -> + ets:update_counter(intercepts_tab, w1c_sync_replies, 1), + {reply, #riak_kv_w1c_put_reply_v1{reply=ok, type=Type}, NewState}; + Any -> Any + end; +count_w1c_handle_command(Req, Sender, State) -> + ?M:handle_command_orig(Req, Sender, State). + %% @doc Simulate dropped gets/network partitions byresponding with %% noreply during get requests. drop_do_get(Sender, BKey, ReqId, State) -> diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index a00e69d9f..76031c4eb 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -1,6 +1,6 @@ %% ------------------------------------------------------------------- %% -%% Copyright (c) 2013 Basho Technologies, Inc. +%% Copyright (c) 2015 Basho Technologies, Inc. %% %% This file is provided to you under the Apache License, %% Version 2.0 (the "License"); you may not use this file @@ -24,79 +24,102 @@ -define(BUCKET_TYPE, <<"write_once">>). -define(BUCKET, {?BUCKET_TYPE, <<"write_once">>}). -%% We've got a separate test for capability negotiation and other mechanisms, so the test here is fairly -%% straightforward: get a list of different versions of nodes and join them into a cluster, making sure that -%% each time our data has been replicated: -confirm() -> - NTestItems = 1000, %% How many test items to write/verify? - NTestNodes = 2, %% How many nodes to spin up for tests? - run_test(NTestItems, NTestNodes), +%% @doc +confirm() -> - lager:info("Test verify_handoff passed."), - pass. + AsyncConfig = create_config(riak_kv_eleveldb_backend), + AsyncCluster = run_test(AsyncConfig, true), -run_test(NTestItems, NTestNodes) -> - lager:info("Testing handoff (items ~p, encoding: default)", [NTestItems]), + rt:clean_cluster(AsyncCluster), - lager:info("Spinning up test nodes"), - [RootNode | TestNodes] = deploy_test_nodes(NTestNodes), + SyncConfig = create_config(riak_kv_memory_backend), + _SyncCluster = run_test(SyncConfig, false), - rt:wait_for_service(RootNode, riak_kv), + pass. +create_config(Backend) -> + [{riak_core, [ + {default_bucket_props, [{n_val, 1}]}, + {ring_creation_size, 8}, + {handoff_acksync_threshold, 20}, + {handoff_concurrency, 4}, + {handoff_receive_timeout, 2000}, + {vnode_management_timer, 100}]}, + {riak_kv, [ + {storage_backend, Backend}]} + ]. + +run_test(Config, AsyncWrites) -> + %% + %% Deploy 2 nodes based on config. Wait for K/V to start on each node. + %% + lager:info("Deploying 2 nodes..."), + Cluster = [RootNode, NewNode] = deploy_test_nodes(2, Config), + [rt:wait_for_service(Node, riak_kv) || Node <- [RootNode, NewNode]], + %% + %% Set up the intercepts + %% + lager:info("Setting up intercepts..."), make_intercepts_tab(RootNode), - % This intercept will tell the backround process (below) to send an event for each % vnode that is being handed off (there will be 4 such vnodes, in this test case) rt_intercept:add( RootNode, {riak_kv_worker, [{{handle_work, 3}, handle_work_intercept}]} ), - %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message rt_intercept:add( - RootNode, {riak_kv_vnode, [{{handle_handoff_command, 3}, count_handoff_w1c_puts}]} + RootNode, {riak_kv_vnode, [ + %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message + {{handle_handoff_command, 3}, count_handoff_w1c_puts}, + %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message + {{handle_command, 3}, count_w1c_handle_command} + ]} ), - - lager:info("Populating root node."), - rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}]), - rt:systest_write(RootNode, 1, NTestItems, ?BUCKET, 1), - - lager:info("Testing handoff for cluster."), - lists:foreach(fun(TestNode) -> test_handoff(RootNode, TestNode, NTestItems) end, TestNodes). - -%% See if we get the same data back from our new nodes as we put into the root node: -test_handoff(RootNode, NewNode, NTestItems) -> - - lager:info("Waiting for service on new node."), - rt:wait_for_service(NewNode, riak_kv), - - %% Set the w1c_put counter to 0 + true = rpc:call(RootNode, ets, insert, [intercepts_tab, {w1c_async_replies, 0}]), + true = rpc:call(RootNode, ets, insert, [intercepts_tab, {w1c_sync_replies, 0}]), true = rpc:call(RootNode, ets, insert, [intercepts_tab, {w1c_put_counter, 0}]), - - lager:info("Joining new node with cluster."), + %% + %% Seed the root node with some data + %% + lager:info("Populating root node..."), + rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}, {n_val, 1}]), + NTestItems = 1000, + rt:systest_write(RootNode, 1, NTestItems, ?BUCKET, 1), + %% + %% Start an asynchronous proc which will send puts into riak during handoff. + %% + lager:info("Joining new node with cluster..."), start_proc(RootNode, NTestItems), timer:sleep(1000), rt:join(NewNode, RootNode), - ?assertEqual(ok, rt:wait_until_nodes_ready([RootNode, NewNode])), - rt:wait_until_no_pending_changes([RootNode, NewNode]), + ?assertEqual(ok, rt:wait_until_nodes_ready(Cluster)), + rt:wait_until_no_pending_changes(Cluster), + rt:wait_until_transfers_complete(Cluster), TotalSent = stop_proc(), - - %% See if we get the same data back from the joined node that we added to the root node. - %% Note: systest_read() returns /non-matching/ items, so getting nothing back is good: + %% + %% Verify the results + %% lager:info("Validating data after handoff..."), Results2 = rt:systest_read(NewNode, 1, TotalSent, ?BUCKET, 1), ?assertEqual([], Results2), - lager:info("Data looks good. Read ~p entries.", [TotalSent]), + lager:info("Read ~p entries.", [TotalSent]), [{_, Count}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_put_counter]), ?assert(Count > 0), - lager:info("Looking Good. We handled ~p write_once puts during handoff.", [Count]). - -deploy_test_nodes(N) -> - Config = [{riak_core, [{default_bucket_props, [{n_val, 1}]}, - {ring_creation_size, 8}, - {handoff_acksync_threshold, 20}, - {handoff_concurrency, 4}, - {handoff_receive_timeout, 2000}, - {vnode_management_timer, 1000}]}], + lager:info("We handled ~p write_once puts during handoff.", [Count]), + [{_, W1CAsyncReplies}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_async_replies]), + [{_, W1CSyncReplies}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_sync_replies]), + case AsyncWrites of + true -> + %?assertEqual(1008, W1CAsyncReplies), + ?assertEqual(0, W1CSyncReplies); + false -> + ?assertEqual(0, W1CAsyncReplies) + %?assertEqual(1008, W1CSyncReplies) + end, + %% + Cluster. + +deploy_test_nodes(N, Config) -> rt:deploy_nodes(N, Config). make_intercepts_tab(Node) -> From 9fae776d41603abccc72772f10fa2652c2cf6026 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 21 Jul 2015 13:31:19 -0600 Subject: [PATCH 26/89] Update documented versions to be 1.4, 2.0 and 2.1 --- README.md | 22 +++++++++------------- bin/rtdev-build-releases.sh | 22 ++++++++++++---------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 527b43549..41181151c 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ contents of `$HOME/rt/riak` might look something like this: ``` $ ls $HOME/rt/riak -current riak-1.2.1 riak-1.3.2 riak-1.4.10 +current riak-1.4.12 riak-2.0.2 riak-2.0.4 riak-2.0.6 ``` Inside each of these directories is a `dev` folder, typically @@ -73,7 +73,7 @@ The first one that we want to look at is `rtdev-build-releases.sh`. If left unchanged, this script is going to do the following: 1. Download the source for the past three major Riak versions (e.g. - 1.3.2, 1.4.10 and 2.0.0) + 1.4.12, 2.0.6 and 2.1.2) 1. Build the proper version of Erlang that release was built with, using kerl (which it will also download) 1. Build those releases of Riak. @@ -113,14 +113,6 @@ same directory that you just built all of your releases into. By default this script initializes the repository into `$HOME/rt/riak` but you can override [`$RT_DEST_DIR`](https://github.com/basho/riak_test/blob/master/bin/rtdev-setup-releases.sh#L11). -**Note**: There is a bug in 1.3.x `leveldb` which does not properly resolve -the location of `pthread.h` when building on Macintosh OS X 10.9, aka -Mavericks. This has been fixed in subsequent releases, but for now a fix -is to manually add `#include ` to the top of -`deps/eleveldb/c_src/leveldb/include/leveldb/env.h`. Also the version -of `meck` needs to be updated, too. This is handled autmatically by -the script. - ### rtdev-current.sh `rtdev-current.sh` is where it gets interesting. You need to run that @@ -182,8 +174,10 @@ to tell riak_test about them. The method of choice is to create a {rt_project, "riak"}, {rtdev_path, [{root, "/home/you/rt/riak"}, {current, "/home/you/rt/riak/current"}, - {previous, "/home/you/rt/riak/riak-1.4.10"}, - {legacy, "/home/you/rt/riak/riak-1.3.2"} + {previous, "/home/you/rt/riak/riak-2.0.6"}, + {legacy, "/home/you/rt/riak/riak-1.4.12"} + {'2.0.2', "/home/you/rt/riak/riak-2.0.2"} + {'2.0.4', "/home/you/rt/riak/riak-2.0.4"} ]} ]}. ``` @@ -228,7 +222,9 @@ Tests that do not include coverage annotations will, if cover is enabled, honor #### Web hooks When reporting is enabled, each test result is posted to [Giddy Up](http://giddyup.basho.com). You can specify any number of webhooks that will also receive a POST request with JSON formatted test information, plus the URL -of the Giddy Up resource page. +of the Giddy Up resource page. + +N.B.: This configuration setting is optional, and *NOT* required any more for GiddyUp. ```erlang {webhooks, [ diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index 4037f644c..e205a6978 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# You need to use this script once to build a set of devrels for prior +# You need to use this script once to build a set of stagedevrels for prior # releases of Riak (for mixed version / upgrade testing). You should # create a directory and then run this script from within that directory. # I have ~/test-releases that I created once, and then re-use for testing. @@ -110,23 +110,25 @@ build() C_INCLUDE_PATH=$ERLROOT/usr/include \ LD_LIBRARY_PATH=$ERLROOT/usr/lib" - echo " - Building devrel in $SRCDIR (this could take a while)" + echo " - Building stagedevrel in $SRCDIR (this could take a while)" cd $SRCDIR - $RUN make all devrel + $RUN make locked-deps + $RUN make all stagedevrel RES=$? if [ "$RES" -ne 0 ]; then - echo "[ERROR] make devrel failed" + echo "[ERROR] make stagedevrel failed" exit 1 fi cd .. echo " - $SRCDIR built." } - -build "riak-1.4.12" $R15B01 1.4.12 -build "riak-2.0.2" $R16B02 2.0.2 -build "riak-2.0.4" $R16B02 2.0.4 -build "riak-2.0.5" $R16B02 2.0.5 -build "riak-2.1.1" $R16B02 2.1.1 +build riak-1.4.12 $R15B01 1.4.12 +echo +build riak-2.0.2 $R16B02 2.0.2 +echo +build riak-2.0.4 $R16B02 2.0.4 +echo +build riak-2.0.6 $R16B02 2.0.6 echo From e8a92c4c28809897611408ecd9cdb750541138b0 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 21 Jul 2015 14:45:43 -0600 Subject: [PATCH 27/89] Back out addition of --ignore-removal from git add command --- bin/rtdev-setup-releases.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index 111bf9d7f..2163f64d7 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -42,6 +42,6 @@ git init git config user.name "Riak Test" git config user.email "dev@basho.com" -git add --ignore-removal . +git add . git commit -a -m "riak_test init" > /dev/null echo " - Successfully completed initial git commit of $RT_DEST_DIR" From a418bae347e63bfbe9c9d559894d7a8115dbe4a3 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 23 Jul 2015 12:25:35 -0400 Subject: [PATCH 28/89] Write-Once test improvements * Made tests of handoff conditions more strict than they were before * Added an intercept to the async write_once test to test that asynchronous writes actually happened * Added a test case for the riak_kv (advanced) allow_async_put configuration variable. --- tests/verify_handoff_write_once.erl | 10 +++-- tests/verify_write_once.erl | 62 +++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 76031c4eb..7ce27a3b2 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -84,6 +84,7 @@ run_test(Config, AsyncWrites) -> lager:info("Populating root node..."), rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}, {n_val, 1}]), NTestItems = 1000, + RingSize = proplists:get_value(ring_creation_size, proplists:get_value(riak_core, Config)), rt:systest_write(RootNode, 1, NTestItems, ?BUCKET, 1), %% %% Start an asynchronous proc which will send puts into riak during handoff. @@ -104,17 +105,18 @@ run_test(Config, AsyncWrites) -> ?assertEqual([], Results2), lager:info("Read ~p entries.", [TotalSent]), [{_, Count}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_put_counter]), - ?assert(Count > 0), + ?assertEqual(RingSize div 2, Count), lager:info("We handled ~p write_once puts during handoff.", [Count]), [{_, W1CAsyncReplies}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_async_replies]), [{_, W1CSyncReplies}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_sync_replies]), + %% NB. We should expect RingSize replies, because 4 handoffs should run in both directions case AsyncWrites of true -> - %?assertEqual(1008, W1CAsyncReplies), + ?assertEqual(NTestItems + RingSize, W1CAsyncReplies), ?assertEqual(0, W1CSyncReplies); false -> - ?assertEqual(0, W1CAsyncReplies) - %?assertEqual(1008, W1CSyncReplies) + ?assertEqual(0, W1CAsyncReplies), + ?assertEqual(NTestItems + RingSize, W1CSyncReplies) end, %% Cluster. diff --git a/tests/verify_write_once.erl b/tests/verify_write_once.erl index 673c1adc4..02fcf41d4 100644 --- a/tests/verify_write_once.erl +++ b/tests/verify_write_once.erl @@ -174,11 +174,68 @@ confirm_rww(Nodes) -> %% async puts as a capability which can be arbitrated through the multi backend. %% confirm_async_put(Node) -> + %% + %% Set up the intercepts on the singleton node in cluster2 + %% + make_intercepts_tab(Node), + rt_intercept:add( + Node, {riak_kv_vnode, [ + %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message + {{handle_command, 3}, count_w1c_handle_command} + ]} + ), + %% + %% Create the bucket type + %% rt:create_and_activate_bucket_type(Node, ?ASYNC_PUT_BUCKET_TYPE, [{write_once, true}, {backend, myeleveldb}]), rt:wait_until_bucket_type_status(?ASYNC_PUT_BUCKET_TYPE, active, [Node]), lager:info("Created ~p bucket type on ~p", [?ASYNC_PUT_BUCKET_TYPE, Node]), + %% + %% Clear the intercept counters + %% + true = rpc:call(Node, ets, insert, [intercepts_tab, {w1c_async_replies, 0}]), + true = rpc:call(Node, ets, insert, [intercepts_tab, {w1c_sync_replies, 0}]), ok = verify_put(Node, ?ASYNC_PUT_BUCKET, <<"confirm_async_put_key">>, <<"confirm_async_put_value">>), + %% + %% verify that we have handled 3 asynchronous writes and 0 synchronous writes + %% + [{_, W1CAsyncReplies}] = rpc:call(Node, ets, lookup, [intercepts_tab, w1c_async_replies]), + [{_, W1CSyncReplies}] = rpc:call(Node, ets, lookup, [intercepts_tab, w1c_sync_replies]), + ?assertEqual(0, W1CSyncReplies), + ?assertEqual(3, W1CAsyncReplies), + %% + %% reconfigure the node to force use of synchronous writes with leveldb + %% + rt:update_app_config(Node, [{riak_kv, [{allow_async_put, false}]}]), + rt:start(Node), + %% + %% Set up the intercepts on the singleton node in cluster2 + %% + make_intercepts_tab(Node), + rt_intercept:add( + Node, {riak_kv_vnode, [ + %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message + {{handle_command, 3}, count_w1c_handle_command} + ]} + ), + %% + %% Clear the intercept counters + %% + true = rpc:call(Node, ets, insert, [intercepts_tab, {w1c_async_replies, 0}]), + true = rpc:call(Node, ets, insert, [intercepts_tab, {w1c_sync_replies, 0}]), + + ok = verify_put(Node, ?ASYNC_PUT_BUCKET, <<"confirm_async_put_key">>, <<"confirm_async_put_value">>), + %% + %% verify that we have handled 0 asynchronous writes and 3 synchronous writes, instead + %% + [{_, W1CAsyncReplies2}] = rpc:call(Node, ets, lookup, [intercepts_tab, w1c_async_replies]), + [{_, W1CSyncReplies2}] = rpc:call(Node, ets, lookup, [intercepts_tab, w1c_sync_replies]), + ?assertEqual(3, W1CSyncReplies2), + ?assertEqual(0, W1CAsyncReplies2), + %% + %% done! + %% lager:info("confirm_async_put...ok"), pass. @@ -269,3 +326,8 @@ parse(Binary) -> {ok, Tokens, _} = erl_scan:string(binary_to_list(Binary) ++ "."), {ok, Term} = erl_parse:parse_term(Tokens), Term. + +make_intercepts_tab(Node) -> + SupPid = rpc:call(Node, erlang, whereis, [sasl_safe_sup]), + intercepts_tab = rpc:call(Node, ets, new, [intercepts_tab, [named_table, + public, set, {heir, SupPid, {}}]]). From 399759c33032231ea9c43f264396900fd997457b Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 23 Jul 2015 14:18:25 -0400 Subject: [PATCH 29/89] Added some documentation to the test. Also changed the name of the internal identifier for the globally registered bg process, to avoid potential name collisions in the future. --- intercepts/riak_kv_worker_intercepts.erl | 2 +- tests/verify_handoff_write_once.erl | 38 ++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/intercepts/riak_kv_worker_intercepts.erl b/intercepts/riak_kv_worker_intercepts.erl index 306cc280f..2bd1e8d59 100644 --- a/intercepts/riak_kv_worker_intercepts.erl +++ b/intercepts/riak_kv_worker_intercepts.erl @@ -17,7 +17,7 @@ % handle_work_intercept({fold, FoldFun, FinishFun}, Sender, State) -> FinishWrapperFun = fun(X) -> - catch global:send(start_fold_started_proc, {write, self()}), + catch global:send(rt_ho_w1c_proc, {write, self()}), receive ok -> ok end, diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 7ce27a3b2..7e3d16301 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -25,7 +25,27 @@ -define(BUCKET, {?BUCKET_TYPE, <<"write_once">>}). -%% @doc +%% @doc This test will run a handoff in the case of write_once buckets, verifying +%% that write-once entries are properly handed off as part of ownership handoff, +%% but more importantly, that riak_kv_vnode properly handles data being written into +%% riak while ownership handoff is taking place. +%% +%% This test will create two nodes each with a ring size of 8, and populate one node +%% with 1k entries. It will then join the two nodes to make a cluster of size 2, which +%% will result in ownership handoff of four of the nodes (in each direction). +%% +%% We have intercepted the riak_kv_worker, which handles handoff for an individual vnode, +%% to ensure what we can send data through Riak while the cluster is in the handoff state, +%% thus ensuring that the riak_kv_vnode:handle_handoff_command callback is exercised in +%% the case of write_once buckets. +%% +%% We install intercepts at key points in the vnode to measure how many time various key +%% parts of the code are called. +%% +%% We run the above test twice, once in the case where we are doing asynchronous writes on the +%% back end, and once when we are using synchronous writes. Currently, this is toggled via +%% the use of a back end that can support async writes (currently, only leveldb) +%% confirm() -> AsyncConfig = create_config(riak_kv_eleveldb_backend), @@ -130,6 +150,20 @@ make_intercepts_tab(Node) -> public, set, {heir, SupPid, {}}]]). +%% +%% Notes on the background process and corresponding intercepts. +%% +%% The code below is used to spawn a background process that is globally +%% registered with the name rt_ho_w1c_proc. This process will +%% wait for a message from the riak_kv_worker handle_work intercept, +%% telling this proc to write a message into Riak. The timing of the +%% intercept is such that the write is guaranteed to take place while +%% handoff is in progress, but before the vnode has been told to finish. +%% Sending this message will trigger this background process to do a +%% write into Riak, which in turn will force the vnode's +%% handle_handoff_command to be called. +%% + -record(state, { node, sender, k }). @@ -137,7 +171,7 @@ make_intercepts_tab(Node) -> start_proc(Node, NTestItems) -> Self = self(), Pid = spawn_link(fun() -> loop(#state{node=Node, sender=Self, k=NTestItems}) end), - global:register_name(start_fold_started_proc, Pid). + global:register_name(rt_ho_w1c_proc, Pid). loop(#state{node=Node, sender=Sender, k=K} = State) -> receive From c8d3138cfcad7c393aa10681aab9e5ad4f090982 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 23 Jul 2015 14:22:12 -0400 Subject: [PATCH 30/89] Missed a change in previous commit --- tests/verify_handoff_write_once.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 7e3d16301..a7a6e7383 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -188,7 +188,7 @@ loop(#state{node=Node, sender=Sender, k=K} = State) -> end. stop_proc() -> - catch global:send(start_fold_started_proc, stop), + catch global:send(rt_ho_w1c_proc, stop), receive K -> K end. From 6f979d69ec5c79794e4544d94a61380fbaa66ac9 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 23 Jul 2015 17:47:30 -0400 Subject: [PATCH 31/89] Mode a change in the protocol between vnodes and our bg proc in the test to keep vnodes in the handoff state until all writes have completed. Plus some miscellaneous tightening of test conditions. --- tests/verify_handoff_write_once.erl | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index a7a6e7383..0fa9ccb04 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -105,12 +105,12 @@ run_test(Config, AsyncWrites) -> rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}, {n_val, 1}]), NTestItems = 1000, RingSize = proplists:get_value(ring_creation_size, proplists:get_value(riak_core, Config)), - rt:systest_write(RootNode, 1, NTestItems, ?BUCKET, 1), + [] = rt:systest_write(RootNode, 1, NTestItems, ?BUCKET, 1), %% %% Start an asynchronous proc which will send puts into riak during handoff. %% lager:info("Joining new node with cluster..."), - start_proc(RootNode, NTestItems), + start_proc(RootNode, NTestItems, RingSize div 2), timer:sleep(1000), rt:join(NewNode, RootNode), ?assertEqual(ok, rt:wait_until_nodes_ready(Cluster)), @@ -165,23 +165,30 @@ make_intercepts_tab(Node) -> %% -record(state, { - node, sender, k + node, sender, k, pids=[], expected }). -start_proc(Node, NTestItems) -> +start_proc(Node, NTestItems, Expected) -> Self = self(), - Pid = spawn_link(fun() -> loop(#state{node=Node, sender=Self, k=NTestItems}) end), + Pid = spawn_link(fun() -> loop(#state{node=Node, sender=Self, k=NTestItems, expected=NTestItems+Expected}) end), global:register_name(rt_ho_w1c_proc, Pid). -loop(#state{node=Node, sender=Sender, k=K} = State) -> +loop(#state{node=Node, sender=Sender, k=K, pids=Pids, expected=Expected} = State) -> receive stop -> Sender ! K; {write, Pid} -> - rt:systest_write(Node, K, K + 1, ?BUCKET, 1), + ThePids = [Pid | Pids], + NumWritten = K + 1, + [] = rt:systest_write(Node, K, NumWritten, ?BUCKET, 1), lager:info("Asynchronously wrote event ~p during handoff.", [K + 1]), - Pid ! ok, - loop(State#state{k=K + 1}); + case NumWritten of + Expected -> + lager:info("Handoff may now complete. Sending ok's back to ~p vnodes", [length(ThePids)]), + [ThePid ! ok || ThePid <- ThePids]; + _ -> ok + end, + loop(State#state{k=NumWritten, pids=ThePids}); Msg -> lager:warning("~p: Unexpected Message: ~p. Ignoring...", [?MODULE, Msg]), loop(State) From 2ea13ff15f54c7fbdcb5eb476397dc5087948ca1 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 24 Jul 2015 08:58:13 -0600 Subject: [PATCH 32/89] Add product name back to VERSION file --- bin/rtdev-setup-releases.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index 2163f64d7..f47c25159 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -24,8 +24,8 @@ then echo " - Initializing $RT_DEST_DIR/$vsn" mkdir -p "$RT_DEST_DIR/$vsn" cp -p -P -R "$rel" "$RT_DEST_DIR/$vsn" - # Route out the version (not product) from Git - (cd "$rel"; VERSION="$(git describe --tags | cut -d- -f2)"; echo -n $VERSION > $RT_DEST_DIR/$vsn/VERSION) + # Route out the product and version from Git + (cd "$rel"; VERSION="$(git describe --tags)"; echo -n $VERSION > $RT_DEST_DIR/$vsn/VERSION) done else # This is useful when only testing with 'current' From 87f3f88d496325b8493c3e2dba9bb48b8de3c024 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Fri, 24 Jul 2015 11:07:15 -0400 Subject: [PATCH 33/89] Modified test to wait to send messages until all waiting vnodes are in the handoff state. Also corrected some of the arithmetic errors in counting the number of entries we were supposed to send. (doh) --- tests/verify_handoff_write_once.erl | 73 ++++++++++++++++------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 0fa9ccb04..c942d68fb 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -75,7 +75,7 @@ run_test(Config, AsyncWrites) -> %% Deploy 2 nodes based on config. Wait for K/V to start on each node. %% lager:info("Deploying 2 nodes..."), - Cluster = [RootNode, NewNode] = deploy_test_nodes(2, Config), + Cluster = [RootNode, NewNode] = rt:deploy_nodes(2, Config), [rt:wait_for_service(Node, riak_kv) || Node <- [RootNode, NewNode]], %% %% Set up the intercepts @@ -103,7 +103,7 @@ run_test(Config, AsyncWrites) -> %% lager:info("Populating root node..."), rt:create_and_activate_bucket_type(RootNode, ?BUCKET_TYPE, [{write_once, true}, {n_val, 1}]), - NTestItems = 1000, + NTestItems = 100, RingSize = proplists:get_value(ring_creation_size, proplists:get_value(riak_core, Config)), [] = rt:systest_write(RootNode, 1, NTestItems, ?BUCKET, 1), %% @@ -111,39 +111,35 @@ run_test(Config, AsyncWrites) -> %% lager:info("Joining new node with cluster..."), start_proc(RootNode, NTestItems, RingSize div 2), - timer:sleep(1000), rt:join(NewNode, RootNode), - ?assertEqual(ok, rt:wait_until_nodes_ready(Cluster)), + TotalSent = wait_until_async_writes_complete(), + ?assertMatch(ok, rt:wait_until_nodes_ready(Cluster)), rt:wait_until_no_pending_changes(Cluster), rt:wait_until_transfers_complete(Cluster), - TotalSent = stop_proc(), + %TotalSent = stop_proc(), %% %% Verify the results %% lager:info("Validating data after handoff..."), Results2 = rt:systest_read(NewNode, 1, TotalSent, ?BUCKET, 1), - ?assertEqual([], Results2), + ?assertMatch([], Results2), lager:info("Read ~p entries.", [TotalSent]), [{_, Count}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_put_counter]), ?assertEqual(RingSize div 2, Count), lager:info("We handled ~p write_once puts during handoff.", [Count]), [{_, W1CAsyncReplies}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_async_replies]), [{_, W1CSyncReplies}] = rpc:call(RootNode, ets, lookup, [intercepts_tab, w1c_sync_replies]), - %% NB. We should expect RingSize replies, because 4 handoffs should run in both directions case AsyncWrites of true -> - ?assertEqual(NTestItems + RingSize, W1CAsyncReplies), + ?assertEqual(NTestItems + RingSize div 2, W1CAsyncReplies), ?assertEqual(0, W1CSyncReplies); false -> ?assertEqual(0, W1CAsyncReplies), - ?assertEqual(NTestItems + RingSize, W1CSyncReplies) + ?assertEqual(NTestItems + RingSize div 2, W1CSyncReplies) end, %% Cluster. -deploy_test_nodes(N, Config) -> - rt:deploy_nodes(N, Config). - make_intercepts_tab(Node) -> SupPid = rpc:call(Node, erlang, whereis, [sasl_safe_sup]), intercepts_tab = rpc:call(Node, ets, new, [intercepts_tab, [named_table, @@ -165,37 +161,48 @@ make_intercepts_tab(Node) -> %% -record(state, { - node, sender, k, pids=[], expected + node, sender, k, pids=[], expected, init=true }). start_proc(Node, NTestItems, Expected) -> Self = self(), - Pid = spawn_link(fun() -> loop(#state{node=Node, sender=Self, k=NTestItems, expected=NTestItems+Expected}) end), - global:register_name(rt_ho_w1c_proc, Pid). + Pid = spawn_link(fun() -> loop(#state{node=Node, sender=Self, k=NTestItems, expected=Expected}) end), + global:register_name(rt_ho_w1c_proc, Pid), + receive ok -> ok end. -loop(#state{node=Node, sender=Sender, k=K, pids=Pids, expected=Expected} = State) -> +loop(#state{node=Node, sender=Sender, k=K, pids=Pids, expected=Expected, init=Init} = State) -> + case Init of + true -> + Sender ! ok; + _ -> ok + end, receive - stop -> - Sender ! K; {write, Pid} -> ThePids = [Pid | Pids], - NumWritten = K + 1, - [] = rt:systest_write(Node, K, NumWritten, ?BUCKET, 1), - lager:info("Asynchronously wrote event ~p during handoff.", [K + 1]), - case NumWritten of + NumPids = length(ThePids), + case NumPids of Expected -> - lager:info("Handoff may now complete. Sending ok's back to ~p vnodes", [length(ThePids)]), - [ThePid ! ok || ThePid <- ThePids]; - _ -> ok - end, - loop(State#state{k=NumWritten, pids=ThePids}); - Msg -> - lager:warning("~p: Unexpected Message: ~p. Ignoring...", [?MODULE, Msg]), - loop(State) + %% + %% The number of expected vnodes are now in the handoff state. Do some writes, and send ok's + %% back to the waiting vnodes. Once they get the ok back, they will complete handoff. At this + %% point, we are done, so we can tell the test to proceed and wait for handoff to complete. + %% + [] = rt:systest_write(Node, K + 1, K + Expected, ?BUCKET, 1), + lager:info( + "Asynchronously wrote entries [~p..~p] during handoff. Sending ok's back to ~p waiting vnode(s)...", + [K + 1, K + Expected, NumPids] + ), + [ThePid ! ok || ThePid <- ThePids], + Sender ! (K + Expected); + _ -> + loop(State#state{pids=ThePids, init=false}) + end end. -stop_proc() -> - catch global:send(rt_ho_w1c_proc, stop), + +wait_until_async_writes_complete() -> receive K -> K - end. + after 60000 -> + throw("Timed out after 60s waiting for async writes to complete.") + end. \ No newline at end of file From 5cf5ef0d7aedaaf2f08f59c46944bbe8450b01c3 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Fri, 24 Jul 2015 12:12:31 -0400 Subject: [PATCH 34/89] Cleaned up commented code. No change in behavior. --- tests/verify_handoff_write_once.erl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index c942d68fb..8c09ee0a9 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -91,7 +91,7 @@ run_test(Config, AsyncWrites) -> RootNode, {riak_kv_vnode, [ %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message {{handle_handoff_command, 3}, count_handoff_w1c_puts}, - %% Count everytime riak_kv_vnode:handle_handoff_command/3 is called with a write_once message + %% Count everytime riak_kv_vnode:handle_command/3 is called with a write_once message {{handle_command, 3}, count_w1c_handle_command} ]} ), @@ -116,7 +116,6 @@ run_test(Config, AsyncWrites) -> ?assertMatch(ok, rt:wait_until_nodes_ready(Cluster)), rt:wait_until_no_pending_changes(Cluster), rt:wait_until_transfers_complete(Cluster), - %TotalSent = stop_proc(), %% %% Verify the results %% @@ -137,7 +136,6 @@ run_test(Config, AsyncWrites) -> ?assertEqual(0, W1CAsyncReplies), ?assertEqual(NTestItems + RingSize div 2, W1CSyncReplies) end, - %% Cluster. make_intercepts_tab(Node) -> From 68ca37a54c17327b30d1a569de4f9a537ad49aee Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 24 Jul 2015 16:31:15 -0600 Subject: [PATCH 35/89] Adjust verify_membackend:get_used_space/2 to work with Riak 2.1+ --- tests/verify_membackend.erl | 63 +++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/tests/verify_membackend.erl b/tests/verify_membackend.erl index 536ef1f56..b6545e772 100644 --- a/tests/verify_membackend.erl +++ b/tests/verify_membackend.erl @@ -152,7 +152,10 @@ check_put_delete(Node) -> Mem = get_used_space(Pid, Node), %% this is meh, but the value isn't always the same length. - case (Mem == MemBaseline - 1142) orelse + %% It seems to be the size of a Riak Object + case (Mem == MemBaseline - 1146) orelse + (Mem == MemBaseline - 1145) orelse + (Mem == MemBaseline - 1142) orelse (Mem == MemBaseline - 1141) of true -> ok; @@ -252,35 +255,55 @@ get_remote_vnode_pid(Node) -> all_vnodes, [riak_kv_vnode]), VNode. -%% this is silly fragile +%% @doc Crack open the VNode state record to find +%% the hidden number of used bytes within the +%% riak_kv_memory_backend:state record. +-spec parse_regular_state_fun(integer()) -> fun(). +parse_regular_state_fun(Offset) -> + fun(X) -> + element(Offset, element(4, element(2, X))) + end. + +%% @doc Crack open the VNode state record to find +%% the hidden number of used bytes within the +%% riak_kv_memory_backend:state record for a multi- +%% backend +-spec parse_multi_state_fun(integer()) -> fun(). +parse_multi_state_fun(Offset) -> + fun(X) -> + element( + 3, lists:nth( + 1, element( + 2, element( + Offset, element( + 4, element(2, X)))))) + end. + +%% this is silly fragile and only works for Riak 2.0+ get_used_space(VNode, Node) -> S = rpc:call(Node, sys, get_state, [VNode]), Mode = get(mode), Version = rt:get_version(), %% lager:info("version mode ~p", [{Version, Mode}]), - TwoOhReg = - fun(X) -> - element(4, element(4, element(2, X))) - end, - TwoOhMulti = - fun(X) -> - element( - 3, lists:nth( - 1, element( - 2, element( - 4, element( - 4, element(2, X)))))) - end, - Extract = + + Extract = case {Version, Mode} of {<<"riak-2.0",_/binary>>, regular} -> - TwoOhReg; + parse_regular_state_fun(4); {<<"riak_ee-2.0",_/binary>>, regular} -> - TwoOhReg; + parse_regular_state_fun(4); {<<"riak-2.0",_/binary>>, multi} -> - TwoOhMulti; + parse_multi_state_fun(4); {<<"riak_ee-2.0",_/binary>>, multi} -> - TwoOhMulti; + parse_multi_state_fun(4); + {<<"riak-2.1",_/binary>>, regular} -> + parse_regular_state_fun(5); + {<<"riak_ee-2.1",_/binary>>, regular} -> + parse_regular_state_fun(5); + {<<"riak-2.1",_/binary>>, multi} -> + parse_multi_state_fun(5); + {<<"riak_ee-2.1",_/binary>>, multi} -> + parse_multi_state_fun(5); _Else -> lager:error("didn't understand version/mode tuple ~p", [{Version, Mode}]), From f349c028d9e093542b4475cd8a960ce1739bc060 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Sat, 25 Jul 2015 21:21:42 -0600 Subject: [PATCH 36/89] Fix riak_control and riak_control_authentication for Riak 2.1+ - Use the actual version string (e.g. riak_ee-2.1.2) of devrel instead of "current", "previous", "legacy" to determine which tests to run - Add rt:get_version/1 to find the version of devrels other than "current" --- src/rt.erl | 7 +++ src/rtdev.erl | 11 +++- tests/riak_control.erl | 87 +++++++++++++++------------ tests/riak_control_authentication.erl | 64 ++++++++++++++------ 4 files changed, 107 insertions(+), 62 deletions(-) diff --git a/src/rt.erl b/src/rt.erl index 7a090660b..6da9adc0d 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -72,6 +72,7 @@ get_replica/5, get_ring/1, get_version/0, + get_version/1, heal/1, http_url/1, https_url/1, @@ -1656,6 +1657,12 @@ get_backend(AppConfigProplist) -> Backend -> Backend end. +%% @doc Gets the string flavor of the version tag specified +%% (e.g. current, legacy, previous, etc). +-spec(get_version(atom()) -> binary()). +get_version(Vsn) -> + ?HARNESS:get_version(Vsn). + %% @doc Gets the current version under test. In the case of an upgrade test %% or something like that, it's the version you're upgrading to. -spec get_version() -> binary(). diff --git a/src/rtdev.erl b/src/rtdev.erl index cb725c363..d7639f7e7 100644 --- a/src/rtdev.erl +++ b/src/rtdev.erl @@ -727,12 +727,19 @@ set_backend(Backend, OtherOpts) -> update_app_config(all, [{riak_kv, Opts}]), get_backends(). -get_version() -> - case file:read_file(relpath(current) ++ "/VERSION") of +%% @doc Read the VERSION file from an arbitrarily tagged +%% version (e.g. current, +-spec(get_version(atom()) -> binary()). +get_version(Vsn) -> + case file:read_file(relpath(Vsn) ++ "/VERSION") of {error, enoent} -> unknown; {ok, Version} -> Version end. +%% @doc Read the VERSION file for the `current` version +get_version() -> + get_version(current). + teardown() -> rt_cover:maybe_stop_on_nodes(), %% Stop all discoverable nodes, not just nodes we'll be using for this test. diff --git a/tests/riak_control.erl b/tests/riak_control.erl index 8ad4878b7..41b1608e4 100644 --- a/tests/riak_control.erl +++ b/tests/riak_control.erl @@ -86,21 +86,6 @@ verify_upgrade_fold({FromVsn, Node}, VersionedNodes0) -> VersionedNodes. -verify_control({current, Node}, VersionedNodes) -> - lager:info("Verifying control on node ~p vsn current.", [Node]), - - %% Verify node resource. - {struct, - [{<<"nodes">>, Nodes}]} = verify_resource(Node, "/admin/nodes"), - validate_nodes(Node, Nodes, VersionedNodes, any), - - %% Verify partitions resource. - {struct, - [{<<"partitions">>, Partitions}, - {<<"default_n_val">>, _}]} = verify_resource(Node, "/admin/partitions"), - validate_partitions({current, Node}, Partitions, VersionedNodes), - - ok; verify_control({Vsn, Node}, VersionedNodes) -> lager:info("Verifying control on node ~p vsn ~p.", [Vsn, Node]), @@ -110,11 +95,27 @@ verify_control({Vsn, Node}, VersionedNodes) -> validate_nodes(Node, Nodes, VersionedNodes, any), %% Verify partitions resource. - {struct, - [{<<"partitions">>, Partitions}]} = verify_resource(Node, "/admin/partitions"), + VersionBinary = rt:get_version(Vsn), + Partitions = case VersionBinary of + <<"riak_ee-2.", _/binary>> -> + {struct, + [{<<"partitions">>, NodePartitions}, + {<<"default_n_val">>, _}]} = verify_resource(Node, "/admin/partitions"), + NodePartitions; + <<"riak-2.", _/binary>> -> + {struct, + [{<<"partitions">>, NodePartitions}, + {<<"default_n_val">>, _}]} = verify_resource(Node, "/admin/partitions"), + NodePartitions; + _ -> + {struct, + [{<<"partitions">>, NodePartitions}]} = verify_resource(Node, "/admin/partitions"), + NodePartitions + end, validate_partitions({previous, Node}, Partitions, VersionedNodes), ok. + verify_control(VersionedNodes) -> [verify_control(NodeVsn, VersionedNodes) || NodeVsn <- VersionedNodes]. @@ -184,33 +185,39 @@ wait_for_control_cycle(Node) when is_atom(Node) -> end). %% @doc Validate partitions response. -validate_partitions({current, _}, _ResponsePartitions, _VersionedNodes) -> +validate_partitions({ControlVsn, _}, ResponsePartitions, VersionedNodes) -> + VersionBinary = rt:get_version(ControlVsn), %% The newest version of the partitions display can derive the %% partition state without relying on data from rpc calls -- it can %% use just the ring to do this. Don't test anything specific here %% yet. - ok; -validate_partitions({ControlVsn, _}, ResponsePartitions, VersionedNodes) -> - MixedCluster = mixed_cluster(VersionedNodes), - lager:info("Mixed cluster: ~p.", [MixedCluster]), - - lists:map(fun({struct, Partition}) -> - - %% Parse JSON further. - BinaryName = proplists:get_value(<<"node">>, Partition), - Status = proplists:get_value(<<"status">>, Partition), - Name = list_to_existing_atom(binary_to_list(BinaryName)), - - %% Find current Vsn of node we are validating, and the - %% vsn of the node running Riak Control that we've - %% queried. - {NodeVsn, _} = lists:keyfind(Name, 2, VersionedNodes), - - %% Validate response. - ?assertEqual(true, - valid_status(MixedCluster, ControlVsn, - NodeVsn, Status)) - end, ResponsePartitions). + case VersionBinary of + <<"riak_ee-2.", _/binary>> -> + ok; + <<"riak-2.", _/binary>> -> + ok; + _ -> + MixedCluster = mixed_cluster(VersionedNodes), + lager:info("Mixed cluster: ~p.", [MixedCluster]), + + lists:map(fun({struct, Partition}) -> + + %% Parse JSON further. + BinaryName = proplists:get_value(<<"node">>, Partition), + Status = proplists:get_value(<<"status">>, Partition), + Name = list_to_existing_atom(binary_to_list(BinaryName)), + + %% Find current Vsn of node we are validating, and the + %% vsn of the node running Riak Control that we've + %% queried. + {NodeVsn, _} = lists:keyfind(Name, 2, VersionedNodes), + + %% Validate response. + ?assertEqual(true, + valid_status(MixedCluster, ControlVsn, + NodeVsn, Status)) + end, ResponsePartitions) + end. %% @doc Validate status based on Vsn. valid_status(false, current, current, <<"incompatible">>) -> diff --git a/tests/riak_control_authentication.erl b/tests/riak_control_authentication.erl index a8bbb7f8f..34c6b2832 100644 --- a/tests/riak_control_authentication.erl +++ b/tests/riak_control_authentication.erl @@ -85,21 +85,45 @@ %% @doc Confirm all authentication methods work for the three supported %% releases. confirm() -> - %% Verify authentication method 'none'. - verify_authentication(legacy, ?RC_AUTH_NONE_CONFIG), - verify_authentication(previous, ?RC_AUTH_NONE_CONFIG), - - %% Verify authentication method 'userlist'. - verify_authentication(legacy, ?RC_AUTH_USERLIST_CONFIG), - verify_authentication(previous, ?RC_AUTH_USERLIST_CONFIG), + TestVersions = [current, previous, legacy], + [determine_test_suite(Vsn) || Vsn <- TestVersions], + pass. +%% @doc Determine if current, legacy or previous is a Riak 2.0+ +%% version or not and test accordingly. +-spec(determine_test_suite(atom()) -> fun()). +determine_test_suite(Vsn) -> + VersionBinary = rt:get_version(Vsn), + case VersionBinary of + <<"riak_ee-2.", _/binary>> -> + verify_authentication_post20(Vsn); + <<"riak-2.", _/binary>> -> + verify_authentication_post20(Vsn); + _ -> + verify_authentication_pre20(Vsn) + end. + +%% @doc Test authentication methods for versions since Riak 2.0 +-spec(verify_authentication_post20(atom()) -> ok). +verify_authentication_post20(Vsn) -> %% Verify authentication none, and then with forced SSL. - verify_authentication(current, ?RC_AUTH_NONE_CONFIG), - verify_authentication(current, ?RC_AUTH_NONE_CONFIG_FORCE_SSL), + verify_authentication(Vsn, ?RC_AUTH_NONE_CONFIG), + verify_authentication(Vsn, ?RC_AUTH_NONE_CONFIG_FORCE_SSL), %% Verify authentication userlist, without SSL and then with SSL. - verify_authentication(current, ?RC_AUTH_USERLIST_CONFIG_FORCE_SSL), - verify_authentication(current, ?RC_AUTH_USERLIST_CONFIG_NO_FORCE_SSL). + verify_authentication(Vsn, ?RC_AUTH_USERLIST_CONFIG_FORCE_SSL), + verify_authentication(Vsn, ?RC_AUTH_USERLIST_CONFIG_NO_FORCE_SSL), + ok. + +%% @doc Test authentication methods for versions before Riak 2.0 +-spec(verify_authentication_pre20(atom()) -> ok). +verify_authentication_pre20(Vsn) -> + %% Verify authentication method 'none'. + verify_authentication(Vsn, ?RC_AUTH_NONE_CONFIG), + + %% Verify authentication method 'userlist'. + verify_authentication(Vsn, ?RC_AUTH_USERLIST_CONFIG), + ok. %% @doc Verify the disabled authentication method works. verify_authentication(Vsn, ?RC_AUTH_NONE_CONFIG) -> @@ -115,9 +139,9 @@ verify_authentication(Vsn, ?RC_AUTH_NONE_CONFIG) -> pass; %% @doc Verify the disabled authentication method works with force SSL. -verify_authentication(current, ?RC_AUTH_NONE_CONFIG_FORCE_SSL) -> - lager:info("Verifying auth 'none', 'force_ssl' 'true', current."), - Nodes = build_singleton_cluster(current, +verify_authentication(Vsn, ?RC_AUTH_NONE_CONFIG_FORCE_SSL) -> + lager:info("Verifying auth 'none', 'force_ssl' 'true', ~p.", [Vsn]), + Nodes = build_singleton_cluster(Vsn, ?RC_AUTH_NONE_CONFIG_FORCE_SSL), Node = lists:nth(1, Nodes), @@ -161,9 +185,9 @@ verify_authentication(Vsn, ?RC_AUTH_USERLIST_CONFIG) -> pass; %% @doc Verify the userlist authentication method works. -verify_authentication(current, ?RC_AUTH_USERLIST_CONFIG_FORCE_SSL) -> - lager:info("Verifying auth 'userlist', 'force_ssl' 'true', current."), - Nodes = build_singleton_cluster(current, ?RC_AUTH_USERLIST_CONFIG_FORCE_SSL), +verify_authentication(Vsn, ?RC_AUTH_USERLIST_CONFIG_FORCE_SSL) -> + lager:info("Verifying auth 'userlist', 'force_ssl' 'true', ~p.", [Vsn]), + Nodes = build_singleton_cluster(Vsn, ?RC_AUTH_USERLIST_CONFIG_FORCE_SSL), Node = lists:nth(1, Nodes), %% Assert that we get redirected if we hit the HTTP port. @@ -188,9 +212,9 @@ verify_authentication(current, ?RC_AUTH_USERLIST_CONFIG_FORCE_SSL) -> pass; %% @doc Verify the userlist authentication method works. -verify_authentication(current, ?RC_AUTH_USERLIST_CONFIG_NO_FORCE_SSL) -> - lager:info("Verifying auth 'userlist', 'force_ssl' 'false', current."), - Nodes = build_singleton_cluster(current, ?RC_AUTH_USERLIST_CONFIG_NO_FORCE_SSL), +verify_authentication(Vsn, ?RC_AUTH_USERLIST_CONFIG_NO_FORCE_SSL) -> + lager:info("Verifying auth 'userlist', 'force_ssl' 'false', ~p.", [Vsn]), + Nodes = build_singleton_cluster(Vsn, ?RC_AUTH_USERLIST_CONFIG_NO_FORCE_SSL), Node = lists:nth(1, Nodes), %% Assert that we can access resource over the SSL port. From 98c68323670dab733c9a4fe20d0205b8b972cb67 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 27 Jul 2015 15:27:52 -0600 Subject: [PATCH 37/89] Switch check_put_delete/1 to use a range of acceptable sizes --- tests/verify_membackend.erl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/verify_membackend.erl b/tests/verify_membackend.erl index b6545e772..7a41186d2 100644 --- a/tests/verify_membackend.erl +++ b/tests/verify_membackend.erl @@ -152,11 +152,9 @@ check_put_delete(Node) -> Mem = get_used_space(Pid, Node), %% this is meh, but the value isn't always the same length. - %% It seems to be the size of a Riak Object - case (Mem == MemBaseline - 1146) orelse - (Mem == MemBaseline - 1145) orelse - (Mem == MemBaseline - 1142) orelse - (Mem == MemBaseline - 1141) of + %% It seems to be the size of a Riak Object put some overhead + case (MemBaseline - Mem >= 1142) andalso + (MemBaseline - Mem =< 1150) of true -> ok; false -> From 7e7cb73d5545e9baae02c632269ffb68fa0126b3 Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Tue, 28 Jul 2015 11:03:45 -0400 Subject: [PATCH 38/89] Add rt:wait_until_bucket_type_visible to avoid floppiness in test - verify would fail because, while handoff had completed, the bucket type information hadn't yet been gossiped to the new node. --- tests/verify_handoff_write_once.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/verify_handoff_write_once.erl b/tests/verify_handoff_write_once.erl index 8c09ee0a9..9753946ed 100644 --- a/tests/verify_handoff_write_once.erl +++ b/tests/verify_handoff_write_once.erl @@ -114,6 +114,7 @@ run_test(Config, AsyncWrites) -> rt:join(NewNode, RootNode), TotalSent = wait_until_async_writes_complete(), ?assertMatch(ok, rt:wait_until_nodes_ready(Cluster)), + rt:wait_until_bucket_type_visible(Cluster, ?BUCKET_TYPE), rt:wait_until_no_pending_changes(Cluster), rt:wait_until_transfers_complete(Cluster), %% From 73c24ab8618e03025e0545a67ed6c20fb2b8deff Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 30 Jul 2015 09:39:22 -0600 Subject: [PATCH 39/89] Update the URL for pulling down kerl --- bin/rtdev-build-releases.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index e205a6978..acbfafc71 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -44,7 +44,7 @@ checkbuild() echo "You need 'curl' to be able to run this script, exiting" exit 1 fi - curl -O https://raw.github.com/spawngrid/kerl/master/kerl > /dev/null 2>&1; chmod a+x kerl + curl -O https://raw.githubusercontent.com/spawngrid/kerl/master/kerl > /dev/null 2>&1; chmod a+x kerl fi fi fi From 1e680333ddfeaadac2d7c64cb9fc800ab7a0b479 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Thu, 30 Jul 2015 17:10:16 -0500 Subject: [PATCH 40/89] If Darwin, don't build hipe/odbc --- bin/rtdev-build-releases.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index acbfafc71..201f1ffc9 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -55,8 +55,12 @@ kerl() RELEASE=$1 BUILDNAME=$2 + if [[ $(uname -s) -eq "Darwin" ]]; then + export KERL_CONFIGURE_OPTIONS='--without-hipe --without-odbc' + fi + echo " - Building Erlang $RELEASE (this could take a while)" - ./kerl build $RELEASE $BUILDNAME > /dev/null 2>&1 + $KERL_FLAGS ./kerl build $RELEASE $BUILDNAME > /dev/null 2>&1 RES=$? if [ "$RES" -ne 0 ]; then echo "[ERROR] Kerl build $BUILDNAME failed" From 13408eb9e28dfbc13ca704c305e18e2d17952ec5 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 31 Jul 2015 16:43:03 -0400 Subject: [PATCH 41/89] add some clarity to ordering of socket close and such --- tests/yz_extractors.erl | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl index 0e1582e86..252701afc 100644 --- a/tests/yz_extractors.erl +++ b/tests/yz_extractors.erl @@ -79,13 +79,17 @@ confirm() -> rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA]), yokozuna_rt:write_data(Cluster, OldPid, ?INDEX1, ?BUCKET1, GenKeys), + + ok = rt:stop_tracing(), + %% wait for solr soft commit timer:sleep(1100), {ok, BProps} = riakc_pb_socket:get_bucket(OldPid, ?BUCKET1), N = proplists:get_value(n_val, BProps), - ok = rt:stop_tracing(), + riakc_pb_socket:stop(OldPid), + PrevGetMapRingCC = rt:get_call_count(Cluster, ?GET_MAP_RING_MFA), PrevGetMapCC = rt:get_call_count(Cluster, ?GET_MAP_MFA), ?assertEqual(KeyCount * N, PrevGetMapRingCC), @@ -93,7 +97,6 @@ confirm() -> %% test query count yokozuna_rt:verify_num_found_query(Cluster, ?INDEX1, KeyCount), - riakc_pb_socket:stop(OldPid), {RingVal1, MDVal1} = get_ring_and_cmd_vals(Node, ?YZ_META_EXTRACTORS, ?YZ_EXTRACTOR_MAP), @@ -120,15 +123,19 @@ confirm() -> %% test query count again yokozuna_rt:verify_num_found_query(Cluster, ?INDEX1, KeyCount), + Pid = rt:pbc(Node), + rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA, ?GET_MAP_READTHROUGH_MFA]), - Pid = rt:pbc(Node), yokozuna_rt:write_data(Cluster, Pid, ?INDEX2, ?BUCKET2, GenKeys), + riakc_pb_socket:stop(Pid), + + ok = rt:stop_tracing(), + %% wait for solr soft commit timer:sleep(1100), - ok = rt:stop_tracing(), CurrGetMapRingCC = rt:get_call_count(Cluster, ?GET_MAP_RING_MFA), CurrGetMapCC = rt:get_call_count(Cluster, ?GET_MAP_MFA), CurrGetMapRTCC = rt:get_call_count(Cluster, ?GET_MAP_READTHROUGH_MFA), @@ -143,8 +150,6 @@ confirm() -> [CurrGetMapRTCC, CurrGetMapCC]), ?assert(CurrGetMapRTCC < CurrGetMapCC), - riakc_pb_socket:stop(Pid), - {_RingVal2, MDVal2} = get_ring_and_cmd_vals(Node, ?YZ_META_EXTRACTORS, ?YZ_EXTRACTOR_MAP), From 4034777d0c0d3ec814fb79deadcf1673269ac2ed Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 30 Jul 2015 17:42:12 -0600 Subject: [PATCH 42/89] Change verify_memorybackend to use calls to core and kv to determine memory used, instead of a brittle parsing of data structures --- tests/verify_membackend.erl | 116 ++++++++++++------------------------ 1 file changed, 37 insertions(+), 79 deletions(-) diff --git a/tests/verify_membackend.erl b/tests/verify_membackend.erl index 7a41186d2..9f1c592d3 100644 --- a/tests/verify_membackend.erl +++ b/tests/verify_membackend.erl @@ -7,14 +7,6 @@ -define(BUCKET, <<"ttl_test">>). -%% from 2.0, but should be valid for 1.1+ --record(state, {data_ref :: ets:tid(), - index_ref :: ets:tid(), - time_ref :: ets:tid(), - max_memory :: undefined | integer(), - used_memory=0 :: integer(), - ttl :: integer()}). - confirm() -> Tests = [ttl, max_memory, combo], [Res1, Res2] = @@ -67,7 +59,7 @@ combo(Mode) -> %% Make sure that expiry is updating used_memory correctly Pid = get_remote_vnode_pid(NodeA), - 0 = get_used_space(Pid, NodeA), + {0, _} = get_used_space(Pid), ?assertEqual(ok, check_put_delete(NodeA)), @@ -141,7 +133,7 @@ check_put_delete(Node) -> lager:info("checking that used mem is reclaimed on delete"), Pid = get_remote_vnode_pid(Node), - {MemBaseline, Key} = put_until_changed(Pid, Node, 1000), + {MemBaseline, PutSize, Key} = put_until_changed(Pid, Node, 1000), {ok, C} = riak:client_connect(Node), @@ -149,12 +141,12 @@ check_put_delete(Node) -> timer:sleep(timer:seconds(5)), - Mem = get_used_space(Pid, Node), + {Mem, _PutSize} = get_used_space(Pid), - %% this is meh, but the value isn't always the same length. - %% It seems to be the size of a Riak Object put some overhead - case (MemBaseline - Mem >= 1142) andalso - (MemBaseline - Mem =< 1150) of + %% The size of the encoded Riak Object varies a bit based on + %% the included metadata, so we consult the riak_kv_memory_backend + %% to determine the actual size of the object written + case (MemBaseline - Mem) =:= PutSize of true -> ok; false -> @@ -166,17 +158,18 @@ check_put_consistent(Node) -> lager:info("checking that used mem doesn't change on re-put"), Pid = get_remote_vnode_pid(Node), - {MemBaseline, Key} = put_until_changed(Pid, Node, 1000), + {MemBaseline, _PutSize, Key} = put_until_changed(Pid, Node, 1000), {ok, C} = riak:client_connect(Node), + %% Write a slightly larger object than before ok = C:put(riak_object:new(?BUCKET, <>, <<0:8192>>)), {ok, _} = C:get(?BUCKET, <>), timer:sleep(timer:seconds(2)), - Mem = get_used_space(Pid, Node), + {Mem, _} = get_used_space(Pid), case abs(Mem - MemBaseline) < 3 of true -> ok; @@ -185,18 +178,22 @@ check_put_consistent(Node) -> end, ok. +%% @doc Keep putting objects until the memory used changes +%% Also return the size of the object after it's been encoded +%% and send to the memory backend. +-spec(put_until_changed(pid(), node(), term()) -> {integer(), integer(), term()}). put_until_changed(Pid, Node, Key) -> {ok, C} = riak:client_connect(Node), - UsedSpace = get_used_space(Pid, Node), + {UsedSpace, _} = get_used_space(Pid), C:put(riak_object:new(?BUCKET, <>, <<0:8192>>)), - + timer:sleep(100), - UsedSpace1 = get_used_space(Pid, Node), + {UsedSpace1, PutObjSize} = get_used_space(Pid), case UsedSpace < UsedSpace1 of true -> - {UsedSpace1, Key}; + {UsedSpace1, PutObjSize, Key}; false -> put_until_changed(Pid, Node, Key+1) end. @@ -253,61 +250,22 @@ get_remote_vnode_pid(Node) -> all_vnodes, [riak_kv_vnode]), VNode. -%% @doc Crack open the VNode state record to find -%% the hidden number of used bytes within the -%% riak_kv_memory_backend:state record. --spec parse_regular_state_fun(integer()) -> fun(). -parse_regular_state_fun(Offset) -> - fun(X) -> - element(Offset, element(4, element(2, X))) - end. - -%% @doc Crack open the VNode state record to find -%% the hidden number of used bytes within the -%% riak_kv_memory_backend:state record for a multi- -%% backend --spec parse_multi_state_fun(integer()) -> fun(). -parse_multi_state_fun(Offset) -> - fun(X) -> - element( - 3, lists:nth( - 1, element( - 2, element( - Offset, element( - 4, element(2, X)))))) - end. - -%% this is silly fragile and only works for Riak 2.0+ -get_used_space(VNode, Node) -> - S = rpc:call(Node, sys, get_state, [VNode]), - Mode = get(mode), - Version = rt:get_version(), - %% lager:info("version mode ~p", [{Version, Mode}]), - - Extract = - case {Version, Mode} of - {<<"riak-2.0",_/binary>>, regular} -> - parse_regular_state_fun(4); - {<<"riak_ee-2.0",_/binary>>, regular} -> - parse_regular_state_fun(4); - {<<"riak-2.0",_/binary>>, multi} -> - parse_multi_state_fun(4); - {<<"riak_ee-2.0",_/binary>>, multi} -> - parse_multi_state_fun(4); - {<<"riak-2.1",_/binary>>, regular} -> - parse_regular_state_fun(5); - {<<"riak_ee-2.1",_/binary>>, regular} -> - parse_regular_state_fun(5); - {<<"riak-2.1",_/binary>>, multi} -> - parse_multi_state_fun(5); - {<<"riak_ee-2.1",_/binary>>, multi} -> - parse_multi_state_fun(5); - _Else -> - lager:error("didn't understand version/mode tuple ~p", - [{Version, Mode}]), - throw(boom) - end, - State = Extract(S), - Mem = State#state.used_memory, - lager:info("got ~p used memory", [Mem]), - Mem. +%% @doc Interrogate the VNode to determine the memory used +-spec(get_used_space(pid()) -> {integer(), integer()}). +get_used_space(VNodePid) -> + {_Mod, State} = riak_core_vnode:get_modstate(VNodePid), + {Mod, ModState} = riak_kv_vnode:get_modstate(State), + Status = case Mod of + riak_kv_memory_backend -> + riak_kv_memory_backend:status(ModState); + riak_kv_multi_backend -> + [{_Name, Stat}] = riak_kv_multi_backend:status(ModState), + Stat; + _Else -> + lager:error("didn't understand backend ~p", [Mod]), + throw(boom) + end, + Mem = proplists:get_value(used_memory, Status), + PutObjSize = proplists:get_value(put_obj_size, Status), + lager:info("got ~p used memory, ~p put object size", [Mem, PutObjSize]), + {Mem, PutObjSize}. From 3c9fae324132934f2eb6b2c352e87084888c6c5e Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Fri, 31 Jul 2015 16:23:40 -0500 Subject: [PATCH 43/89] Make the release build process more reliable Many changes here: * Only download source once from the Interwebs * Skip a directory if it's already built * Use locked-deps when appropriate --- bin/rtdev-build-releases.sh | 73 ++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index 4037f644c..ac3364b22 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -24,6 +24,7 @@ : ${RT_USE_EE:=""} GITURL_RIAK="git://github.com/basho/riak" GITURL_RIAK_EE="git@github.com:basho/riak_ee" +GITDIR="riak-src" checkbuild() @@ -77,6 +78,12 @@ build() SRCDIR=$1 ERLROOT=$2 TAG="$3" + if [ -z $4 ]; then + LOCKED=true + else + LOCKED=$4 + fi + if [ -z "$RT_USE_EE" ]; then GITURL=$GITURL_RIAK GITTAG=riak-$TAG @@ -85,6 +92,15 @@ build() GITTAG=riak_ee-$TAG fi + echo "Getting sources from github" + if [ ! -d $GITDIR ]; then + git clone $GITURL $GITDIR + else + cd $GITDIR + git pull origin master + cd .. + fi + echo "Building $SRCDIR:" checkbuild $ERLROOT @@ -94,37 +110,52 @@ build() kerl $RELEASE $BUILDNAME fi - GITRES=1 - echo " - Cloning $GITURL" - rm -rf $SRCDIR - git clone $GITURL $SRCDIR - GITRES=$? - if [ $GITRES -eq 0 -a -n "$TAG" ]; then - cd $SRCDIR - git checkout $GITTAG + if [ ! -d $SRCDIR ] + then + GITRES=1 + echo " - Cloning $GITURL" + git clone $GITDIR $SRCDIR GITRES=$? - cd .. + if [ $GITRES -eq 0 -a -n "$TAG" ]; then + cd $SRCDIR + git checkout $GITTAG + GITRES=$? + cd .. + fi fi - RUN="env PATH=$ERLROOT/bin:$ERLROOT/lib/erlang/bin:$PATH \ + if [ ! -f "$SRCDIR/built" ] + then + + RUN="env PATH=$ERLROOT/bin:$ERLROOT/lib/erlang/bin:$PATH \ C_INCLUDE_PATH=$ERLROOT/usr/include \ LD_LIBRARY_PATH=$ERLROOT/usr/lib" - echo " - Building devrel in $SRCDIR (this could take a while)" - cd $SRCDIR + echo " - Building devrel in $SRCDIR (this could take a while)" + cd $SRCDIR - $RUN make all devrel - RES=$? - if [ "$RES" -ne 0 ]; then - echo "[ERROR] make devrel failed" - exit 1 + if $LOCKED + then + CMD="make locked-deps devrel" + else + CMD="make all devrel" + fi + + $RUN $CMD + RES=$? + if [ "$RES" -ne 0 ]; then + echo "[ERROR] make devrel failed" + exit 1 + fi + touch built + cd .. + echo " - $SRCDIR built." + else + echo " - already built" fi - cd .. - echo " - $SRCDIR built." } - -build "riak-1.4.12" $R15B01 1.4.12 +build "riak-1.4.12" $R15B01 1.4.12 false build "riak-2.0.2" $R16B02 2.0.2 build "riak-2.0.4" $R16B02 2.0.4 build "riak-2.0.5" $R16B02 2.0.5 From c746eb92909dfc9afea7fca5b9d9a9d11195b6ab Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Fri, 31 Jul 2015 21:09:48 -0500 Subject: [PATCH 44/89] Make the erlang builds more official --- bin/rtdev-build-releases.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index 201f1ffc9..15aa928f3 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -55,12 +55,18 @@ kerl() RELEASE=$1 BUILDNAME=$2 - if [[ $(uname -s) -eq "Darwin" ]]; then - export KERL_CONFIGURE_OPTIONS='--without-hipe --without-odbc' + BUILDFLAGS="--disable-hipe --enable-smp-support --without-odbc" + if [ $(uname -s) = "Darwin" ]; then + BUILDFLAGS="$BUILDFLAGS --enable-darwin-64bit --with-dynamic-trace=dtrace" + else + BUILDFLAGS="$BUILDFLAGS --enable-m64-build" fi + KERL_ENV="KERL_CONFIGURE_OPTIONS=${BUILDFLAGS}" + MAKE="make -j10" + echo " - Building Erlang $RELEASE (this could take a while)" - $KERL_FLAGS ./kerl build $RELEASE $BUILDNAME > /dev/null 2>&1 + env "$KERL_ENV" "MAKE=$MAKE" ./kerl build $RELEASE $BUILDNAME > /dev/null 2>&1 RES=$? if [ "$RES" -ne 0 ]; then echo "[ERROR] Kerl build $BUILDNAME failed" From 659f96ae345db1c7c0eac0b7a1f3e926da138161 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 3 Aug 2015 19:10:30 +0000 Subject: [PATCH 45/89] Build Basho-patched Erlang and add tweaks for CentOS --- bin/rtdev-build-releases.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index b6cbc40fe..d54b316b3 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -56,8 +56,14 @@ kerl() RELEASE=$1 BUILDNAME=$2 + export CFLAGS="-g -O2" + export LDFLAGS="-g" + if [ -n "`uname -r | grep el6`" ]; then + export CFLAGS="-g -DOPENSSL_NO_EC=1" + fi BUILDFLAGS="--disable-hipe --enable-smp-support --without-odbc" if [ $(uname -s) = "Darwin" ]; then + export CFLAGS="-g -O0" BUILDFLAGS="$BUILDFLAGS --enable-darwin-64bit --with-dynamic-trace=dtrace" else BUILDFLAGS="$BUILDFLAGS --enable-m64-build" @@ -67,7 +73,14 @@ kerl() MAKE="make -j10" echo " - Building Erlang $RELEASE (this could take a while)" - env "$KERL_ENV" "MAKE=$MAKE" ./kerl build $RELEASE $BUILDNAME > /dev/null 2>&1 + # Use the Basho-patched version of Erlang + if [ "$RELEASE" == "R15B01" ]; then + env "$KERL_ENV" "MAKE=$MAKE" ./kerl build git git://github.com/basho/otp.git basho_OTP_R15B01p $BUILDNAME + elif [ "$RELEASE" == "R16B02" ]; then + env "$KERL_ENV" "MAKE=$MAKE" ./kerl build git git://github.com/basho/otp.git r16 $BUILDNAME + else + env "$KERL_ENV" "MAKE=$MAKE" ./kerl build $RELEASE $BUILDNAME + fi RES=$? if [ "$RES" -ne 0 ]; then echo "[ERROR] Kerl build $BUILDNAME failed" @@ -89,9 +102,9 @@ build() ERLROOT=$2 TAG="$3" if [ -z $4 ]; then - LOCKED=true + LOCKED_DEPS=true else - LOCKED=$4 + LOCKED_DEPS=$4 fi if [ -z "$RT_USE_EE" ]; then @@ -107,7 +120,7 @@ build() git clone $GITURL $GITDIR else cd $GITDIR - git pull origin master + git pull origin develop cd .. fi @@ -144,7 +157,7 @@ build() echo " - Building devrel in $SRCDIR (this could take a while)" cd $SRCDIR - if $LOCKED + if $LOCKED_DEPS then CMD="make locked-deps devrel" else From 920ddf7c03ee98df9bb2c397bd0d9c6eccec402e Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 3 Aug 2015 19:18:38 +0000 Subject: [PATCH 46/89] Update R16 version to OTP_R16B02_basho9 tag --- bin/rtdev-build-releases.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index d54b316b3..bc5bedf93 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -77,7 +77,7 @@ kerl() if [ "$RELEASE" == "R15B01" ]; then env "$KERL_ENV" "MAKE=$MAKE" ./kerl build git git://github.com/basho/otp.git basho_OTP_R15B01p $BUILDNAME elif [ "$RELEASE" == "R16B02" ]; then - env "$KERL_ENV" "MAKE=$MAKE" ./kerl build git git://github.com/basho/otp.git r16 $BUILDNAME + env "$KERL_ENV" "MAKE=$MAKE" ./kerl build git git://github.com/basho/otp.git OTP_R16B02_basho9 $BUILDNAME else env "$KERL_ENV" "MAKE=$MAKE" ./kerl build $RELEASE $BUILDNAME fi From c192f92faf6a8037aa318ecf7fcb581cbd500633 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Mon, 3 Aug 2015 14:41:28 -0500 Subject: [PATCH 47/89] Make release tags BASH variable --- bin/rtdev-build-releases.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index bc5bedf93..24c553fb2 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -19,6 +19,13 @@ : ${R15B01:=$HOME/erlang-R15B01} : ${R16B02:=$HOME/erlang-R16B02} +# These are the default tags to use when building basho OTP releases. +# Export different tags to get a different build. N.B. You will need to +# remove the builds from kerl (e.g., kerl delete build $BUILDNAME) and +# possibly remove the directories above. +: ${R16_TAG:="OTP_R16B02_basho9"} +: ${R15_TAG:="basho_OTP_R15B01p"} + # By default the Open Source version of Riak will be used, but for internal # testing you can override this variable to use `riak_ee` instead : ${RT_USE_EE:=""} @@ -75,12 +82,13 @@ kerl() echo " - Building Erlang $RELEASE (this could take a while)" # Use the Basho-patched version of Erlang if [ "$RELEASE" == "R15B01" ]; then - env "$KERL_ENV" "MAKE=$MAKE" ./kerl build git git://github.com/basho/otp.git basho_OTP_R15B01p $BUILDNAME + BUILD_CMD="./kerl build git git://github.com/basho/otp.git $R15_TAG $BUILDNAME" elif [ "$RELEASE" == "R16B02" ]; then - env "$KERL_ENV" "MAKE=$MAKE" ./kerl build git git://github.com/basho/otp.git OTP_R16B02_basho9 $BUILDNAME + BUILD_CMD="./kerl build git git://github.com/basho/otp.git $R16_TAG $BUILDNAME" else - env "$KERL_ENV" "MAKE=$MAKE" ./kerl build $RELEASE $BUILDNAME + BUILD_CMD="./kerl build $RELEASE $BUILDNAME" fi + env "$KERL_ENV" "MAKE=$MAKE" $BUILD_CMD RES=$? if [ "$RES" -ne 0 ]; then echo "[ERROR] Kerl build $BUILDNAME failed" @@ -88,7 +96,7 @@ kerl() fi echo " - Installing $RELEASE into $HOME/$BUILDNAME" - ./kerl install $BUILDNAME $HOME/$BUILDNAME > /dev/null 2>&1 + ./kerl install $BUILDNAME "$HOME/$BUILDNAME" > /dev/null 2>&1 RES=$? if [ "$RES" -ne 0 ]; then echo "[ERROR] Kerl install $BUILDNAME failed" From ff1f73c82184fb83dc1bf224f0f12713ddae2e34 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 4 Aug 2015 02:42:01 +0000 Subject: [PATCH 48/89] Update versions built --- bin/rtdev-build-releases.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/rtdev-build-releases.sh b/bin/rtdev-build-releases.sh index 24c553fb2..a4e4b175e 100755 --- a/bin/rtdev-build-releases.sh +++ b/bin/rtdev-build-releases.sh @@ -189,6 +189,5 @@ build() build "riak-1.4.12" $R15B01 1.4.12 false build "riak-2.0.2" $R16B02 2.0.2 build "riak-2.0.4" $R16B02 2.0.4 -build "riak-2.0.5" $R16B02 2.0.5 -build "riak-2.1.1" $R16B02 2.1.1 +build "riak-2.0.6" $R16B02 2.0.6 echo From ae7815d57d55e977de8586490e3922be834d831c Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 5 Aug 2015 11:32:58 -0400 Subject: [PATCH 49/89] switcheroo --- src/yokozuna_rt.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl index e7a17c123..7063ec0db 100644 --- a/src/yokozuna_rt.erl +++ b/src/yokozuna_rt.erl @@ -59,7 +59,7 @@ host_entries(ClusterConnInfo) -> write_data(Cluster, Pid, Index, Bucket, Keys) -> riakc_pb_socket:set_options(Pid, [queue_if_disconnected]), - create_and_set_index(Cluster, Pid, Index, Bucket), + create_and_set_index(Cluster, Pid, Bucket, Index), timer:sleep(1000), %% Write keys From f6fb08c53a11c30443407c8de7013c85b188132d Mon Sep 17 00:00:00 2001 From: Kazuhiro Suzuki Date: Tue, 18 Aug 2015 06:04:47 +0900 Subject: [PATCH 50/89] Move rt_cs_dev to cs repo --- src/rt_cs_dev.erl | 511 ---------------------------------------------- 1 file changed, 511 deletions(-) delete mode 100644 src/rt_cs_dev.erl diff --git a/src/rt_cs_dev.erl b/src/rt_cs_dev.erl deleted file mode 100644 index 10e34b344..000000000 --- a/src/rt_cs_dev.erl +++ /dev/null @@ -1,511 +0,0 @@ -%% ------------------------------------------------------------------- -%% -%% Copyright (c) 2013 Basho Technologies, Inc. -%% -%% This file is provided to you under the Apache License, -%% Version 2.0 (the "License"); you may not use this file -%% except in compliance with the License. You may obtain -%% a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, -%% software distributed under the License is distributed on an -%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -%% KIND, either express or implied. See the License for the -%% specific language governing permissions and limitations -%% under the License. -%% -%% ------------------------------------------------------------------- - -%% @private --module(rt_cs_dev). --compile(export_all). --include_lib("eunit/include/eunit.hrl"). - --define(DEVS(N), lists:concat(["dev", N, "@127.0.0.1"])). --define(DEV(N), list_to_atom(?DEVS(N))). --define(BUILD_PATHS, (rt_config:get(build_paths))). --define(SRC_PATHS, (rt_config:get(src_paths))). - -get_deps() -> - lists:flatten(io_lib:format("~s/dev/dev1/lib", [relpath(current)])). - -setup_harness(_Test, _Args) -> - confirm_build_type(rt_config:get(build_type, oss)), - Path = relpath(root), - %% Stop all discoverable nodes, not just nodes we'll be using for this test. - rt:pmap(fun(X) -> stop_all(X ++ "/dev") end, devpaths()), - - %% Reset nodes to base state - lager:info("Resetting nodes to fresh state"), - rtdev:run_git(Path, "reset HEAD --hard"), - rtdev:run_git(Path, "clean -fd"), - - lager:info("Cleaning up lingering pipe directories"), - rt:pmap(fun(Dir) -> - %% when joining two absolute paths, filename:join intentionally - %% throws away the first one. ++ gets us around that, while - %% keeping some of the security of filename:join. - %% the extra slashes will be pruned by filename:join, but this - %% ensures that there will be at least one between "/tmp" and Dir - PipeDir = filename:join(["/tmp//" ++ Dir, "dev"]), - %% when using filelib:wildcard/2, there must be a wildchar char - %% before the first '/'. - Files = filelib:wildcard("dev?/*.{r,w}", PipeDir), - [ file:delete(filename:join(PipeDir, File)) || File <- Files], - file:del_dir(PipeDir) - end, devpaths()), - ok. - -confirm_build_type(BuildType) -> - [ok = confirm_build_type(BuildType, Vsn) || Vsn <- [cs_current, cs_previous]]. - -confirm_build_type(BuildType, Vsn) -> - ReplPB = filename:join([relpath(Vsn), "dev/dev1/lib/riak_repl_pb_api*"]), - case {BuildType, filelib:wildcard(ReplPB)} of - {oss, []} -> ok; - {ee, [_|_]} -> ok; - _ -> - lager:error("Build type of ~p is not ~p", - [Vsn, BuildType]), - {error, {build_type_mismatch, Vsn}} - end. - -relpath(Vsn) -> - Path = ?BUILD_PATHS, - path(Vsn, Path). - -srcpath(Vsn) -> - Path = ?SRC_PATHS, - path(Vsn, Path). - -path(Vsn, Paths=[{_,_}|_]) -> - orddict:fetch(Vsn, orddict:from_list(Paths)); -path(current, Path) -> - Path; -path(root, Path) -> - Path; -path(_, _) -> - throw("Version requested but only one path provided"). - -upgrade(Node, NewVersion) -> - N = node_id(Node), - Version = node_version(N), - lager:info("Upgrading ~p : ~p -> ~p", [Node, Version, NewVersion]), - catch stop(Node), - rt:wait_until_unpingable(Node), - OldPath = relpath(Version), - NewPath = relpath(NewVersion), - - Commands = [ - io_lib:format("cp -p -P -R \"~s/dev/dev~b/data\" \"~s/dev/dev~b\"", - [OldPath, N, NewPath, N]), - io_lib:format("rm -rf ~s/dev/dev~b/data/*", - [OldPath, N]), - io_lib:format("cp -p -P -R \"~s/dev/dev~b/etc\" \"~s/dev/dev~b\"", - [OldPath, N, NewPath, N]) - ], - [ begin - lager:info("Running: ~s", [Cmd]), - os:cmd(Cmd) - end || Cmd <- Commands], - VersionMap = orddict:store(N, NewVersion, rt_config:get(rt_versions)), - rt_config:set(rt_versions, VersionMap), - start(Node), - rt:wait_until_pingable(Node), - ok. - -all_the_app_configs(DevPath) -> - lager:error("The dev path is ~p", [DevPath]), - case filelib:is_dir(DevPath) of - true -> - Devs = filelib:wildcard(DevPath ++ "/dev/dev*"), - [ Dev ++ "/etc/app.config" || Dev <- Devs]; - _ -> - lager:debug("~s is not a directory.", [DevPath]), - [] - end. - -update_app_config(all, Config) -> - lager:info("rtdev:update_app_config(all, ~p)", [Config]), - [ update_app_config(DevPath, Config) || DevPath <- devpaths()]; -update_app_config(Node, Config) when is_atom(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - FileFormatString = "~s/dev/dev~b/etc/~s.config", - - AppConfigFile = io_lib:format(FileFormatString, [Path, N, "app"]), - AdvConfigFile = io_lib:format(FileFormatString, [Path, N, "advanced"]), - %% If there's an app.config, do it old style - %% if not, use cuttlefish's adavnced.config - case filelib:is_file(AppConfigFile) of - true -> - update_app_config_file(AppConfigFile, Config); - _ -> - update_app_config_file(AdvConfigFile, Config) - end; -update_app_config(DevPath, Config) -> - [update_app_config_file(AppConfig, Config) || AppConfig <- all_the_app_configs(DevPath)]. - -update_app_config_file(ConfigFile, Config) -> - lager:info("rtdev:update_app_config_file(~s, ~p)", [ConfigFile, Config]), - - BaseConfig = case file:consult(ConfigFile) of - {ok, [ValidConfig]} -> - ValidConfig; - {error, enoent} -> - [] - end, - MergeA = orddict:from_list(Config), - MergeB = orddict:from_list(BaseConfig), - NewConfig = - orddict:merge(fun(_, VarsA, VarsB) -> - MergeC = orddict:from_list(VarsA), - MergeD = orddict:from_list(VarsB), - orddict:merge(fun(_, ValA, _ValB) -> - ValA - end, MergeC, MergeD) - end, MergeA, MergeB), - NewConfigOut = io_lib:format("~p.", [NewConfig]), - ?assertEqual(ok, file:write_file(ConfigFile, NewConfigOut)), - ok. - -%% Appropriate backend will be set by rtcs later. -get_backends() -> - cs_multi_backend. - -node_path(Node) -> - N = node_id(Node), - Path = relpath(node_version(N)), - lists:flatten(io_lib:format("~s/dev/dev~b", [Path, N])). - -create_dirs(Nodes) -> - Snmp = [node_path(Node) ++ "/data/snmp/agent/db" || Node <- Nodes], - [?assertCmd("mkdir -p " ++ Dir) || Dir <- Snmp]. - -clean_data_dir(Nodes, SubDir) when is_list(Nodes) -> - DataDirs = [node_path(Node) ++ "/data/" ++ SubDir || Node <- Nodes], - lists:foreach(fun rm_dir/1, DataDirs). - -rm_dir(Dir) -> - lager:info("Removing directory ~s", [Dir]), - ?assertCmd("rm -rf " ++ Dir), - ?assertEqual(false, filelib:is_dir(Dir)). - -add_default_node_config(Nodes) -> - case rt_config:get(rt_default_config, undefined) of - undefined -> ok; - Defaults when is_list(Defaults) -> - rt:pmap(fun(Node) -> - update_app_config(Node, Defaults) - end, Nodes), - ok; - BadValue -> - lager:error("Invalid value for rt_default_config : ~p", [BadValue]), - throw({invalid_config, {rt_default_config, BadValue}}) - end. - -deploy_nodes(NodeConfig) -> - Path = relpath(root), - lager:info("Riak path: ~p", [Path]), - NumNodes = length(NodeConfig), - NodesN = lists:seq(1, NumNodes), - Nodes = [?DEV(N) || N <- NodesN], - NodeMap = orddict:from_list(lists:zip(Nodes, NodesN)), - {Versions, Configs} = lists:unzip(NodeConfig), - VersionMap = lists:zip(NodesN, Versions), - - %% Check that you have the right versions available - [ check_node(Version) || Version <- VersionMap ], - rt_config:set(rt_nodes, NodeMap), - rt_config:set(rt_versions, VersionMap), - - create_dirs(Nodes), - - %% Set initial config - add_default_node_config(Nodes), - rt:pmap(fun({_, default}) -> - ok; - ({Node, Config}) -> - update_app_config(Node, Config) - end, - lists:zip(Nodes, Configs)), - - %% create snmp dirs, for EE - create_dirs(Nodes), - - %% Start nodes - %%[run_riak(N, relpath(node_version(N)), "start") || N <- Nodes], - rt:pmap(fun(N) -> rtdev:run_riak(N, relpath(node_version(N)), "start") end, NodesN), - - %% Ensure nodes started - [ok = rt:wait_until_pingable(N) || N <- Nodes], - - %% %% Enable debug logging - %% [rpc:call(N, lager, set_loglevel, [lager_console_backend, debug]) || N <- Nodes], - - %% We have to make sure that riak_core_ring_manager is running before we can go on. - [ok = rt:wait_until_registered(N, riak_core_ring_manager) || N <- Nodes], - - %% Ensure nodes are singleton clusters - [ok = rt:check_singleton_node(?DEV(N)) || {N, Version} <- VersionMap, - Version /= "0.14.2"], - - lager:info("Deployed nodes: ~p", [Nodes]), - Nodes. - -stop_all(DevPath) -> - case filelib:is_dir(DevPath) of - true -> - Devs = filelib:wildcard(DevPath ++ "/{dev,stanchion}*"), - - %% Works, but I'd like it to brag a little more about it. - Stop = fun(C) -> - Cmd = stop_command(C), - [Output | _Tail] = string:tokens(os:cmd(Cmd), "\n"), - Status = case Output of - "ok" -> "ok"; - _ -> "wasn't running" - end, - lager:info("Stopping Node... ~s ~~ ~s.", [Cmd, Status]) - end, - [Stop(D) || D <- Devs]; - _ -> lager:info("~s is not a directory.", [DevPath]) - end, - ok. - -stop_command(C) -> - IsRiakCS = string:str(C, "riak_cs"), - IsStanchion = string:str(C, "stanchion"), - if - IsRiakCS > 0 -> - C ++ "/bin/riak-cs stop"; - IsStanchion > 0 -> - C ++ "/bin/stanchion stop"; - true -> - C ++ "/bin/riak stop" - end. - -stop(Node) -> - RiakPid = rpc:call(Node, os, getpid, []), - N = node_id(Node), - rtdev:run_riak(N, relpath(node_version(N)), "stop"), - F = fun(_N) -> - os:cmd("kill -0 " ++ RiakPid) =/= [] - end, - ?assertEqual(ok, rt:wait_until(Node, F)), - ok. - -start(Node) -> - N = node_id(Node), - rtdev:run_riak(N, relpath(node_version(N)), "start"), - ok. - -attach(Node, Expected) -> - interactive(Node, "attach", Expected). - -attach_direct(Node, Expected) -> - interactive(Node, "attach-direct", Expected). - -console(Node, Expected) -> - interactive(Node, "console", Expected). - -interactive(Node, Command, Exp) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Cmd = rtdev:riakcmd(Path, N, Command), - lager:info("Opening a port for riak ~s.", [Command]), - lager:debug("Calling open_port with cmd ~s", [binary_to_list(iolist_to_binary(Cmd))]), - P = open_port({spawn, binary_to_list(iolist_to_binary(Cmd))}, - [stream, use_stdio, exit_status, binary, stderr_to_stdout]), - interactive_loop(P, Exp). - -interactive_loop(Port, Expected) -> - receive - {Port, {data, Data}} -> - %% We've gotten some data, so the port isn't done executing - %% Let's break it up by newline and display it. - Tokens = string:tokens(binary_to_list(Data), "\n"), - [lager:debug("~s", [Text]) || Text <- Tokens], - - %% Now we're going to take hd(Expected) which is either {expect, X} - %% or {send, X}. If it's {expect, X}, we foldl through the Tokenized - %% data looking for a partial match via rt:str/2. If we find one, - %% we pop hd off the stack and continue iterating through the list - %% with the next hd until we run out of input. Once hd is a tuple - %% {send, X}, we send that test to the port. The assumption is that - %% once we send data, anything else we still have in the buffer is - %% meaningless, so we skip it. That's what that {sent, sent} thing - %% is about. If there were a way to abort mid-foldl, I'd have done - %% that. {sent, _} -> is just a pass through to get out of the fold. - - NewExpected = lists:foldl(fun(X, Expect) -> - [{Type, Text}|RemainingExpect] = case Expect of - [] -> [{done, "done"}|[]]; - E -> E - end, - case {Type, rt:str(X, Text)} of - {expect, true} -> - RemainingExpect; - {expect, false} -> - [{Type, Text}|RemainingExpect]; - {send, _} -> - port_command(Port, list_to_binary(Text ++ "\n")), - [{sent, "sent"}|RemainingExpect]; - {sent, _} -> - Expect; - {done, _} -> - [] - end - end, Expected, Tokens), - %% Now that the fold is over, we should remove {sent, sent} if it's there. - %% The fold might have ended not matching anything or not sending anything - %% so it's possible we don't have to remove {sent, sent}. This will be passed - %% to interactive_loop's next iteration. - NewerExpected = case NewExpected of - [{sent, "sent"}|E] -> E; - E -> E - end, - %% If NewerExpected is empty, we've met all expected criteria and in order to boot - %% Otherwise, loop. - case NewerExpected of - [] -> ?assert(true); - _ -> interactive_loop(Port, NewerExpected) - end; - {Port, {exit_status,_}} -> - %% This port has exited. Maybe the last thing we did was {send, [4]} which - %% as Ctrl-D would have exited the console. If Expected is empty, then - %% We've met every expectation. Yay! If not, it means we've exited before - %% something expected happened. - ?assertEqual([], Expected) - after rt_config:get(rt_max_wait_time) -> - %% interactive_loop is going to wait until it matches expected behavior - %% If it doesn't, the test should fail; however, without a timeout it - %% will just hang forever in search of expected behavior. See also: Parenting - ?assertEqual([], Expected) - end. - -admin(Node, Args, Options) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Cmd = rtdev:riak_admin_cmd(Path, N, Args), - lager:info("Running: ~s", [Cmd]), - Result = execute_admin_cmd(Cmd, Options), - lager:info("~s", [Result]), - {ok, Result}. - -execute_admin_cmd(Cmd, Options) -> - {_ExitCode, Result} = FullResult = wait_for_cmd(spawn_cmd(Cmd)), - case lists:member(return_exit_code, Options) of - true -> - FullResult; - false -> - Result - end. - -riak(Node, Args) -> - N = node_id(Node), - Path = relpath(node_version(N)), - Result = rtdev:run_riak(N, Path, Args), - lager:info("~s", [Result]), - {ok, Result}. - -node_id(Node) -> - NodeMap = rt_config:get(rt_nodes), - orddict:fetch(Node, NodeMap). - -node_version(N) -> - VersionMap = rt_config:get(rt_versions), - orddict:fetch(N, VersionMap). - -spawn_cmd(Cmd) -> - spawn_cmd(Cmd, []). -spawn_cmd(Cmd, Opts) -> - Port = open_port({spawn, Cmd}, [stream, in, exit_status, stderr_to_stdout] ++ Opts), - Port. - -wait_for_cmd(Port) -> - rt:wait_until(node(), - fun(_) -> - receive - {Port, Msg={data, _}} -> - self() ! {Port, Msg}, - false; - {Port, Msg={exit_status, _}} -> - catch port_close(Port), - self() ! {Port, Msg}, - true - after 0 -> - false - end - end), - get_cmd_result(Port, []). - -cmd(Cmd) -> - cmd(Cmd, []). - -cmd(Cmd, Opts) -> - wait_for_cmd(spawn_cmd(Cmd, Opts)). - -get_cmd_result(Port, Acc) -> - receive - {Port, {data, Bytes}} -> - get_cmd_result(Port, [Bytes|Acc]); - {Port, {exit_status, Status}} -> - Output = lists:flatten(lists:reverse(Acc)), - {Status, Output} - after 0 -> - timeout - end. - -check_node({_N, Version}) -> - case proplists:is_defined(Version, rt_config:get(build_paths)) of - true -> ok; - _ -> - lager:error("You don't have Riak ~s installed or configured", [Version]), - erlang:error("You don't have Riak " ++ atom_to_list(Version) ++ " installed or configured") - end. - -set_backend(Backend) -> - lager:info("rtdev:set_backend(~p)", [Backend]), - update_app_config(all, [{riak_kv, [{storage_backend, Backend}]}]), - get_backends(). - -get_version() -> - case file:read_file(relpath(cs_current) ++ "/VERSION") of - {error, enoent} -> unknown; - {ok, Version} -> Version - end. - -teardown() -> - %% Stop all discoverable nodes, not just nodes we'll be using for this test. - rt:pmap(fun(X) -> stop_all(X ++ "/dev") end, - devpaths()). - -whats_up() -> - io:format("Here's what's running...~n"), - - Up = [rpc:call(Node, os, cmd, ["pwd"]) || Node <- nodes()], - [io:format(" ~s~n",[string:substr(Dir, 1, length(Dir)-1)]) || Dir <- Up]. - -devpaths() -> - lists:usort([ DevPath || {Name, DevPath} <- rt_config:get(build_paths), - not lists:member(Name, [root, ee_root, cs_root, stanchion_root]) - ]). - -versions() -> - proplists:get_keys(rt_config:get(build_paths)) -- [root]. - -get_node_logs() -> - lists:flatmap(fun get_node_logs/1, [root, ee_root, cs_root, stanchion_root]). - -get_node_logs(Base) -> - Root = filename:absname(proplists:get_value(Base, ?BUILD_PATHS)), - %% Unlike Riak, Riak CS has multiple things in the root and so we need - %% to distinguish between them. - RootLen = length(filename:dirname(Root)) + 1, %% Remove the leading slash - [ begin - {ok, Port} = file:open(Filename, [read, binary]), - {lists:nthtail(RootLen, Filename), Port} - end || Filename <- filelib:wildcard(Root ++ "/*/dev/dev*/log/*") ]. From 31b1002b3ec59c224ec7acf5267a73bf8070836c Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Tue, 25 Aug 2015 10:20:42 -0400 Subject: [PATCH 51/89] Added rt:create_activate_and_wait_for_bucket_type so that the test can specify it wants all nodes in the cluster to have the type before continuing. Update http_bucket_types to use said function. --- src/rt.erl | 4 ++++ tests/http_bucket_types.erl | 11 +++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/rt.erl b/src/rt.erl index 22560a3ab..3955ca0b4 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -1802,6 +1802,10 @@ create_and_activate_bucket_type(Node, Type, Props) -> ok = rpc:call(Node, riak_core_bucket_type, activate, [Type]), wait_until_bucket_type_status(Type, active, Node). +create_activate_and_wait_for_bucket_type([Node|_Rest]=Cluster, Type, Props) -> + create_and_activate_bucket_type(Node, Type, Props), + wait_until_bucket_type_visible(Cluster, Type). + wait_until_bucket_type_status(Type, ExpectedStatus, Nodes) when is_list(Nodes) -> [wait_until_bucket_type_status(Type, ExpectedStatus, Node) || Node <- Nodes]; wait_until_bucket_type_status(Type, ExpectedStatus, Node) -> diff --git a/tests/http_bucket_types.erl b/tests/http_bucket_types.erl index bc56b9d11..d5994b159 100644 --- a/tests/http_bucket_types.erl +++ b/tests/http_bucket_types.erl @@ -101,10 +101,7 @@ confirm() -> lager:info("custom type get/put test"), %% create a new type - ok = rt:create_and_activate_bucket_type(Node, <<"mytype">>, [{n_val,3}]), - - %% allow cluster metadata some time to propogate - timer:sleep(1000), + ok = rt:create_activate_and_wait_for_bucket_type(Nodes, <<"mytype">>, [{n_val,3}]), lager:info("doing put"), ok = rhc:put(RHC, riakc_obj:new({<<"mytype">>, <<"bucket">>}, @@ -136,7 +133,7 @@ confirm() -> UCBBin = {UnicodeTypeBin, UnicodeBucketBin}, - ok = rt:create_and_activate_bucket_type(Node, UnicodeTypeBin, [{n_val,3}]), + ok = rt:create_activate_and_wait_for_bucket_type(Nodes, UnicodeTypeBin, [{n_val,3}]), lager:info("doing put"), ok = rhc:put(RHC, riakc_obj:new(UCBBin, @@ -225,9 +222,7 @@ confirm() -> %% make sure a newly created type is not affected either %% create a new type - ok = rt:create_and_activate_bucket_type(Node, <<"mynewtype">>, []), - %% allow cluster metadata some time to propogate - timer:sleep(1000), + ok = rt:create_activate_and_wait_for_bucket_type(Nodes, <<"mynewtype">>, []), {ok, BProps11} = rhc:get_bucket_type(RHC, <<"mynewtype">>), From 9ee856546a2020e42c026a0ac2f93cca049235e3 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Tue, 25 Aug 2015 14:39:28 -0400 Subject: [PATCH 52/89] Fail with the bare atom fail `riak_test_runner` does not support tagged tuples for failures. Fix verify_snmp_repl module so that it returns the bare atom `fail` when it fails, and write more detailed error messages to the log instead. --- tests/verify_snmp_repl.erl | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/verify_snmp_repl.erl b/tests/verify_snmp_repl.erl index 71307fd36..1d6c7b1ad 100644 --- a/tests/verify_snmp_repl.erl +++ b/tests/verify_snmp_repl.erl @@ -64,11 +64,18 @@ intercept_riak_snmp_stat_poller(Node) -> wait_for_snmp_stat_poller() -> receive - pass -> pass; - {fail, Reason} -> {fail, Reason}; - X -> {fail, {unknown, X}} + pass -> + pass; + {fail, Reason} -> + lager:error("Failure in wait_for_snmp_stat_poller: ~p~n", [Reason]), + fail; + X -> + lager:error("Unknown failure in wait_for_snmp_stat_poller: ~p~n", [X]), + fail after - 1000 -> {fail, timeout} + 1000 -> + lager:error("Timeout failure in wait_for_snmp_stat_poller."), + fail end. make_nodes(NodeCount, ClusterCount, Config) -> From 8641f6edc6d473754694fba94e4a7770dae4cd2f Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Tue, 25 Aug 2015 16:24:55 -0400 Subject: [PATCH 53/89] Crash test process instead of returning fail atom --- tests/verify_snmp_repl.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/verify_snmp_repl.erl b/tests/verify_snmp_repl.erl index 1d6c7b1ad..4a8a40e96 100644 --- a/tests/verify_snmp_repl.erl +++ b/tests/verify_snmp_repl.erl @@ -68,14 +68,15 @@ wait_for_snmp_stat_poller() -> pass; {fail, Reason} -> lager:error("Failure in wait_for_snmp_stat_poller: ~p~n", [Reason]), - fail; + error({fail, Reason}); X -> lager:error("Unknown failure in wait_for_snmp_stat_poller: ~p~n", [X]), - fail + error(X) after 1000 -> - lager:error("Timeout failure in wait_for_snmp_stat_poller."), - fail + Message = "Timeout waiting for snmp_stat_poller.", + lager:error(Message), + error({timeout, Message}) end. make_nodes(NodeCount, ClusterCount, Config) -> From b71e582dc076072a4abe2c76e38b2906b9f6c2fb Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Wed, 26 Aug 2015 09:21:30 -0400 Subject: [PATCH 54/89] Log when original version of intercepted function fails --- tests/verify_snmp_repl.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/verify_snmp_repl.erl b/tests/verify_snmp_repl.erl index 4a8a40e96..9bab62bbe 100644 --- a/tests/verify_snmp_repl.erl +++ b/tests/verify_snmp_repl.erl @@ -57,6 +57,7 @@ intercept_riak_snmp_stat_poller(Node) -> RiakTestProcess ! pass catch Exception:Reason -> + lager:error("Failure in riak_snmp_stat_poller_orig:set_rows_orig: ~p~n", [{Exception, Reason}]), RiakTestProcess ! {fail, {Exception, Reason}}, error({Exception, Reason}) end From 1f719ab8a51062d91d7b14066dc6ea5a6eba05f1 Mon Sep 17 00:00:00 2001 From: Mike Wallace Date: Fri, 11 Sep 2015 17:06:34 +0100 Subject: [PATCH 55/89] Add copyright and Apache License Version 2.0 --- intercepts/hashtree_intercepts.erl | 20 +++++++++++++++++++ intercepts/init_intercepts.erl | 20 +++++++++++++++++++ intercepts/intercept.erl | 20 +++++++++++++++++++ intercepts/intercept.hrl | 20 +++++++++++++++++++ intercepts/riak_core_broadcast_intercepts.erl | 20 +++++++++++++++++++ .../riak_core_connection_intercepts.erl | 20 +++++++++++++++++++ .../riak_core_handoff_sender_intercepts.erl | 20 +++++++++++++++++++ .../riak_core_ring_manager_intercepts.erl | 20 +++++++++++++++++++ .../riak_core_vnode_master_intercepts.erl | 20 +++++++++++++++++++ .../riak_core_vnode_proxy_sup_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_ensemble_peer_intercepts.erl | 20 +++++++++++++++++++ .../riak_kv_bitcask_backend_intercepts.erl | 20 +++++++++++++++++++ .../riak_kv_eleveldb_backend_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_kv_get_fsm_intercepts.erl | 20 +++++++++++++++++++ .../riak_kv_index_hashtree_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_kv_put_fsm_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_kv_vnode_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_kv_worker_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_object_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_pipe_vnode_intercepts.erl | 20 +++++++++++++++++++ ...riak_repl2_fs_node_reserver_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_repl2_fssource_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_repl2_leader_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_repl2_rtq_intercepts.erl | 20 +++++++++++++++++++ .../riak_repl2_rtsink_conn_intercepts.erl | 20 +++++++++++++++++++ .../riak_repl2_rtsource_helper_intercepts.erl | 20 +++++++++++++++++++ .../riak_repl_aae_source_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_repl_console_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_repl_reduced_intercepts.erl | 20 +++++++++++++++++++ .../riak_repl_ring_handler_intercepts.erl | 20 +++++++++++++++++++ intercepts/riak_repl_util_intercepts.erl | 20 +++++++++++++++++++ 31 files changed, 620 insertions(+) diff --git a/intercepts/hashtree_intercepts.erl b/intercepts/hashtree_intercepts.erl index a9bfb32ce..9439a0319 100644 --- a/intercepts/hashtree_intercepts.erl +++ b/intercepts/hashtree_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(hashtree_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/init_intercepts.erl b/intercepts/init_intercepts.erl index ea423e080..b7c61bb47 100644 --- a/intercepts/init_intercepts.erl +++ b/intercepts/init_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(init_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/intercept.erl b/intercepts/intercept.erl index b907ed072..bf355fbd3 100644 --- a/intercepts/intercept.erl +++ b/intercepts/intercept.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(intercept). %% Export explicit API but also send compile directive to export all %% because some of these private functions are useful in their own diff --git a/intercepts/intercept.hrl b/intercepts/intercept.hrl index 021080504..d292e0ba1 100644 --- a/intercepts/intercept.hrl +++ b/intercepts/intercept.hrl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -define(I_TAG(S), "INTERCEPT: " ++ S). -define(I_INFO(Msg), error_logger:info_msg(?I_TAG(Msg))). -define(I_INFO(Msg, Args), error_logger:info_msg(?I_TAG(Msg), Args)). diff --git a/intercepts/riak_core_broadcast_intercepts.erl b/intercepts/riak_core_broadcast_intercepts.erl index 4e1419e99..9a391bf58 100644 --- a/intercepts/riak_core_broadcast_intercepts.erl +++ b/intercepts/riak_core_broadcast_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_core_broadcast_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_core_connection_intercepts.erl b/intercepts/riak_core_connection_intercepts.erl index 196adbe07..f47ac52ec 100644 --- a/intercepts/riak_core_connection_intercepts.erl +++ b/intercepts/riak_core_connection_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_core_connection_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_core_handoff_sender_intercepts.erl b/intercepts/riak_core_handoff_sender_intercepts.erl index f5fb62ad5..f8e0f2d84 100644 --- a/intercepts/riak_core_handoff_sender_intercepts.erl +++ b/intercepts/riak_core_handoff_sender_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_core_handoff_sender_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_core_ring_manager_intercepts.erl b/intercepts/riak_core_ring_manager_intercepts.erl index dfd4aae37..7f8416d43 100644 --- a/intercepts/riak_core_ring_manager_intercepts.erl +++ b/intercepts/riak_core_ring_manager_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_core_ring_manager_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_core_vnode_master_intercepts.erl b/intercepts/riak_core_vnode_master_intercepts.erl index b581ca769..87e28a29b 100644 --- a/intercepts/riak_core_vnode_master_intercepts.erl +++ b/intercepts/riak_core_vnode_master_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_core_vnode_master_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_core_vnode_proxy_sup_intercepts.erl b/intercepts/riak_core_vnode_proxy_sup_intercepts.erl index 43f44ad13..70ab8e450 100644 --- a/intercepts/riak_core_vnode_proxy_sup_intercepts.erl +++ b/intercepts/riak_core_vnode_proxy_sup_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_core_vnode_proxy_sup_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_ensemble_peer_intercepts.erl b/intercepts/riak_ensemble_peer_intercepts.erl index b434345aa..452e51d51 100644 --- a/intercepts/riak_ensemble_peer_intercepts.erl +++ b/intercepts/riak_ensemble_peer_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_ensemble_peer_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_kv_bitcask_backend_intercepts.erl b/intercepts/riak_kv_bitcask_backend_intercepts.erl index caec3ebf4..dc42cbeb3 100644 --- a/intercepts/riak_kv_bitcask_backend_intercepts.erl +++ b/intercepts/riak_kv_bitcask_backend_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_kv_bitcask_backend_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_kv_eleveldb_backend_intercepts.erl b/intercepts/riak_kv_eleveldb_backend_intercepts.erl index a6fe80f8a..e3feab244 100644 --- a/intercepts/riak_kv_eleveldb_backend_intercepts.erl +++ b/intercepts/riak_kv_eleveldb_backend_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_kv_eleveldb_backend_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_kv_get_fsm_intercepts.erl b/intercepts/riak_kv_get_fsm_intercepts.erl index 87a2393f2..e4119bd73 100644 --- a/intercepts/riak_kv_get_fsm_intercepts.erl +++ b/intercepts/riak_kv_get_fsm_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_kv_get_fsm_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_kv_index_hashtree_intercepts.erl b/intercepts/riak_kv_index_hashtree_intercepts.erl index d608fe69f..14a13ba86 100644 --- a/intercepts/riak_kv_index_hashtree_intercepts.erl +++ b/intercepts/riak_kv_index_hashtree_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_kv_index_hashtree_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_kv_put_fsm_intercepts.erl b/intercepts/riak_kv_put_fsm_intercepts.erl index bb276e06b..441399517 100644 --- a/intercepts/riak_kv_put_fsm_intercepts.erl +++ b/intercepts/riak_kv_put_fsm_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_kv_put_fsm_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_kv_vnode_intercepts.erl b/intercepts/riak_kv_vnode_intercepts.erl index ad3d6c893..153bc3e47 100644 --- a/intercepts/riak_kv_vnode_intercepts.erl +++ b/intercepts/riak_kv_vnode_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_kv_vnode_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_kv_worker_intercepts.erl b/intercepts/riak_kv_worker_intercepts.erl index 2bd1e8d59..745a8153d 100644 --- a/intercepts/riak_kv_worker_intercepts.erl +++ b/intercepts/riak_kv_worker_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_kv_worker_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_object_intercepts.erl b/intercepts/riak_object_intercepts.erl index 7ed350800..dd72b66df 100644 --- a/intercepts/riak_object_intercepts.erl +++ b/intercepts/riak_object_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_object_intercepts). -export([skippable_index_specs/1, skippable_diff_index_specs/2]). diff --git a/intercepts/riak_pipe_vnode_intercepts.erl b/intercepts/riak_pipe_vnode_intercepts.erl index b3e7ca2be..9018c2bf5 100644 --- a/intercepts/riak_pipe_vnode_intercepts.erl +++ b/intercepts/riak_pipe_vnode_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_pipe_vnode_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_repl2_fs_node_reserver_intercepts.erl b/intercepts/riak_repl2_fs_node_reserver_intercepts.erl index 3e3b99e1f..9deed6bf0 100644 --- a/intercepts/riak_repl2_fs_node_reserver_intercepts.erl +++ b/intercepts/riak_repl2_fs_node_reserver_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_repl2_fs_node_reserver_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_repl2_fssource_intercepts.erl b/intercepts/riak_repl2_fssource_intercepts.erl index cf37ff1d9..cf9f69aae 100644 --- a/intercepts/riak_repl2_fssource_intercepts.erl +++ b/intercepts/riak_repl2_fssource_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_repl2_fssource_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_repl2_leader_intercepts.erl b/intercepts/riak_repl2_leader_intercepts.erl index 46571ad86..c02a3c1b0 100644 --- a/intercepts/riak_repl2_leader_intercepts.erl +++ b/intercepts/riak_repl2_leader_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_repl2_leader_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_repl2_rtq_intercepts.erl b/intercepts/riak_repl2_rtq_intercepts.erl index c961a1c29..743d10cdc 100644 --- a/intercepts/riak_repl2_rtq_intercepts.erl +++ b/intercepts/riak_repl2_rtq_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + %% Intercepts functions for the riak_test in ../tests/repl_rt_heartbeat.erl -module(riak_repl2_rtq_intercepts). -compile(export_all). diff --git a/intercepts/riak_repl2_rtsink_conn_intercepts.erl b/intercepts/riak_repl2_rtsink_conn_intercepts.erl index 85a139739..0744c8deb 100644 --- a/intercepts/riak_repl2_rtsink_conn_intercepts.erl +++ b/intercepts/riak_repl2_rtsink_conn_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + %% Intercepts functions for the riak_test in ../tests/repl_rt_heartbeat.erl -module(riak_repl2_rtsink_conn_intercepts). -compile(export_all). diff --git a/intercepts/riak_repl2_rtsource_helper_intercepts.erl b/intercepts/riak_repl2_rtsource_helper_intercepts.erl index d4ac50fc2..2986e553b 100644 --- a/intercepts/riak_repl2_rtsource_helper_intercepts.erl +++ b/intercepts/riak_repl2_rtsource_helper_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + %% Intercepts functions for the riak_test in ../tests/repl_rt_heartbeat.erl -module(riak_repl2_rtsource_helper_intercepts). -compile(export_all). diff --git a/intercepts/riak_repl_aae_source_intercepts.erl b/intercepts/riak_repl_aae_source_intercepts.erl index e12203ee3..11f65a691 100644 --- a/intercepts/riak_repl_aae_source_intercepts.erl +++ b/intercepts/riak_repl_aae_source_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_repl_aae_source_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_repl_console_intercepts.erl b/intercepts/riak_repl_console_intercepts.erl index 736d7efc4..0a81ecc6d 100644 --- a/intercepts/riak_repl_console_intercepts.erl +++ b/intercepts/riak_repl_console_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_repl_console_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_repl_reduced_intercepts.erl b/intercepts/riak_repl_reduced_intercepts.erl index 5ed0fc414..32c4c0ef6 100644 --- a/intercepts/riak_repl_reduced_intercepts.erl +++ b/intercepts/riak_repl_reduced_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + %% @doc Put some bugs on the reduced N mutators. This allows one to %% capture and inspect the objects actually set to the mutate_put and %% mutate_get funcitons. This module is currently used by diff --git a/intercepts/riak_repl_ring_handler_intercepts.erl b/intercepts/riak_repl_ring_handler_intercepts.erl index 460ebda9a..c0ef9a20e 100644 --- a/intercepts/riak_repl_ring_handler_intercepts.erl +++ b/intercepts/riak_repl_ring_handler_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_repl_ring_handler_intercepts). -compile(export_all). -include("intercept.hrl"). diff --git a/intercepts/riak_repl_util_intercepts.erl b/intercepts/riak_repl_util_intercepts.erl index aba688484..8d4dc767a 100644 --- a/intercepts/riak_repl_util_intercepts.erl +++ b/intercepts/riak_repl_util_intercepts.erl @@ -1,3 +1,23 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + -module(riak_repl_util_intercepts). -compile(export_all). -include("intercept.hrl"). From 9bcd4a6c0c499860e002899d393f7901d5ab5766 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Fri, 28 Aug 2015 17:05:00 -0400 Subject: [PATCH 56/89] Induce race condition in riak_snmp_stat_poller There is a race in riak_snmp_stat_poller, which occurs when realtime replication is enabled for more than two clusters. If the replication is enabled quickly enough, then the bug does not manifest itself. This change introduces a sleep between enabling replication between the individual clusters, thus causing the test to fail consistently instead of sporadically. --- tests/verify_snmp_repl.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/verify_snmp_repl.erl b/tests/verify_snmp_repl.erl index 9bab62bbe..d15fcb935 100644 --- a/tests/verify_snmp_repl.erl +++ b/tests/verify_snmp_repl.erl @@ -102,7 +102,8 @@ wait_until_leader_converge({_Name, Nodes}) -> enable_realtime([{_, Node, _}|OtherClusters]) -> lists:foreach( fun({Cluster, _, _}) -> - repl_util:enable_realtime(Node, Cluster) + repl_util:enable_realtime(Node, Cluster), + timer:sleep(1000) end, OtherClusters). From 99e1babac1cadbb8b865659a4ef2dae01a098b90 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 9 Sep 2015 19:42:19 -0400 Subject: [PATCH 57/89] update extractors test to check for possible issue w/ expiry and custom extractors --- tests/yz_extractors.erl | 195 +++++++++++++++++++++++++++++++++++----- 1 file changed, 172 insertions(+), 23 deletions(-) diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl index 252701afc..003f39a94 100644 --- a/tests/yz_extractors.erl +++ b/tests/yz_extractors.erl @@ -27,10 +27,95 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("riakc/include/riakc.hrl"). +-define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). -define(INDEX1, <<"test_idx1">>). -define(BUCKET1, <<"test_bkt1">>). -define(INDEX2, <<"test_idx2">>). -define(BUCKET2, <<"test_bkt2">>). +-define(SCHEMANAME, <<"test">>). +-define(TEST_SCHEMA, +<<" + + + + + + + + + + + + + + + + + +_yz_id + + + + + + + + + + + + + + + + + + + + +">>). +-define(TEST_SCHEMA_UPGRADE, +<<" + + + + + + + + + + + + + + + + + + +_yz_id + + + + + + + + + + + + + + + + + + + + +">>). -define(YZ_CAP, {yokozuna, extractor_map_in_cmd}). -define(GET_MAP_RING_MFA, {yz_extractor, get_map, 1}). -define(GET_MAP_MFA, {yz_extractor, get_map, 0}). @@ -38,6 +123,8 @@ -define(YZ_META_EXTRACTORS, {yokozuna, extractors}). -define(YZ_EXTRACTOR_MAP, yokozuna_extractor_map). -define(NEW_EXTRACTOR, {"application/httpheader", yz_noop_extractor}). +-define(EXTRACTOR_CT, element(1, ?NEW_EXTRACTOR)). +-define(EXTRACTOR_MOD, element(2, ?NEW_EXTRACTOR)). -define(DEFAULT_MAP, [{default, yz_noop_extractor}, {"application/json",yz_json_extractor}, {"application/riak_counter", yz_dt_extractor}, @@ -51,6 +138,13 @@ -define(SEQMAX, 20). -define(CFG, [ + {riak_kv, + [ + %% allow AAE to build trees and exchange rapidly + {anti_entropy_build_limit, {100, 1000}}, + {anti_entropy_concurrency, 8}, + {anti_entropy_tick, 1000} + ]}, {yokozuna, [ {enabled, true} @@ -78,13 +172,12 @@ confirm() -> rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA]), - yokozuna_rt:write_data(Cluster, OldPid, ?INDEX1, ?BUCKET1, GenKeys), + yokozuna_rt:write_data(Cluster, OldPid, ?INDEX1, + {?SCHEMANAME, ?TEST_SCHEMA}, ?BUCKET1, GenKeys), + yokozuna_rt:commit(Cluster, ?INDEX1), ok = rt:stop_tracing(), - %% wait for solr soft commit - timer:sleep(1100), - {ok, BProps} = riakc_pb_socket:get_bucket(OldPid, ?BUCKET1), N = proplists:get_value(n_val, BProps), @@ -108,9 +201,8 @@ confirm() -> ?assertEqual(?DEFAULT_MAP, get_map(Node)), - %% Custom Register - ExtractMap = register_extractor(Node, element(1, ?NEW_EXTRACTOR), - element(2, ?NEW_EXTRACTOR)), + %% %% Custom Register + ExtractMap = register_extractor(Node, ?EXTRACTOR_CT, ?EXTRACTOR_MOD), ?assertEqual(?EXTRACTMAPEXPECT, ExtractMap), @@ -118,7 +210,8 @@ confirm() -> yokozuna_rt:rolling_upgrade(Cluster, current), [rt:assert_capability(ANode, ?YZ_CAP, true) || ANode <- Cluster], - [rt:assert_supported(rt:capability(ANode, all), ?YZ_CAP, [true, false]) || ANode <- Cluster], + [rt:assert_supported(rt:capability(ANode, all), ?YZ_CAP, [true, false]) || + ANode <- Cluster], %% test query count again yokozuna_rt:verify_num_found_query(Cluster, ?INDEX1, KeyCount), @@ -128,13 +221,13 @@ confirm() -> rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA, ?GET_MAP_READTHROUGH_MFA]), - yokozuna_rt:write_data(Cluster, Pid, ?INDEX2, ?BUCKET2, GenKeys), - riakc_pb_socket:stop(Pid), + yokozuna_rt:write_data(Cluster, Pid, ?INDEX2, {?SCHEMANAME, ?TEST_SCHEMA}, + ?BUCKET2, GenKeys), + yokozuna_rt:commit(Cluster, ?INDEX2), ok = rt:stop_tracing(), - %% wait for solr soft commit - timer:sleep(1100), + riakc_pb_socket:stop(Pid), CurrGetMapRingCC = rt:get_call_count(Cluster, ?GET_MAP_RING_MFA), CurrGetMapCC = rt:get_call_count(Cluster, ?GET_MAP_MFA), @@ -148,7 +241,7 @@ confirm() -> ?assert(CurrGetMapCC =< PrevGetMapCC), lager:info("Number of calls to get_map_read_through/0: ~p~n, Number of calls to get_map/0: ~p~n", [CurrGetMapRTCC, CurrGetMapCC]), - ?assert(CurrGetMapRTCC < CurrGetMapCC), + ?assert(CurrGetMapRTCC =< CurrGetMapCC), {_RingVal2, MDVal2} = get_ring_and_cmd_vals(Node, ?YZ_META_EXTRACTORS, ?YZ_EXTRACTOR_MAP), @@ -156,17 +249,11 @@ confirm() -> ?assertEqual(?EXTRACTMAPEXPECT, MDVal2), ?assertEqual(?EXTRACTMAPEXPECT, get_map(Node)), - rt_intercept:add(Node, {yz_noop_extractor, - [{{extract, 1}, extract_httpheader}]}), - rt_intercept:wait_until_loaded(Node), + Packet = <<"GET http://www.google.com HTTP/1.1\n">>, + test_extractor_works(Cluster, Packet), + test_extractor_with_aae_expire(Cluster, ?INDEX2, ?BUCKET2, Packet), - ExpectedExtraction = [{method,'GET'}, - {host,<<"www.google.com">>}, - {uri,<<"/">>}], - ?assertEqual(ExpectedExtraction, - verify_extractor(Node, - <<"GET http://www.google.com HTTP/1.1\n">>, - element(2, ?NEW_EXTRACTOR))), + rt:clean_cluster(Cluster), pass. @@ -194,3 +281,65 @@ get_map(Node) -> verify_extractor(Node, PacketData, Mod) -> rpc:call(Node, yz_extractor, run, [PacketData, Mod]). + +bucket_url({Host,Port}, BName, Key) -> + ?FMT("http://~s:~B/buckets/~s/keys/~s", + [Host, Port, BName, Key]). + +test_extractor_works(Cluster, Packet) -> + [rt_intercept:add(ANode, {yz_noop_extractor, + [{{extract, 1}, extract_httpheader}]}) || + ANode <- Cluster], + [rt_intercept:wait_until_loaded(ANode) || ANode <- Cluster], + + ExpectedExtraction = [{method, 'GET'}, + {host, <<"www.google.com">>}, + {uri, <<"/">>}], + ?assertEqual(ExpectedExtraction, + verify_extractor(rt:select_random(Cluster), Packet, ?EXTRACTOR_MOD)). + +test_extractor_with_aae_expire(Cluster, Index, Bucket, Packet) -> + %% Now make sure we register extractor across all nodes + [register_extractor(ANode, ?EXTRACTOR_CT, ?EXTRACTOR_MOD) || + ANode <- Cluster], + + Key = <<"google">>, + + {Host, Port} = rt:select_random(yokozuna_rt:host_entries( + rt:connection_info( + Cluster))), + URL = bucket_url({Host, Port}, mochiweb_util:quote_plus(Bucket), + mochiweb_util:quote_plus(Key)), + + CT = ?EXTRACTOR_CT, + {ok, "204", _, _} = ibrowse:send_req( + URL, [{"Content-Type", CT}], put, Packet), + + yokozuna_rt:commit(Cluster, Index), + + yokozuna_rt:search_expect({Host, Port}, Index, <<"host">>, + <<"www*">>, 1), + + yokozuna_rt:expire_trees(Cluster), + yokozuna_rt:wait_for_full_exchange_round(Cluster, erlang:now()), + + yokozuna_rt:search_expect({Host, Port}, Index, <<"host">>, + <<"www*">>, 1), + + APid = rt:pbc(rt:select_random(Cluster)), + yokozuna_rt:override_schema(APid, Cluster, Index, ?SCHEMANAME, + ?TEST_SCHEMA_UPGRADE), + + {ok, "204", _, _} = ibrowse:send_req( + URL, [{"Content-Type", CT}], put, Packet), + yokozuna_rt:commit(Cluster, Index), + + yokozuna_rt:search_expect({Host, Port}, Index, <<"method">>, + <<"GET">>, 1), + + yokozuna_rt:expire_trees(Cluster), + yokozuna_rt:wait_for_full_exchange_round(Cluster, erlang:now()), + + yokozuna_rt:search_expect({Host, Port}, Index, <<"method">>, + <<"GET">>, 1), + riakc_pb_socket:stop(APid). From dfcad14eba331e4976d6006ba52c0292c6382d6d Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 9 Sep 2015 19:42:36 -0400 Subject: [PATCH 58/89] update yz tests to use commit fn instead of sleep --- src/yokozuna_rt.erl | 27 +++++++++++++++++--- tests/yz_core_properties_create_unload.erl | 8 +++--- tests/yz_crdt.erl | 3 +-- tests/yz_default_bucket_type_upgrade.erl | 5 ++-- tests/yz_ensemble.erl | 10 ++++---- tests/yz_handoff.erl | 2 +- tests/yz_schema_change_reset.erl | 29 ++++++++-------------- 7 files changed, 48 insertions(+), 36 deletions(-) diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl index 7063ec0db..15e4fe4dd 100644 --- a/src/yokozuna_rt.erl +++ b/src/yokozuna_rt.erl @@ -23,8 +23,10 @@ -include("yokozuna_rt.hrl"). -export([check_exists/2, + commit/2, expire_trees/1, host_entries/1, + override_schema/5, remove_index_dirs/2, rolling_upgrade/2, rolling_upgrade/3, @@ -48,6 +50,7 @@ -type json_string() :: atom | string() | binary(). -define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). +-define(SOFTCOMMIT, 1000). -spec host_entries(rt:conn_info()) -> [{host(), portnum()}]. host_entries(ClusterConnInfo) -> @@ -289,7 +292,7 @@ search(Type, {Host, Port}, Index, Name0, Term0) -> verify_count_http(Expected, Resp) -> Count = get_count_http(Resp), lager:info("Expected: ~p, Actual: ~p", [Expected, Count]), - Expected == Count. + ?assertEqual(Expected, Count). -spec get_count_http(json_string()) -> count(). get_count_http(Resp) -> @@ -338,8 +341,8 @@ create_and_set_index(Cluster, Pid, Bucket, Index) -> schema_name()) -> ok. create_and_set_index(Cluster, Pid, Bucket, Index, Schema) -> %% Create a search index and associate with a bucket - lager:info("Create a search index ~s with a custom schema named ~s and - associate it with bucket ~s", [Index, Schema, Bucket]), + lager:info("Create a search index ~s with a custom schema named ~s and " ++ + "associate it with bucket ~s", [Index, Schema, Bucket]), ok = riakc_pb_socket:create_search_index(Pid, Index, Schema, []), %% For possible legacy upgrade reasons, wrap create index in a wait wait_for_index(Cluster, Index), @@ -360,3 +363,21 @@ internal_solr_url(Host, Port, Index, Name, Term, Shards) -> quote_unicode(Value) -> mochiweb_util:quote_plus(binary_to_list( unicode:characters_to_binary(Value))). + +-spec commit([node()], index_name()) -> ok. +commit(Nodes, Index) -> + %% Wait for yokozuna index to trigger, then force a commit + timer:sleep(?SOFTCOMMIT), + lager:info("Commit search writes to ~s at softcommit (default) ~p", + [Index, ?SOFTCOMMIT]), + rpc:multicall(Nodes, yz_solr, commit, [Index]), + ok. + +-spec override_schema(pid(), [node()], index_name(), schema_name(), string()) -> + {ok, [node()]}. +override_schema(Pid, Cluster, Index, Schema, RawUpdate) -> + lager:info("Overwrite schema with updated schema"), + ok = riakc_pb_socket:create_search_schema(Pid, Schema, RawUpdate), + yokozuna_rt:wait_for_schema(Cluster, Schema, RawUpdate), + [Node|_] = Cluster, + {ok, _} = rpc:call(Node, yz_index, reload, [Index]). diff --git a/tests/yz_core_properties_create_unload.erl b/tests/yz_core_properties_create_unload.erl index 29a6180d3..70bf967b7 100644 --- a/tests/yz_core_properties_create_unload.erl +++ b/tests/yz_core_properties_create_unload.erl @@ -67,7 +67,7 @@ confirm() -> %% Write keys and wait for soft commit lager:info("Writing ~p keys", [KeyCount]), [ok = rt:pbc_write(Pid, ?BUCKET, Key, Key, "text/plain") || Key <- Keys], - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), verify_count(Pid, KeyCount), @@ -87,7 +87,7 @@ test_core_props_removal(Cluster, RandNodes, KeyCount, Pid) -> lager:info("Write one more piece of data"), ok = rt:pbc_write(Pid, ?BUCKET, <<"foo">>, <<"foo">>, "text/plain"), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), verify_count(Pid, KeyCount + 1). @@ -102,7 +102,7 @@ test_remove_index_dirs(Cluster, RandNodes, KeyCount, Pid) -> lager:info("Write second piece of data"), ok = rt:pbc_write(Pid, ?BUCKET, <<"food">>, <<"foody">>, "text/plain"), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), verify_count(Pid, KeyCount + 2). @@ -121,7 +121,7 @@ test_remove_segment_infos_and_rebuild(Cluster, RandNodes, KeyCount, Pid) -> lager:info("Write third piece of data"), ok = rt:pbc_write(Pid, ?BUCKET, <<"baz">>, <<"bar">>, "text/plain"), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), verify_count(Pid, KeyCount + 3). diff --git a/tests/yz_crdt.erl b/tests/yz_crdt.erl index 815f9bd98..1a103815e 100644 --- a/tests/yz_crdt.erl +++ b/tests/yz_crdt.erl @@ -59,8 +59,7 @@ confirm() -> ?KEY, riakc_map:to_op(Map2)), - %% Wait for yokozuna index to trigger. - timer:sleep(1000), + yokozuna_rt:commit(Nodes, ?INDEX), %% Perform simple queries, check for register, set fields. {ok, {search_results, Results1a, _, _}} = riakc_pb_socket:search( diff --git a/tests/yz_default_bucket_type_upgrade.erl b/tests/yz_default_bucket_type_upgrade.erl index 20d197779..2d853f701 100644 --- a/tests/yz_default_bucket_type_upgrade.erl +++ b/tests/yz_default_bucket_type_upgrade.erl @@ -70,8 +70,7 @@ confirm() -> OldPid = rt:pbc(Node), yokozuna_rt:write_data(Cluster, OldPid, ?INDEX, ?BUCKET, GenKeys), - %% wait for solr soft commit - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount), @@ -86,7 +85,7 @@ confirm() -> lager:info("Write one more piece of data"), Pid = rt:pbc(Node), ok = rt:pbc_write(Pid, ?BUCKET, <<"foo">>, <<"foo">>, "text/plain"), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), yokozuna_rt:expire_trees(Cluster), yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 1), diff --git a/tests/yz_ensemble.erl b/tests/yz_ensemble.erl index 5b0361e58..f4a604592 100644 --- a/tests/yz_ensemble.erl +++ b/tests/yz_ensemble.erl @@ -34,26 +34,26 @@ confirm() -> create_index(Node, Index), set_bucket_props(Node, Bucket, Index), - verify_ensemble_delete_support(Node, Bucket, Index), + verify_ensemble_delete_support(Nodes, Bucket, Index), pass. %% @private %% @doc Populates then deletes from SC bucket -verify_ensemble_delete_support(Node, Bucket, Index) -> +verify_ensemble_delete_support(Cluster, Bucket, Index) -> %% Yz only supports UTF-8 compatible keys Keys = [<> || N <- lists:seq(1,2000), not lists:any(fun(E) -> E > 127 end,binary_to_list(<>))], - PBC = rt:pbc(Node), + PBC = rt:pbc(hd(Cluster)), lager:info("Writing ~p keys", [length(Keys)]), [ok = rt:pbc_write(PBC, Bucket, Key, Key, "text/plain") || Key <- Keys], + yokozuna_rt:commit(Cluster, Index), %% soft commit wait, then check that last key is indexed lager:info("Search for keys to verify they exist"), - timer:sleep(1000), LKey = lists:last(Keys), rt:wait_until(fun() -> {M, _} = riakc_pb_socket:search(PBC, Index, query_value(LKey)), @@ -64,7 +64,7 @@ verify_ensemble_delete_support(Node, Bucket, Index) -> lager:info("Deleting keys"), [riakc_pb_socket:delete(PBC, Bucket, Key) || Key <- Keys], - timer:sleep(1000), + yokozuna_rt:commit(Cluster, Index), rt:wait_until(fun() -> case riakc_pb_socket:search(PBC, Index, query_value(LKey)) of {ok,{search_results,Res,_,_}} -> diff --git a/tests/yz_handoff.erl b/tests/yz_handoff.erl index 0c4ac95cd..ab91d3bdb 100644 --- a/tests/yz_handoff.erl +++ b/tests/yz_handoff.erl @@ -78,7 +78,7 @@ confirm() -> Pid = rt:pbc(Node2), yokozuna_rt:write_data(Nodes, Pid, ?INDEX, ?BUCKET, Keys), - timer:sleep(1100), + yokozuna_rt:commit(Nodes, ?INDEX), %% Separate out shards for multiple runs [Shard1|Shards2Rest] = Shards, diff --git a/tests/yz_schema_change_reset.erl b/tests/yz_schema_change_reset.erl index 9980ff69b..c5f98de26 100644 --- a/tests/yz_schema_change_reset.erl +++ b/tests/yz_schema_change_reset.erl @@ -147,8 +147,7 @@ confirm() -> yokozuna_rt:write_data(Cluster, Pid, ?INDEX, {?SCHEMANAME, ?TEST_SCHEMA}, ?BUCKET1, GenKeys), - timer:sleep(1100), - + yokozuna_rt:commit(Cluster, ?INDEX), lager:info("Create and activate map-based bucket type ~s and tie it to search_index ~s", [?TYPE, ?INDEX]), rt:create_and_activate_bucket_type(Node1, ?TYPE, [{datatype, map}, @@ -165,9 +164,9 @@ confirm() -> "application/json"), {ok, _ObjA} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), {ok, _ObjB} = riakc_pb_socket:put(Pid, NewObj1B, [return_head]), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 2), @@ -197,7 +196,7 @@ confirm() -> <<"keyMap1">>, riakc_map:to_op(Map3)), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), assert_search(Pid, Cluster, <<"0_foo_register:44ab">>, {<<"0_foo_register">>, <<"44ab">>}, []), @@ -208,8 +207,7 @@ confirm() -> yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 3), - lager:info("Overwrite schema with updated schema"), - override_schema(Pid, Cluster, ?INDEX, ?SCHEMANAME, ?TEST_SCHEMA_UPDATE), + yokozuna_rt:override_schema(Pid, Cluster, ?INDEX, ?SCHEMANAME, ?TEST_SCHEMA_UPDATE), lager:info("Write and check hello_i at integer per schema update"), @@ -218,7 +216,7 @@ confirm() -> "application/json"), {ok, _Obj2} = riakc_pb_socket:put(Pid, NewObj2, [return_head]), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 4), assert_search(Pid, Cluster, <<"hello_i:36">>, {<<"hello_i">>, <<"36">>}, []), @@ -230,6 +228,7 @@ confirm() -> "application/json"), {ok, _Obj3} = riakc_pb_socket:put(Pid, NewObj3, [return_head]), + yokozuna_rt:commit(Cluster, ?INDEX), yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), assert_search(Pid, Cluster, <<"age:3jlkjkl">>, @@ -244,7 +243,7 @@ confirm() -> yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), HP = rt:select_random(yokozuna_rt:host_entries(rt:connection_info(Cluster))), - yokozuna_rt:search_expect(HP, ?INDEX, <<"age">>, <<"*">>, 2), + yokozuna_rt:search_expect(HP, ?INDEX, <<"age">>, <<"*">>, 3), lager:info("Re-Put because AAE won't find a diff even though the types have changed, as it only compares based on bkey currently. @@ -252,7 +251,7 @@ confirm() -> with allow_mult=false... no siblings"), {ok, _Obj4} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), assert_search(Pid, Cluster, <<"age:26">>, {<<"age">>, <<"26">>}, []), @@ -272,7 +271,7 @@ confirm() -> <<"keyMap1">>, riakc_map:to_op(Map5)), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), assert_search(Pid, Cluster, <<"0_foo_register:44ab">>, {<<"0_foo_register">>, <<"44ab">>}, []), assert_search(Pid, Cluster, <<"1_baz_counter:10">>, {<<"1_baz_counter">>, @@ -286,7 +285,7 @@ confirm() -> "application/json"), {ok, _Obj5} = riakc_pb_socket:put(Pid, NewObj5, [return_head]), - timer:sleep(1100), + yokozuna_rt:commit(Cluster, ?INDEX), assert_search(Pid, Cluster, <<"paths.quip:88">>, {<<"paths.quip">>, <<"88">>}, []), @@ -294,12 +293,6 @@ confirm() -> pass. -override_schema(Pid, Cluster, Index, Schema, RawUpdate) -> - ok = riakc_pb_socket:create_search_schema(Pid, Schema, RawUpdate), - yokozuna_rt:wait_for_schema(Cluster, Schema, RawUpdate), - [Node|_] = Cluster, - {ok, _} = rpc:call(Node, yz_index, reload, [Index]). - assert_search(Pid, Cluster, Search, SearchExpect, Params) -> F = fun(_) -> lager:info("Searching ~p and asserting it exists", From 35cb49f00259f7b02757863f7d12d3594ac5080e Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Fri, 11 Sep 2015 09:48:18 -0400 Subject: [PATCH 59/89] Verify cluster stats in SNMP Update verify_snmp_repl test to prove that cluster stats are correctly recorded in SNMP. --- tests/verify_snmp_repl.erl | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/verify_snmp_repl.erl b/tests/verify_snmp_repl.erl index d15fcb935..ead7bff2e 100644 --- a/tests/verify_snmp_repl.erl +++ b/tests/verify_snmp_repl.erl @@ -24,11 +24,17 @@ -include_lib("eunit/include/eunit.hrl"). -compile({parse_transform, rt_intercept_pt}). +-define(FIRST_CLUSTER, "cluster-1"). +-define(OTHER_CLUSTERS, ["cluster-2", "cluster-3"]). +-define(CLUSTERS, [?FIRST_CLUSTER] ++ ?OTHER_CLUSTERS). + confirm() -> - Clusters = make_clusters(["cluster-1", "cluster-2", "cluster-3"], 1), + Clusters = make_clusters(?CLUSTERS, 1), [{_, Leader, _}|_] = Clusters, intercept_riak_snmp_stat_poller(Leader), - wait_for_snmp_stat_poller(). + wait_for_snmp_stat_poller(), + verify_snmp_cluster_stats(Clusters), + pass. make_clusters(Names, NodeCount) -> ClusterCount = length(Names), @@ -107,3 +113,17 @@ enable_realtime([{_, Node, _}|OtherClusters]) -> end, OtherClusters). +%% snmpwalk gives the following output containing the OIDs we're interested in: +%% SNMPv2-SMI::enterprises.31130.200.1.1.1.99.108.117.115.116.101.114.45.50 = STRING: "cluster-2" +%% SNMPv2-SMI::enterprises.31130.200.1.1.1.99.108.117.115.116.101.114.45.51 = STRING: "cluster-3" +-define(CLUSTER_OIDS, + [[1,3,6,1,4,1,31130,200,1,1,1,99,108,117,115,116,101,114,45] ++ [Tail] + || Tail <- [50, 51]]). + +verify_snmp_cluster_stats(Clusters) -> + [{_Name, Leader, [_Nodes]} | _Rest] = Clusters, + rpc:call(Leader, riak_core, wait_for_application, [snmp]), + rpc:call(Leader, riak_core, wait_for_application, [riak_snmp]), + ClusterJoins = rpc:call(Leader, snmpa, get, [snmp_master_agent, ?CLUSTER_OIDS]), + ?assertEqual(?OTHER_CLUSTERS, ClusterJoins). + From ea5f62df9a7a352e0f6c71c871c641b2622328c2 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Thu, 24 Sep 2015 10:13:48 -0400 Subject: [PATCH 60/89] Refactor some Yokozuna tests to eliminate duplication Specifically, introduce the `gen_keys/1` and `assert_search/6` functions into yokozuna_rt module and refactor tests to use them instead of copied and pasted code fragments. (cherry picked from commit 49d5c361c51e73b18757961437dcf382b6d162ea) --- src/yokozuna_rt.erl | 24 ++++++++++ tests/yz_default_bucket_type_upgrade.erl | 6 +-- tests/yz_extractors.erl | 6 +-- tests/yz_schema_change_reset.erl | 60 +++++++++++------------- 4 files changed, 54 insertions(+), 42 deletions(-) diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl index 15e4fe4dd..de8b63ec0 100644 --- a/src/yokozuna_rt.erl +++ b/src/yokozuna_rt.erl @@ -25,6 +25,7 @@ -export([check_exists/2, commit/2, expire_trees/1, + gen_keys/1, host_entries/1, override_schema/5, remove_index_dirs/2, @@ -35,6 +36,7 @@ search_expect/5, search_expect/6, search_expect/7, + assert_search/6, verify_num_found_query/3, wait_for_aae/1, wait_for_full_exchange_round/2, @@ -56,6 +58,14 @@ host_entries(ClusterConnInfo) -> [riak_http(I) || {_,I} <- ClusterConnInfo]. +%% @doc Generate `SeqMax' keys. Yokozuna supports only UTF-8 compatible keys. +-spec gen_keys(pos_integer()) -> list(). +gen_keys(SeqMax) -> + [<> || N <- lists:seq(1, SeqMax), + not lists:any( + fun(E) -> E > 127 end, + binary_to_list(<>))]. + %% @doc Write `Keys' via the PB inteface to a `Bucket' and have them %% searchable in an `Index'. -spec write_data([node()], pid(), index_name(), bucket(), [binary()]) -> ok. @@ -264,6 +274,20 @@ search_expect(solr, {Host, Port}, Index, Name0, Term0, Shards, Expect) {ok, "200", _, R} = ibrowse:send_req(URL, [], get, [], Opts), verify_count_http(Expect, R). +assert_search(Pid, Cluster, Index, Search, SearchExpect, Params) -> + F = fun(_) -> + lager:info("Searching ~p and asserting it exists", + [SearchExpect]), + {ok,{search_results,[{_Index,Fields}], _Score, Found}} = + riakc_pb_socket:search(Pid, Index, Search, Params), + ?assert(lists:member(SearchExpect, Fields)), + case Found of + 1 -> true; + 0 -> false + end + end, + rt:wait_until(Cluster, F). + search(HP, Index, Name, Term) -> search(yokozuna, HP, Index, Name, Term). diff --git a/tests/yz_default_bucket_type_upgrade.erl b/tests/yz_default_bucket_type_upgrade.erl index 2d853f701..f3d94e849 100644 --- a/tests/yz_default_bucket_type_upgrade.erl +++ b/tests/yz_default_bucket_type_upgrade.erl @@ -59,11 +59,7 @@ confirm() -> [rt:assert_capability(ANode, ?YZ_CAP, {unknown_capability, ?YZ_CAP}) || ANode <- Cluster], - %% Generate keys, YZ only supports UTF-8 compatible keys - GenKeys = [<> || N <- lists:seq(1, ?SEQMAX), - not lists:any( - fun(E) -> E > 127 end, - binary_to_list(<>))], + GenKeys = yokozuna_rt:gen_keys(?SEQMAX), KeyCount = length(GenKeys), lager:info("KeyCount ~p", [KeyCount]), diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl index 003f39a94..84f7d8b5e 100644 --- a/tests/yz_extractors.erl +++ b/tests/yz_extractors.erl @@ -163,11 +163,7 @@ confirm() -> OldPid = rt:pbc(Node), - %% Generate keys, YZ only supports UTF-8 compatible keys - GenKeys = [<> || N <- lists:seq(1, ?SEQMAX), - not lists:any( - fun(E) -> E > 127 end, - binary_to_list(<>))], + GenKeys = yokozuna_rt:gen_keys(?SEQMAX), KeyCount = length(GenKeys), rt:count_calls(Cluster, [?GET_MAP_RING_MFA, ?GET_MAP_MFA]), diff --git a/tests/yz_schema_change_reset.erl b/tests/yz_schema_change_reset.erl index c5f98de26..26bc17ea4 100644 --- a/tests/yz_schema_change_reset.erl +++ b/tests/yz_schema_change_reset.erl @@ -131,11 +131,7 @@ confirm() -> [Node1|_RestNodes] = Cluster = rt:build_cluster(4, ?CFG), rt:wait_for_cluster_service(Cluster, yokozuna), - %% Generate keys, YZ only supports UTF-8 compatible keys - GenKeys = [<> || N <- lists:seq(1, ?SEQMAX), - not lists:any( - fun(E) -> E > 127 end, - binary_to_list(<>))], + GenKeys = yokozuna_rt:gen_keys(?SEQMAX), KeyCount = length(GenKeys), lager:info("KeyCount ~p", [KeyCount]), @@ -170,8 +166,10 @@ confirm() -> yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 2), - assert_search(Pid, Cluster, <<"age:26">>, {<<"age">>, <<"26">>}, []), - assert_search(Pid, Cluster, <<"age:99">>, {<<"age">>, <<"99">>}, []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"age:26">>, {<<"age">>, <<"26">>}, []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"age:99">>, {<<"age">>, <<"99">>}, []), Map1 = riakc_map:update( {<<"0_foo">>, register}, @@ -197,8 +195,10 @@ confirm() -> riakc_map:to_op(Map3)), yokozuna_rt:commit(Cluster, ?INDEX), - assert_search(Pid, Cluster, <<"0_foo_register:44ab">>, {<<"0_foo_register">>, - <<"44ab">>}, []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"0_foo_register:44ab">>, + {<<"0_foo_register">>, <<"44ab">>}, + []), lager:info("Expire and re-check count before updating schema"), @@ -219,7 +219,8 @@ confirm() -> yokozuna_rt:commit(Cluster, ?INDEX), yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 4), - assert_search(Pid, Cluster, <<"hello_i:36">>, {<<"hello_i">>, <<"36">>}, []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"hello_i:36">>, {<<"hello_i">>, <<"36">>}, []), lager:info("Write and check age at string per schema update"), @@ -231,8 +232,9 @@ confirm() -> yokozuna_rt:commit(Cluster, ?INDEX), yokozuna_rt:verify_num_found_query(Cluster, ?INDEX, KeyCount + 5), - assert_search(Pid, Cluster, <<"age:3jlkjkl">>, - {<<"age">>, <<"3jlkjkl">>}, []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"age:3jlkjkl">>, {<<"age">>, <<"3jlkjkl">>}, + []), lager:info("Expire and re-check count to make sure we're correctly indexed by the new schema"), @@ -253,7 +255,8 @@ confirm() -> {ok, _Obj4} = riakc_pb_socket:put(Pid, NewObj1A, [return_head]), yokozuna_rt:commit(Cluster, ?INDEX), - assert_search(Pid, Cluster, <<"age:26">>, {<<"age">>, <<"26">>}, []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"age:26">>, {<<"age">>, <<"26">>}, []), lager:info("Re-Put Map data by dec/inc counter to account for *change* and allow previously unindexed counter to be searchable"), @@ -272,10 +275,14 @@ confirm() -> riakc_map:to_op(Map5)), yokozuna_rt:commit(Cluster, ?INDEX), - assert_search(Pid, Cluster, <<"0_foo_register:44ab">>, {<<"0_foo_register">>, - <<"44ab">>}, []), - assert_search(Pid, Cluster, <<"1_baz_counter:10">>, {<<"1_baz_counter">>, - <<"10">>}, []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"0_foo_register:44ab">>, + {<<"0_foo_register">>, <<"44ab">>}, + []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"1_baz_counter:10">>, + {<<"1_baz_counter">>, <<"10">>}, + []), lager:info("Test nested json searches w/ unsearched fields ignored"), @@ -286,23 +293,12 @@ confirm() -> {ok, _Obj5} = riakc_pb_socket:put(Pid, NewObj5, [return_head]), yokozuna_rt:commit(Cluster, ?INDEX), - assert_search(Pid, Cluster, <<"paths.quip:88">>, - {<<"paths.quip">>, <<"88">>}, []), + yokozuna_rt:assert_search(Pid, Cluster, ?INDEX, + <<"paths.quip:88">>, + {<<"paths.quip">>, <<"88">>}, + []), riakc_pb_socket:stop(Pid), pass. -assert_search(Pid, Cluster, Search, SearchExpect, Params) -> - F = fun(_) -> - lager:info("Searching ~p and asserting it exists", - [SearchExpect]), - {ok,{search_results,[{_Index,Fields}], _Score, Found}} = - riakc_pb_socket:search(Pid, ?INDEX, Search, Params), - ?assert(lists:member(SearchExpect, Fields)), - case Found of - 1 -> true; - 0 -> false - end - end, - rt:wait_until(Cluster, F). From 0768ebc3bc0447d2163c279756e884a8f2b27cc4 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 9 Oct 2015 17:52:36 -0400 Subject: [PATCH 61/89] yokozuna_rt additions for types and better testing w/ allow-mult=true extractors test --- src/yokozuna_rt.erl | 26 ++++++++++++++++++++------ tests/yz_extractors.erl | 21 ++++++++++++++------- tests/yz_handoff.erl | 3 +-- 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl index de8b63ec0..84e7149b4 100644 --- a/src/yokozuna_rt.erl +++ b/src/yokozuna_rt.erl @@ -23,6 +23,7 @@ -include("yokozuna_rt.hrl"). -export([check_exists/2, + clear_trees/1, commit/2, expire_trees/1, gen_keys/1, @@ -225,6 +226,15 @@ expire_trees(Cluster) -> timer:sleep(100), ok. +%% @doc Expire YZ trees +-spec clear_trees([node()]) -> ok. +clear_trees(Cluster) -> + lager:info("Expire all trees"), + _ = [ok = rpc:call(Node, yz_entropy_mgr, clear_trees, []) + || Node <- Cluster], + ok. + + %% @doc Remove index directories, removing the index. -spec remove_index_dirs([node()], index_name()) -> ok. remove_index_dirs(Nodes, IndexName) -> @@ -360,20 +370,24 @@ create_and_set_index(Cluster, Pid, Bucket, Index) -> ok = riakc_pb_socket:create_search_index(Pid, Index), %% For possible legacy upgrade reasons, wrap create index in a wait wait_for_index(Cluster, Index), - set_index(Pid, Bucket, Index). + set_index(Pid, hd(Cluster), Bucket, Index). -spec create_and_set_index([node()], pid(), bucket(), index_name(), schema_name()) -> ok. create_and_set_index(Cluster, Pid, Bucket, Index, Schema) -> %% Create a search index and associate with a bucket lager:info("Create a search index ~s with a custom schema named ~s and " ++ - "associate it with bucket ~s", [Index, Schema, Bucket]), + "associate it with bucket ~p", [Index, Schema, Bucket]), ok = riakc_pb_socket:create_search_index(Pid, Index, Schema, []), %% For possible legacy upgrade reasons, wrap create index in a wait wait_for_index(Cluster, Index), - set_index(Pid, Bucket, Index). - --spec set_index(pid(), bucket(), index_name()) -> ok. -set_index(Pid, Bucket, Index) -> + set_index(Pid, hd(Cluster), Bucket, Index). + +-spec set_index(pid(), node(), bucket(), index_name()) -> ok. +set_index(_Pid, Node, {BucketType, _Bucket}, Index) -> + lager:info("Create and activate map-based bucket type ~s and tie it to search_index ~s", + [BucketType, Index]), + rt:create_and_activate_bucket_type(Node, BucketType, [{search_index, Index}]); +set_index(Pid, _Node, Bucket, Index) -> ok = riakc_pb_socket:set_search_index(Pid, Bucket, Index). internal_solr_url(Host, Port, Index) -> diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl index 84f7d8b5e..33d6ee8e9 100644 --- a/tests/yz_extractors.erl +++ b/tests/yz_extractors.erl @@ -28,10 +28,12 @@ -include_lib("riakc/include/riakc.hrl"). -define(FMT(S, Args), lists:flatten(io_lib:format(S, Args))). +-define(TYPE1, <<"extractors_in_paradise">>). +-define(TYPE2, <<"extractors_in_paradiso">>). -define(INDEX1, <<"test_idx1">>). --define(BUCKET1, <<"test_bkt1">>). +-define(BUCKET1, {?TYPE1, <<"test_bkt1">>}). -define(INDEX2, <<"test_idx2">>). --define(BUCKET2, <<"test_bkt2">>). +-define(BUCKET2, {?TYPE2, <<"test_bkt2">>}). -define(SCHEMANAME, <<"test">>). -define(TEST_SCHEMA, <<" @@ -278,9 +280,9 @@ get_map(Node) -> verify_extractor(Node, PacketData, Mod) -> rpc:call(Node, yz_extractor, run, [PacketData, Mod]). -bucket_url({Host,Port}, BName, Key) -> - ?FMT("http://~s:~B/buckets/~s/keys/~s", - [Host, Port, BName, Key]). +bucket_url({Host,Port}, {BType, BName}, Key) -> + ?FMT("http://~s:~B/types/~s/buckets/~s/keys/~s", + [Host, Port, BType, BName, Key]). test_extractor_works(Cluster, Packet) -> [rt_intercept:add(ANode, {yz_noop_extractor, @@ -304,7 +306,7 @@ test_extractor_with_aae_expire(Cluster, Index, Bucket, Packet) -> {Host, Port} = rt:select_random(yokozuna_rt:host_entries( rt:connection_info( Cluster))), - URL = bucket_url({Host, Port}, mochiweb_util:quote_plus(Bucket), + URL = bucket_url({Host, Port}, Bucket, mochiweb_util:quote_plus(Key)), CT = ?EXTRACTOR_CT, @@ -326,8 +328,13 @@ test_extractor_with_aae_expire(Cluster, Index, Bucket, Packet) -> yokozuna_rt:override_schema(APid, Cluster, Index, ?SCHEMANAME, ?TEST_SCHEMA_UPGRADE), + {ok, "200", RHeaders, _} = ibrowse:send_req(URL, [{"Content-Type", CT}], get, + [], []), + VC = proplists:get_value("X-Riak-Vclock", RHeaders), + {ok, "204", _, _} = ibrowse:send_req( - URL, [{"Content-Type", CT}], put, Packet), + URL, [{"Content-Type", CT}, {"X-Riak-Vclock", VC}], + put, Packet), yokozuna_rt:commit(Cluster, Index), yokozuna_rt:search_expect({Host, Port}, Index, <<"method">>, diff --git a/tests/yz_handoff.erl b/tests/yz_handoff.erl index ab91d3bdb..4c5b1af6d 100644 --- a/tests/yz_handoff.erl +++ b/tests/yz_handoff.erl @@ -99,8 +99,7 @@ confirm() -> join_node = Node1, admin_node = Node2}], - %% Run Shell Script to count/test # of replicas and leave/join - %% nodes from the cluster + %% Run set of leave/join trials and count/test #'s from the cluster [[begin check_data(Nodes, KeyCount, BucketURL, SearchURL, State), check_counts(Pid, KeyCount, BucketURL) From 753940a8e3baec20050641ac4b7b267b358bab92 Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Wed, 18 Nov 2015 12:50:14 -0500 Subject: [PATCH 62/89] sure up possible race between upgrade and first check (cherry picked from commit 867dc1a0364e8a19e1a929dd9f3385f46703cfc4) --- tests/yz_extractors.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl index 33d6ee8e9..cf596c43c 100644 --- a/tests/yz_extractors.erl +++ b/tests/yz_extractors.erl @@ -207,6 +207,8 @@ confirm() -> %% Upgrade yokozuna_rt:rolling_upgrade(Cluster, current), + [rt:wait_until_ready(ANode) || ANode <- Cluster], + [rt:assert_capability(ANode, ?YZ_CAP, true) || ANode <- Cluster], [rt:assert_supported(rt:capability(ANode, all), ?YZ_CAP, [true, false]) || ANode <- Cluster], From 2be9c2f83b16c9fc764246e5f026b3374df6e417 Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Mon, 23 Nov 2015 11:09:21 -0500 Subject: [PATCH 63/89] Fix race in repl_rt_heartbeat due to short timeout One particular timeout in the repl_rt_heartbeat test was slightly too short, which could cause us to occasionally hit a false positive on this test if various timings lined up just right. This PR bumps up the timeout, which should prevent this from happening again. I would really like to do a proper fix for this, which would use intercepts or something to confirm that the actual timeout is being hit in the code...but we don't really have time for that, and a half fix is better than no fix I suppose. --- tests/repl_rt_heartbeat.erl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/repl_rt_heartbeat.erl b/tests/repl_rt_heartbeat.erl index 7306404a1..88e004030 100644 --- a/tests/repl_rt_heartbeat.erl +++ b/tests/repl_rt_heartbeat.erl @@ -65,8 +65,14 @@ confirm() -> suspend_heartbeat_messages(LeaderA), %% sleep longer than the HB timeout interval to force re-connection; - %% and give it time to restart the RT connection. Wait an extra 2 seconds. - timer:sleep(timer:seconds(?HB_TIMEOUT) + 2000), + %% and give it time to restart the RT connection. + %% Since it's possible we may disable heartbeats right after a heartbeat has been fired, + %% it can take up to 2*?HB_TIMEOUT seconds to detect a missed heartbeat. The extra second + %% is to avoid rare race conditions due to the timeouts lining up exactly. Not the prettiest + %% solution, but it failed so rarely at 2*HB_TIMEOUT, that this should be good enough + %% in practice, and it beats having to write a bunch of fancy intercepts to verify that + %% the timeout has been hit internally. + timer:sleep(timer:seconds(?HB_TIMEOUT*2) + 1000), %% Verify that RT connection has restarted by noting that it's Pid has changed RTConnPid2 = get_rt_conn_pid(LeaderA), From 0d1b0bee9f1c0a99261754d6aae977eec81e2d54 Mon Sep 17 00:00:00 2001 From: Kazuhiro Suzuki Date: Thu, 17 Dec 2015 16:52:43 +0900 Subject: [PATCH 64/89] Add cleanup intercepts function This function purges specified target module, which was created as a proxy, and its backuped original module, then reloads original module. --- intercepts/intercept.erl | 14 +++++++++++++- src/rt_intercept.erl | 6 ++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/intercepts/intercept.erl b/intercepts/intercept.erl index bf355fbd3..a70875257 100644 --- a/intercepts/intercept.erl +++ b/intercepts/intercept.erl @@ -22,7 +22,7 @@ %% Export explicit API but also send compile directive to export all %% because some of these private functions are useful in their own %% right. --export([add/3, add/4]). +-export([add/3, add/4, clean/1]). -compile(export_all). -type abstract_code() :: term(). @@ -69,6 +69,18 @@ add(Target, Intercept, Mapping, OutDir) -> add(Target, Intercept, Mapping) -> add(Target, Intercept, Mapping, undefined). +%% @doc Cleanup proxy and backuped original module +-spec clean(module()) -> ok|{error, term()}. +clean(Target) -> + _ = code:purge(Target), + _ = code:purge(?ORIGINAL(Target)), + case code:load_file(Target) of + {module, Target} -> + ok; + {error, Reason} -> + {error, Reason} + end. + %% @private %% %% @doc Compile the abstract code `AC' and load it into the code server. diff --git a/src/rt_intercept.erl b/src/rt_intercept.erl index abb5890f8..4e2803935 100644 --- a/src/rt_intercept.erl +++ b/src/rt_intercept.erl @@ -78,6 +78,12 @@ add(Node, {Target, Intercept, Mapping}, OutDir) -> NMapping = [transform_anon_fun(M) || M <- Mapping], ok = rpc:call(Node, intercept, add, [Target, Intercept, NMapping, OutDir]). +clean(Node, Targets) when is_list(Targets) -> + [ok = clean(Node, T) || T <- Targets], + ok; +clean(Node, Target) -> + ok = rpc:call(Node, intercept, clean, [Target]). + %% The following function transforms anonymous function mappings passed %% from an Erlang shell. Anonymous intercept functions from compiled code %% require the developer to supply free variables themselves, and also From 6157fe2ca504fa2bcf2967e08bacf619c13c6b6c Mon Sep 17 00:00:00 2001 From: Kazuhiro Suzuki Date: Fri, 18 Dec 2015 11:39:16 +0900 Subject: [PATCH 65/89] Update spec --- intercepts/intercept.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/intercepts/intercept.erl b/intercepts/intercept.erl index a70875257..7d8bf119c 100644 --- a/intercepts/intercept.erl +++ b/intercepts/intercept.erl @@ -54,7 +54,7 @@ %% functions. %% %% E.g. `[{{update_perform,2}, sleep_update_perform}]' --spec add(module(), module(), mapping()) -> ok. +-spec add(module(), module(), mapping(), string()) -> ok. add(Target, Intercept, Mapping, OutDir) -> Original = ?ORIGINAL(Target), TargetAC = get_abstract_code(Target), @@ -66,6 +66,7 @@ add(Target, Intercept, Mapping, OutDir) -> ok = compile_and_load(Original, OrigAC, OutDir), ok = compile_and_load(Target, ProxyAC, OutDir). +-spec add(module(), module(), mapping()) -> ok. add(Target, Intercept, Mapping) -> add(Target, Intercept, Mapping, undefined). From dca01640cea26f5bd9683dd8699cdb27a361b83b Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 25 Jan 2016 17:21:14 -0800 Subject: [PATCH 66/89] Syntax error in .riak_test.config --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 02987ce08..de9680283 100644 --- a/README.md +++ b/README.md @@ -181,8 +181,8 @@ to tell riak_test about them. The method of choice is to create a {rtdev_path, [{root, "/home/you/rt/riak"}, {current, "/home/you/rt/riak/current"}, {previous, "/home/you/rt/riak/riak-2.0.6"}, - {legacy, "/home/you/rt/riak/riak-1.4.12"} - {'2.0.2', "/home/you/rt/riak/riak-2.0.2"} + {legacy, "/home/you/rt/riak/riak-1.4.12"}, + {'2.0.2', "/home/you/rt/riak/riak-2.0.2"}, {'2.0.4', "/home/you/rt/riak/riak-2.0.4"} ]} ]}. From 9b521b9d7def64297634e5951bcc3b0f999c99f1 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 26 Jan 2016 04:59:05 -0800 Subject: [PATCH 67/89] Ensure git adds everything and ignores any global ignore settings --- bin/rtdev-install.sh | 2 +- bin/rtdev-setup-releases.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/rtdev-install.sh b/bin/rtdev-install.sh index 00ed12001..920ac9ecc 100755 --- a/bin/rtdev-install.sh +++ b/bin/rtdev-install.sh @@ -37,5 +37,5 @@ echo " - Writing $RT_DEST_DIR/$RELEASE/VERSION" echo -n $VERSION > $RT_DEST_DIR/$RELEASE/VERSION cd $RT_DEST_DIR echo " - Reinitializing git state" -git add . +git add --all --force . git commit -a -m "riak_test init" --amend > /dev/null diff --git a/bin/rtdev-setup-releases.sh b/bin/rtdev-setup-releases.sh index f47c25159..cbc1f529b 100755 --- a/bin/rtdev-setup-releases.sh +++ b/bin/rtdev-setup-releases.sh @@ -42,6 +42,6 @@ git init git config user.name "Riak Test" git config user.email "dev@basho.com" -git add . +git add --all --force . git commit -a -m "riak_test init" > /dev/null echo " - Successfully completed initial git commit of $RT_DEST_DIR" From d66bf7546b2d0e91a409a39a9c438563b5f69468 Mon Sep 17 00:00:00 2001 From: Brian Sparrow Date: Wed, 2 Mar 2016 16:47:06 -0500 Subject: [PATCH 68/89] Add tests for read repair object limits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests to verify that object limits are ignored for read repair. This requires an intercept to spoof all puts as read repair since there’s no external API for forcing read repair. Intercept used to replace both put/6 and coord_put/6 and simply adds the `rr` property to the put options. --- intercepts/riak_kv_vnode_intercepts.erl | 6 +++ tests/verify_object_limits.erl | 50 +++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/intercepts/riak_kv_vnode_intercepts.erl b/intercepts/riak_kv_vnode_intercepts.erl index 153bc3e47..c6b3fba70 100644 --- a/intercepts/riak_kv_vnode_intercepts.erl +++ b/intercepts/riak_kv_vnode_intercepts.erl @@ -159,3 +159,9 @@ corrupting_handle_handoff_data(BinObj0, State) -> corrupt_binary(O) -> crypto:rand_bytes(byte_size(O)). + +put_as_readrepair(Preflist, BKey, Obj, ReqId, StartTime, Options) -> + ?M:put_orig(Preflist, BKey, Obj, ReqId, StartTime, [rr | Options]). + +coord_put_as_readrepair(Preflist, BKey, Obj, ReqId, StartTime, Options) -> + ?M:coord_put_orig(Preflist, BKey, Obj, ReqId, StartTime, [rr | Options]). diff --git a/tests/verify_object_limits.erl b/tests/verify_object_limits.erl index fd8af35dc..1661042fd 100644 --- a/tests/verify_object_limits.erl +++ b/tests/verify_object_limits.erl @@ -36,6 +36,7 @@ confirm() -> [Node1] = rt:build_cluster(1, [{riak_kv, [ {ring_creation_size, 8}, + {anti_entropy, {off,[]}}, {max_object_size, ?MAX_SIZE}, {warn_object_size, ?WARN_SIZE}, {max_siblings, ?MAX_SIBLINGS}, @@ -51,6 +52,9 @@ confirm() -> [{allow_mult, true}])), verify_size_limits(C, Node1), verify_sibling_limits(C, Node1), + lager:notice("Starting readrepair section of test"), + verify_readrepair_ignore_max_size(C, Node1), + verify_readrepair_ignore_max_sib(C, Node1), pass. verify_size_limits(C, Node1) -> @@ -128,3 +132,49 @@ verify_sibling_limits(C, Node1) -> lager:info("Result when too many siblings : ~p", [Res]), ?assertMatch({error,_}, Res), ok. + +verify_readrepair_ignore_max_size(C, Node1) -> + % Add intercept to treat all vnode puts as readrepairs + Intercept = {riak_kv_vnode, [{{put, 6}, put_as_readrepair},{{coord_put,6}, coord_put_as_readrepair}]}, + ok = rt_intercept:add(Node1, Intercept), + timer:sleep(100), + % Do put with value greater than max size and confirm warning + lager:info("Checking readrepair put of size ~p, expecting ok result and log warning", [?MAX_SIZE*2]), + K = <<"rrsizetest">>, + V = <<0:(?MAX_SIZE*2)/integer-unit:8>>, + O = riakc_obj:new(?BUCKET, K, V), + ?assertMatch(ok, riakc_pb_socket:put(C, O)), + verify_size_write_warning(Node1, K, ?MAX_SIZE*2), + % Clean intercept + ok = rt_intercept:clean(Node1, riak_kv_vnode), + ok. + +verify_readrepair_ignore_max_sib(C, Node1) -> + lager:info("Checking sibling warning on readrepair above max siblings=~p", [?MAX_SIBLINGS]), + K = <<"rrsibtest">>, + V = <<"sibtest">>, + O = riakc_obj:new(?BUCKET, K, V), + % Force sibling error + [?assertMatch(ok, riakc_pb_socket:put(C, O)) + || _ <- lists:seq(1, ?MAX_SIBLINGS)], + Res = riakc_pb_socket:put(C, O), + lager:info("Result when too many siblings : ~p", [Res]), + ?assertMatch({error,_}, Res), + % Add intercept to spoof writes as readrepair + Intercept = {riak_kv_vnode, [{{put, 6}, put_as_readrepair},{{coord_put,6}, coord_put_as_readrepair}]}, + ok = rt_intercept:add(Node1, Intercept), + timer:sleep(100), + % Verify readrepair writes return ok and log warning + lager:info("Verifying succesful put above max_siblings with readrepair"), + ?assertMatch(ok, riakc_pb_socket:put(C, O)), + P = io_lib:format("warning.*siblings.*~p.*~p.*(~p)", + [?BUCKET, K, ?MAX_SIBLINGS+1]), + Found = rt:expect_in_log(Node1, P), + lager:info("Looking for sibling warning: ~p", [Found]), + ?assertEqual(true, Found), + % Clean intercept + ok = rt_intercept:clean(Node1, riak_kv_vnode), + ok. + + + From 1d6ab06cfb7e17a181f62bcfb670b548cd033281 Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Fri, 4 Mar 2016 15:36:15 -0500 Subject: [PATCH 69/89] Remove unnecessary sleeps in verify_object_limits Discussed this with Brian and turned out they were just left over from some debugging he'd been doing earlier. I tried the test without them and it works fine. --- tests/verify_object_limits.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/verify_object_limits.erl b/tests/verify_object_limits.erl index 1661042fd..e8a0d4d06 100644 --- a/tests/verify_object_limits.erl +++ b/tests/verify_object_limits.erl @@ -137,7 +137,6 @@ verify_readrepair_ignore_max_size(C, Node1) -> % Add intercept to treat all vnode puts as readrepairs Intercept = {riak_kv_vnode, [{{put, 6}, put_as_readrepair},{{coord_put,6}, coord_put_as_readrepair}]}, ok = rt_intercept:add(Node1, Intercept), - timer:sleep(100), % Do put with value greater than max size and confirm warning lager:info("Checking readrepair put of size ~p, expecting ok result and log warning", [?MAX_SIZE*2]), K = <<"rrsizetest">>, @@ -163,7 +162,6 @@ verify_readrepair_ignore_max_sib(C, Node1) -> % Add intercept to spoof writes as readrepair Intercept = {riak_kv_vnode, [{{put, 6}, put_as_readrepair},{{coord_put,6}, coord_put_as_readrepair}]}, ok = rt_intercept:add(Node1, Intercept), - timer:sleep(100), % Verify readrepair writes return ok and log warning lager:info("Verifying succesful put above max_siblings with readrepair"), ?assertMatch(ok, riakc_pb_socket:put(C, O)), From 4402a7b0617a523f9da5cb7270cf94fcf4feb6e2 Mon Sep 17 00:00:00 2001 From: "Engel A. Sanchez" Date: Fri, 31 Jul 2015 18:24:20 -0400 Subject: [PATCH 70/89] First EQC test for secondary indexing It needs more love to match the functionality of the regular 2i tests, but this is a good beginning. It can show a race in 2i pagination that has been in the code for years. The race does not show up in run in a single node with very few vnodes. I tried 32. Ideally we could use PULSE to make scheduling in the riak nodes less deterministic and help find this kind of issue, but currently PULSE is a single node joint. This test uses a different bucket for each EQC command run. It is not perfect isolation, but it is good enough and very fast compared to taking down the entire cluster in between each run. (cherry picked from commit fa493858373642b4219273fcae3f98cd34f1c7d1) --- tests/verify_2i_eqc.erl | 492 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 492 insertions(+) create mode 100644 tests/verify_2i_eqc.erl diff --git a/tests/verify_2i_eqc.erl b/tests/verify_2i_eqc.erl new file mode 100644 index 000000000..f1fafdfdb --- /dev/null +++ b/tests/verify_2i_eqc.erl @@ -0,0 +1,492 @@ +%% +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(verify_2i_eqc). +-compile(export_all). + +-ifdef(EQC). +-include_lib("riakc/include/riakc.hrl"). +-include_lib("eqc/include/eqc.hrl"). +-include_lib("eqc/include/eqc_fsm.hrl"). +-include_lib("eunit/include/eunit.hrl"). + +-behaviour(riak_test). +-export([confirm/0]). + +-define(NUM_TESTS, 5). +-define(MAX_CLUSTER_SIZE, 1). +-define(MAX_BUCKETS, 1). +-define(BUCKETS, [iolist_to_binary(["bucket", integer_to_binary(N)]) + || N <- lists:seq(1, ?MAX_BUCKETS)]). +-define(MAX_FIELDS, 1). +-define(FIELDS, ["i" ++ integer_to_list(N) || N <- lists:seq(1, ?MAX_FIELDS)]). +-define(VALUE, <<"v">>). + +-type index_field() :: {int | bin, binary()}. +-type index_value() :: binary(). +-type index_pair() :: {index_term(), [index_value()]}. +-type index_data() :: {index_field(), [index_pair()]}. + +-record(bucket_data, { + keys = [] :: [key()], + indexes = [] :: list(index_data()) + }). +-record(state, { + nodes = [], + clients, + bucket, + data = #bucket_data{} + }). + +-record(query, { + bucket, + field, + start_term, + end_term, + page_size, + continuation + }). + +%% Initialize counter used to use a different bucket per run. +init_bucket_counter() -> + ets:new(bucket_table, [named_table, public]), + ets:insert_new(bucket_table, [{bucket_number, 0}]). + +confirm() -> + init_bucket_counter(), + Size = random:uniform(?MAX_CLUSTER_SIZE), + TestingTime = rt_config:get(eqc_testing_time, 120), + lager:info("Will run in cluster of size ~p for ~p seconds.", + [Size, TestingTime]), + Nodes = rt:build_cluster(Size), + %% Run for 2 minutes by default. + ?assert(eqc:quickcheck( + eqc_statem:show_states( + eqc:testing_time(TestingTime, ?MODULE:prop_test(Nodes))))), + pass. + +%% ==================================================================== +%% EQC Properties +%% ==================================================================== +prop_test(Nodes) -> + ?FORALL(Cmds, noshrink(commands(?MODULE)), + ?WHENFAIL( + begin + _ = lager:error("*********************** FAILED!!!!" + "*******************") + end, + ?TRAPEXIT( + begin + lager:info("========================" + " Will run commands with Nodes:~p:", [Nodes]), + [lager:info(" Command : ~p~n", [Cmd]) || Cmd <- Cmds], + {H, _S, Res} = run_commands(?MODULE, Cmds, + [{nodelist, Nodes}]), + lager:info("======================== Ran commands"), + aggregate(zip(state_names(H),command_names(Cmds)), + equals(Res, ok)) + end))). + +%% ==================================================================== +%% Value generators +%% ==================================================================== + +gen_bucket() -> + oneof(?BUCKETS). + +gen_field() -> + oneof(?FIELDS). + +gen_int_term() -> + choose(0, 9999). + +gen_bin_term() -> + binary(). + +gen_key() -> + gen_int_term(). + +to_bin_key(N) -> + iolist_to_binary(io_lib:format("~5..0b", [N])). + +gen_term() -> + oneof([gen_int_term(), gen_bin_term()]). + +gen_page_size() -> + oneof([undefined, gen_small_page_size(), choose(1, 10000)]). + +gen_small_page_size() -> + ?LET(N, nat(), N + 1). + +%% Generate a {bucket, key} for an existing object. +%% Will fail if no objects exist, so the command will not be generated +%% if no key exists. +gen_existing_key(#state{data = #bucket_data {keys = Keys}}) -> + oneof(Keys). + +new_query(Bucket, Field, Term1, Term2, PageSize) when Term1 > Term2 -> + #query{bucket = Bucket, field = Field, + start_term = Term2, end_term = Term1, + page_size = PageSize}; +new_query(Bucket, Field, Term1, Term2, PageSize) -> + #query{bucket = Bucket, field = Field, + start_term = Term1, end_term = Term2, + page_size = PageSize}. + +gen_query(S) -> + oneof([gen_some_query(S), gen_all_query(S), gen_random_query(S)]). + +gen_random_query(#state{bucket = Bucket}) -> + oneof([gen_int_query(Bucket), gen_bin_query(Bucket)]). + +first_term(TermKeys) -> + {Term, _} = hd(TermKeys), + Term. + +last_term(TermKeys) -> + {Term, _} = lists:last(TermKeys), + Term. + +%% Query that includes all terms for a given field. +gen_all_query(#state{bucket = Bucket, data = BData}) -> + ?LET({{{_Type, Field}, Terms}, PageSize}, + {oneof(BData#bucket_data.indexes), gen_page_size()}, + new_query(Bucket, Field, first_term(Terms), last_term(Terms), + PageSize)). + +%% Start from an existing term and end on another. +gen_some_query(#state{bucket = Bucket, data = BData}) -> + ?LET({{{_Type, Field}, Terms}, PageSize}, + {oneof(BData#bucket_data.indexes), gen_page_size()}, + ?LET({{Term1, _}, {Term2, _}}, {oneof(Terms), oneof(Terms)}, + new_query(Bucket, Field, Term1, Term2, PageSize))). + +gen_int_query(Bucket) -> + ?LET({Field, Term1, Term2, PageSize}, + {gen_field(), gen_int_term(), gen_int_term(), gen_page_size()}, + new_query(Bucket, Field, Term1, Term2, PageSize)). + +gen_bin_query(Bucket) -> + ?LET({Field, Term1, Term2, PageSize}, + {gen_field(), gen_bin_term(), gen_bin_term(), gen_page_size()}, + new_query(Bucket, Field, Term1, Term2, PageSize)). + +%% ====================================================== +%% States spec +%% ====================================================== +initial_state() -> + pre_setup_state1. + +initial_state_data() -> + #state{}. + +pre_setup_state1(_S) -> + [{pre_setup_state2, {call, ?MODULE, create_clients, [{var, nodelist}]}}]. + +pre_setup_state2(_S) -> + [{default_state, {call, ?MODULE, choose_bucket, []}}]. + +default_state(S) -> + #state{clients = Clients, nodes = Nodes, bucket = Bucket} = S, + [ + {default_state, {call, ?MODULE, index_objects, + [Clients, Nodes, Bucket, gen_key(), choose(1, 100), + gen_field(), gen_term()]}}, + {default_state, {call, ?MODULE, index_objects2, + [Clients, Nodes, Bucket, gen_key(), choose(1, 100), + gen_field(), gen_term()]}}, + {default_state, {call, ?MODULE, delete_one, + [Clients, Nodes, Bucket, gen_existing_key(S)]}}, + {default_state, {call, ?MODULE, query_range, + [Clients, Nodes, gen_query(S)]}} + ]. + +%% Deletes are rare. Queries are less frequent than indexing objects. +weight(default_state, default_state, {call, _, delete_one, _}) -> + 1; +weight(_, _, _) -> + 100. + +next_state_data(_, _, S, Clients, {call, _, create_clients, [Nodes]}) -> + S#state{nodes = Nodes, clients = Clients}; +next_state_data(_, _, S, Bucket, {call, _, choose_bucket, []}) -> + S#state{bucket = Bucket}; +next_state_data(default_state, default_state, S, _, + {call, _, index_objects, + [_, _, _Bucket, IntKey, NKeys, Field, Term]}) -> + #state{data = BData0} = S, + #bucket_data{keys = Keys0, indexes = Idx0} = BData0, + TField = to_tfield(Field, Term), + Keys1 = to_key_list(IntKey, NKeys), + Keys2 = lists:umerge(Keys0, Keys1), + Idx2 = orddict:update(TField, update_fterm_fn(Term, Keys1), + [{Term, Keys1}], Idx0), + BData1 = BData0#bucket_data{keys = Keys2, indexes = Idx2}, + S1 = S#state{data = BData1}, + S1; +next_state_data(default_state, default_state, S, _, + {call, _, index_objects2, + [_, _, _Bucket, IntKey, NKeys, Field, Term]}) -> + #state{data = BData0} = S, + #bucket_data{keys = Keys0, indexes = Idx0} = BData0, + TField = to_tfield(Field, Term), + Keys1 = to_key_list(IntKey, NKeys), + Keys2 = lists:umerge(Keys0, Keys1), + TermKeys = to_term_keys(NKeys, IntKey, Term), + Idx2 = orddict:update(TField, update_fterm2_fn(TermKeys), + TermKeys, Idx0), + BData1 = BData0#bucket_data{keys = Keys2, indexes = Idx2}, + S1 = S#state{data = BData1}, + S1; +next_state_data(default_state, default_state, S, _, + {call, _, delete_one, [_, _, _Bucket, Key]}) -> + #state{data = BData0} = S, + #bucket_data{keys = Keys0, indexes = Idx0} = BData0, + Keys1 = lists:delete(Key, Keys0), + Idx1 = delete_key_from_idx(Key, Idx0), + BData1 = BData0#bucket_data{keys = Keys1, indexes = Idx1}, + S#state{data = BData1}; +next_state_data(_, _, S, _, _) -> + %% Assume anything not handled leaves the state unchanged. + S. + +precondition(_From, _To, _S, {call, _, _, _}) -> + true. + +postcondition(_, _, S, {call, _, query_range, [_, _, Query]}, {error, Err}) -> + {state, S, query, Query, error, Err}; +postcondition(_, _, S, {call, _, query_range, [_, _, Query]}, Keys) -> + ExpectedKeys = query_state(S, Query), + case lists:usort(Keys) =:= ExpectedKeys of + true -> true; + false -> {state, S, query, Query, expected, ExpectedKeys, actual, Keys} + end; +postcondition(_, _, _, _Call, _) -> + true. + +%% ====================================================== +%% Internal +%% ====================================================== + +to_key_list(BaseKey, N) -> + [Key || Key <- lists:seq(BaseKey, BaseKey + N)]. + +update_keys_fn(NewKeys) -> + fun(OldKeys) -> + lists:umerge(OldKeys, NewKeys) + end. + +update_fterm_fn(Term, Keys) -> + fun(TermKeys) -> + orddict:update(Term, update_keys_fn(Keys), Keys, TermKeys) + end. + +update_fterm2_fn(NewTermKeys) -> + fun(TermKeys) -> + orddict:merge(fun(_K, Keys1, Keys2) -> + lists:umerge(Keys1, Keys2) + end, TermKeys, NewTermKeys) + end. + +update_ft_add_fn(Term, Key) -> + fun(OldTermKeys) -> + orddict:update(Term, update_termkey_add_fn(Key), OldTermKeys) + end. + +update_termkey_add_fn(Key) -> + fun(OldKeys) -> + lists:usort([Key|OldKeys]) + end. + +to_tfield(FieldName, Term) -> + case is_integer(Term) of + true -> {int, FieldName}; + false -> {bin, FieldName} + end. + +to_term_keys(Count, Key, Term) when is_integer(Term) -> + [{Term + N, [Key]} || N <- lists:seq(0, Count - 1)]; +to_term_keys(Count, Key, Term) when is_binary(Term) -> + [begin + Suffix = iolist_to_binary(io_lib:format("~0..5b", [N])), + {<>, [Key]} + end || N <- lists:seq(0, Count - 1)]. + +delete_key_from_term_keys(Key, TermKeys) -> + [{Term, lists:delete(Key, Keys)} || {Term, Keys} <- TermKeys]. + +delete_key_from_idx(Key, Idx) -> + [{Field, delete_key_from_term_keys(Key, TermKeys)} + || {Field, TermKeys} <- Idx]. + +%% Query against the modeled data. +query_state(#state{ data = #bucket_data{indexes = Idx}}, + #query{ field = Field, + start_term = Start, end_term = End }) -> + TField = to_tfield(Field, Start), + %% Collect all keys with terms within the given range, ignore others. + Scanner = fun({Term, Keys}, Acc) when Term >= Start, End >= Term -> + [Keys | Acc]; + ({_Term, _Keys}, Acc) -> + Acc + end, + case orddict:find(TField, Idx) of + error -> + []; + {ok, FieldIdx} -> + KeyGroups = lists:foldl(Scanner, [], FieldIdx), + IntKeys = lists:umerge(KeyGroups), + [to_bin_key(Key) || Key <- IntKeys] + end. + +pb_field_term(Field, Term) when is_integer(Term) -> + {Field ++ "_int", Term}; +pb_field_term(Field, Term) when is_binary(Term) -> + {Field ++ "_bin", Term}. + +pb_field(Field, Term) when is_integer(Term) -> + {integer_index, Field}; +pb_field(Field, Term) when is_binary(Term) -> + {binary_index, Field}. + +%% ====================================================== +%% Transition functions. +%% ====================================================== +create_clients(Nodes) -> + dict:from_list([{Node, rt:pbc(Node)} || Node <- Nodes]). + +%% Returns a different bucket name each time it's called. +choose_bucket() -> + N = ets:update_counter(bucket_table, bucket_number, 1), + NBin = integer_to_binary(N), + <<"bucket", NBin/binary>>. + +index_objects(Clients, Nodes, Bucket, Key0, NKeys, Field, Term) -> + Node = hd(Nodes), + lager:info("Indexing ~p in ~p starting at key ~p " + "under (~p, ~p) on node ~p", + [NKeys, Bucket, Key0, Field, Term, Node]), + PB = dict:fetch(Node, Clients), + FT = pb_field_term(Field, Term), + [begin + Key = to_bin_key(N), + Obj = + case riakc_pb_socket:get(PB, Bucket, Key) of + {ok, O} -> + MD = riakc_obj:get_metadata(O), + IndexMD2 = + case dict:find(<<"index">>, MD) of + {ok, IndexMD} -> + lists:usort([FT | IndexMD]); + error -> + [FT] + end, + MD2 = dict:store(<<"index">>, IndexMD2, MD), + riakc_obj:update_metadata(O, MD2); + {error, notfound} -> + O = riakc_obj:new(Bucket, Key, Key), + MD = dict:from_list([{<<"index">>, [FT]}]), + riakc_obj:update_metadata(O, MD) + end, + ok = riakc_pb_socket:put(PB, Obj) + end || N <- to_key_list(Key0, NKeys)], + ok. + +index_objects2(Clients, Nodes, Bucket, Key0, NKeys, Field, Term0) -> + Node = hd(Nodes), + lager:info("Indexing (2) ~p in ~p starting at key ~p, term ~p " + "under ~p, on node ~p", + [NKeys, Bucket, Key0, Term0, Field, Node]), + PB = dict:fetch(Node, Clients), + [begin + Key = to_bin_key(N), + FT = pb_field_term(Field, Term), + Obj = + case riakc_pb_socket:get(PB, Bucket, Key) of + {ok, O} -> + MD = riakc_obj:get_metadata(O), + IndexMD2 = + case dict:find(<<"index">>, MD) of + {ok, IndexMD} -> + lists:usort([FT | IndexMD]); + error -> + [FT] + end, + MD2 = dict:store(<<"index">>, IndexMD2, MD), + riakc_obj:update_metadata(O, MD2); + {error, notfound} -> + O = riakc_obj:new(Bucket, Key, Key), + MD = dict:from_list([{<<"index">>, [FT]}]), + riakc_obj:update_metadata(O, MD) + end, + ok = riakc_pb_socket:put(PB, Obj) + end || {Term, [N]} <- to_term_keys(NKeys, Key0, Term0)], + ok. + +delete_one(Clients, Nodes, Bucket, IntKey) -> + Key = to_bin_key(IntKey), + Node = hd(Nodes), + PB = dict:fetch(Node, Clients), + lager:info("Deleting key ~p from bucket ~p", [Key, Bucket]), + ok = riakc_pb_socket:delete(PB, Bucket, Key), + ok = rt:wait_until(fun() -> rt:pbc_really_deleted(PB, Bucket, [Key]) end), + ok. + +is_true(true) -> true; +is_true(_) -> false. + +query_range(Clients, Nodes, Query) -> + Node = hd(Nodes), + PB = dict:fetch(Node, Clients), + Keys = lists:sort(query_range_pb(PB, Query, [])), + lager:info("Query ~p, ~p from ~p to ~p, page = ~p, returned ~p keys.", + [Query#query.bucket, Query#query.field, Query#query.start_term, + Query#query.end_term, Query#query.page_size, length(Keys)]), + %% Re-run with page sizes 1 -> 100, verify it's always the same result. + PageChecks = + [begin + Q2 = Query#query{page_size = PSize}, + OKeys = lists:sort(query_range_pb(PB, Q2, [])), + OKeys =:= Keys + end || PSize <- lists:seq(1, 100)], + case lists:all(fun is_true/1, PageChecks) of + true -> + Keys; + false -> + {error, mismatch_when_paged} + end. + +query_range_pb(PB, + #query { bucket = Bucket, field = FieldName, + start_term = Start, end_term = End, + page_size = PageSize, continuation = Cont } = Query, + AccKeys) -> + Field = pb_field(FieldName, Start), + case riakc_pb_socket:get_index_range(PB, Bucket, Field, Start, End, + [{max_results, PageSize}, + {continuation, Cont}]) of + {ok, ?INDEX_RESULTS{keys = Keys, continuation = undefined}} -> + AccKeys ++ Keys; + {ok, ?INDEX_RESULTS{keys = Keys, continuation = Cont1}} -> + Query1 = Query#query{continuation = Cont1}, + query_range_pb(PB, Query1, AccKeys ++ Keys) + end. + +-endif. From 55709dbb18ae4730deb3be65055101ffa259d7e1 Mon Sep 17 00:00:00 2001 From: "Engel A. Sanchez" Date: Tue, 11 Aug 2015 10:46:48 -0400 Subject: [PATCH 71/89] Clean up and improve General clean up and organization, with an eye on making this a template for other EQC tests. Preparing code to handle more than the PB client. (cherry picked from commit 02b5e2128d98679c893e800103a5e580b1f81e84) --- tests/verify_2i_eqc.erl | 645 ++++++++++++++++++++++------------------ 1 file changed, 363 insertions(+), 282 deletions(-) diff --git a/tests/verify_2i_eqc.erl b/tests/verify_2i_eqc.erl index f1fafdfdb..2c50b472f 100644 --- a/tests/verify_2i_eqc.erl +++ b/tests/verify_2i_eqc.erl @@ -1,4 +1,3 @@ -%% %% ------------------------------------------------------------------- %% %% Copyright (c) 2015 Basho Technologies, Inc. @@ -18,6 +17,50 @@ %% under the License. %% %% ------------------------------------------------------------------- +%% @doc EQC test for secondary indexing using eqc_fsm to generate +%% sequences of indexing and query commands. +%% +%% The state machine is very simple. Mostly it: +%% - Indexes a bunch of keys under a single integer or binary term. +%% - Indexes a bunch of keys under an equal number of consecutive integer +%% or binary terms. +%% - Deletes a single item and all its associated index entries. +%% - Generates random queries and verifies results match the model. +%% Notice how we are only checking against the entire set of results and +%% not against each page of the results. I suggest that as an improvement. +%% +%% A couple of dummy states exist just to ensure that each run starts by +%% first creating a bunch of clients, then choosing a new unique bucket. +%% +%% The test model stores a list of keys +%% and index data for a configurable number of fields. The keys are all +%% numeric for simpler presentation and get converted to and from binary +%% as needed. For example, if two objects are created and indexed like this: +%% +%% - key 10, "i1_int" -> 1, "i1_bin" -> "a" +%% - key 20, "i1_int" -> 1, "i1_bin" -> "b" +%% +%% The model data would look like this: +%% +%% keys = [10, 20] +%% indexes = +%% [ +%% {{bin, "i1"}, [ +%% {<<"a">>, [10]}, +%% {<<"b">>, [20]} +%% ]}, +%% {{int, "i1"}, [ +%% {1, [10, 20]} +%% ]} +%% ] +%% +%% All lists in the indexes field are sorted and manipulated using orddict. +%% The indexes data structure is an orddict that maps a typed field to +%% an orddict mapping terms to lists of keys. +%% As in Riak, here "i1_int" and "i1_bin" are the fields, and the values +%% such as 1 or "a" are called terms. +%% +%% ------------------------------------------------------------------- -module(verify_2i_eqc). -compile(export_all). @@ -30,29 +73,22 @@ -behaviour(riak_test). -export([confirm/0]). --define(NUM_TESTS, 5). -define(MAX_CLUSTER_SIZE, 1). --define(MAX_BUCKETS, 1). --define(BUCKETS, [iolist_to_binary(["bucket", integer_to_binary(N)]) - || N <- lists:seq(1, ?MAX_BUCKETS)]). -define(MAX_FIELDS, 1). -define(FIELDS, ["i" ++ integer_to_list(N) || N <- lists:seq(1, ?MAX_FIELDS)]). --define(VALUE, <<"v">>). +-define(CLIENT_TYPES, [pb]). -type index_field() :: {int | bin, binary()}. -type index_value() :: binary(). -type index_pair() :: {index_term(), [index_value()]}. -type index_data() :: {index_field(), [index_pair()]}. --record(bucket_data, { - keys = [] :: [key()], - indexes = [] :: list(index_data()) - }). -record(state, { nodes = [], - clients, + clients = [], bucket, - data = #bucket_data{} + keys = [] :: [key()], + indexes = [] :: list(index_data()) }). -record(query, { @@ -64,29 +100,25 @@ continuation }). -%% Initialize counter used to use a different bucket per run. -init_bucket_counter() -> - ets:new(bucket_table, [named_table, public]), - ets:insert_new(bucket_table, [{bucket_number, 0}]). - confirm() -> + %% Set up monotonic bucket name generator. init_bucket_counter(), Size = random:uniform(?MAX_CLUSTER_SIZE), + %% Run for 2 minutes by default. TestingTime = rt_config:get(eqc_testing_time, 120), lager:info("Will run in cluster of size ~p for ~p seconds.", [Size, TestingTime]), Nodes = rt:build_cluster(Size), - %% Run for 2 minutes by default. ?assert(eqc:quickcheck( - eqc_statem:show_states( - eqc:testing_time(TestingTime, ?MODULE:prop_test(Nodes))))), + eqc:testing_time(TestingTime, ?MODULE:prop_test(Nodes)))), pass. %% ==================================================================== %% EQC Properties %% ==================================================================== prop_test(Nodes) -> - ?FORALL(Cmds, noshrink(commands(?MODULE)), + InitState = #state{nodes = Nodes}, + ?FORALL(Cmds, commands(?MODULE, {initial_state(), InitState}), ?WHENFAIL( begin _ = lager:error("*********************** FAILED!!!!" @@ -97,84 +129,96 @@ prop_test(Nodes) -> lager:info("========================" " Will run commands with Nodes:~p:", [Nodes]), [lager:info(" Command : ~p~n", [Cmd]) || Cmd <- Cmds], - {H, _S, Res} = run_commands(?MODULE, Cmds, - [{nodelist, Nodes}]), + {H, {_SName, S}, Res} = run_commands(?MODULE, Cmds), lager:info("======================== Ran commands"), + %% Each run creates a new pool of clients. Clean up. + close_clients(S#state.clients), + %% Record stats on what commands were generated on + %% successful runs. This is printed after the test + %% finishes. aggregate(zip(state_names(H),command_names(Cmds)), equals(Res, ok)) end))). %% ==================================================================== -%% Value generators +%% Value generators and utilities. %% ==================================================================== -gen_bucket() -> - oneof(?BUCKETS). +gen_node(S) -> + oneof(S#state.nodes). + +gen_client_id(S) -> + {oneof(S#state.nodes), oneof(?CLIENT_TYPES)}. +%% Generates a key in the range 0-999. +%% TODO: How to determine an optimal range for coverage? +%% If too large, we wouldn't update the same key very often, for example. +gen_key() -> + choose(0, 999). + +%% Pick one of a fixed list of possible base field names. gen_field() -> oneof(?FIELDS). +%% Produces either a binary or integer term value. +gen_term() -> + oneof([gen_int_term(), gen_bin_term()]). + +%% Generates, with equal likelihood, either a smallish or a largish integer. gen_int_term() -> - choose(0, 9999). + oneof([int(), largeint()]). +%% Generates a random binary. gen_bin_term() -> binary(). -gen_key() -> - gen_int_term(). +%% Generates a list of integer keys without duplicates. +gen_key_list() -> + ?LET(L, non_empty(list(gen_key())), lists:usort(L)). -to_bin_key(N) -> - iolist_to_binary(io_lib:format("~5..0b", [N])). - -gen_term() -> - oneof([gen_int_term(), gen_bin_term()]). +%% Generates non-empty lists of {Key, Field, Term} triplets. +gen_key_field_terms() -> + non_empty(list({gen_key(), gen_field(), gen_term()})). +%% Produces, with equal likelihood, either no page size, a smallish one or +%% a largish one. gen_page_size() -> - oneof([undefined, gen_small_page_size(), choose(1, 10000)]). + oneof([undefined, gen_small_page_size(), gen_large_page_size()]). +%% Based on EQC's nat() so numbers tend to be smallish. +%% Adjusting with LET to avoid zero, which is invalid. gen_small_page_size() -> ?LET(N, nat(), N + 1). -%% Generate a {bucket, key} for an existing object. -%% Will fail if no objects exist, so the command will not be generated -%% if no key exists. -gen_existing_key(#state{data = #bucket_data {keys = Keys}}) -> - oneof(Keys). +%% Adjusts largeint() to make the result strictly positive. +gen_large_page_size() -> + choose(1, 16#ffffFFFF). -new_query(Bucket, Field, Term1, Term2, PageSize) when Term1 > Term2 -> - #query{bucket = Bucket, field = Field, - start_term = Term2, end_term = Term1, - page_size = PageSize}; -new_query(Bucket, Field, Term1, Term2, PageSize) -> - #query{bucket = Bucket, field = Field, - start_term = Term1, end_term = Term2, - page_size = PageSize}. +%% Chooses one of the keys in the model at random. +gen_existing_key(#state{keys = Keys}) -> + oneof(Keys). -gen_query(S) -> +%% Generates either a query on an integer or binary field that: +%% - Uses a couple of existing terms as start/ends +%% - Includes all terms in the index +%% - Generates start/end terms randomly, which may not span any existing items. +gen_range_query(S) -> oneof([gen_some_query(S), gen_all_query(S), gen_random_query(S)]). gen_random_query(#state{bucket = Bucket}) -> oneof([gen_int_query(Bucket), gen_bin_query(Bucket)]). -first_term(TermKeys) -> - {Term, _} = hd(TermKeys), - Term. - -last_term(TermKeys) -> - {Term, _} = lists:last(TermKeys), - Term. - %% Query that includes all terms for a given field. -gen_all_query(#state{bucket = Bucket, data = BData}) -> +gen_all_query(#state{bucket = Bucket, indexes = Idx}) -> ?LET({{{_Type, Field}, Terms}, PageSize}, - {oneof(BData#bucket_data.indexes), gen_page_size()}, + {oneof(Idx), gen_page_size()}, new_query(Bucket, Field, first_term(Terms), last_term(Terms), PageSize)). -%% Start from an existing term and end on another. -gen_some_query(#state{bucket = Bucket, data = BData}) -> +%% Chooses two existing terms as start and end. +gen_some_query(#state{bucket = Bucket, indexes = Idx}) -> ?LET({{{_Type, Field}, Terms}, PageSize}, - {oneof(BData#bucket_data.indexes), gen_page_size()}, + {oneof(Idx), gen_page_size()}, ?LET({{Term1, _}, {Term2, _}}, {oneof(Terms), oneof(Terms)}, new_query(Bucket, Field, Term1, Term2, PageSize))). @@ -188,6 +232,28 @@ gen_bin_query(Bucket) -> {gen_field(), gen_bin_term(), gen_bin_term(), gen_page_size()}, new_query(Bucket, Field, Term1, Term2, PageSize)). +%% Populates a new query record. For convenience, corrects the order of the +%% start and end terms so that start is always less than or equal to end. +%% That way we don't need any generator tricks for those. +new_query(Bucket, Field, Term1, Term2, PageSize) when Term1 > Term2 -> + #query{bucket = Bucket, field = Field, + start_term = Term2, end_term = Term1, + page_size = PageSize}; +new_query(Bucket, Field, Term1, Term2, PageSize) -> + #query{bucket = Bucket, field = Field, + start_term = Term1, end_term = Term2, + page_size = PageSize}. + +%% First term in a term to keys orddict. +first_term(TermKeys) -> + {Term, _} = hd(TermKeys), + Term. + +%% Last term in a term to keys orddict. +last_term(TermKeys) -> + {Term, _} = lists:last(TermKeys), + Term. + %% ====================================================== %% States spec %% ====================================================== @@ -197,83 +263,81 @@ initial_state() -> initial_state_data() -> #state{}. -pre_setup_state1(_S) -> - [{pre_setup_state2, {call, ?MODULE, create_clients, [{var, nodelist}]}}]. +pre_setup_state1(S) -> + #state{nodes = Nodes} = S, + [{pre_setup_state2, {call, ?MODULE, tx_create_clients, [Nodes]}}]. pre_setup_state2(_S) -> - [{default_state, {call, ?MODULE, choose_bucket, []}}]. + [{default_state, {call, ?MODULE, tx_next_bucket, []}}]. default_state(S) -> - #state{clients = Clients, nodes = Nodes, bucket = Bucket} = S, + #state{clients = Clients, bucket = Bucket} = S, [ - {default_state, {call, ?MODULE, index_objects, - [Clients, Nodes, Bucket, gen_key(), choose(1, 100), - gen_field(), gen_term()]}}, - {default_state, {call, ?MODULE, index_objects2, - [Clients, Nodes, Bucket, gen_key(), choose(1, 100), - gen_field(), gen_term()]}}, - {default_state, {call, ?MODULE, delete_one, - [Clients, Nodes, Bucket, gen_existing_key(S)]}}, - {default_state, {call, ?MODULE, query_range, - [Clients, Nodes, gen_query(S)]}} + {default_state, {call, ?MODULE, tx_index_single_term, + [Clients, gen_client_id(S), Bucket, + gen_key_list(), gen_field(), gen_term()]}}, + {default_state, {call, ?MODULE, tx_index_multi_term, + [Clients, gen_client_id(S), Bucket, + gen_key_field_terms()]}}, + {default_state, {call, ?MODULE, tx_delete_one, + [Clients, gen_client_id(S), Bucket, + gen_existing_key(S)]}}, + {default_state, {call, ?MODULE, tx_query_range, + [Clients, gen_client_id(S), gen_range_query(S)]}} ]. -%% Deletes are rare. Queries are less frequent than indexing objects. -weight(default_state, default_state, {call, _, delete_one, _}) -> +%% Tweak transition weights such that deletes are rare. +%% Indexing a bunch or querying a bunch of items are equally likely. +weight(default_state, default_state, {call, _, tx_delete_one, _}) -> 1; weight(_, _, _) -> 100. -next_state_data(_, _, S, Clients, {call, _, create_clients, [Nodes]}) -> - S#state{nodes = Nodes, clients = Clients}; -next_state_data(_, _, S, Bucket, {call, _, choose_bucket, []}) -> +%% State data mutations for each transition. +next_state_data(_, _, S, Clients, {call, _, tx_create_clients, [_]}) -> + S#state{clients = Clients}; +next_state_data(_, _, S, Bucket, {call, _, tx_next_bucket, []}) -> S#state{bucket = Bucket}; next_state_data(default_state, default_state, S, _, - {call, _, index_objects, - [_, _, _Bucket, IntKey, NKeys, Field, Term]}) -> - #state{data = BData0} = S, - #bucket_data{keys = Keys0, indexes = Idx0} = BData0, - TField = to_tfield(Field, Term), - Keys1 = to_key_list(IntKey, NKeys), - Keys2 = lists:umerge(Keys0, Keys1), - Idx2 = orddict:update(TField, update_fterm_fn(Term, Keys1), - [{Term, Keys1}], Idx0), - BData1 = BData0#bucket_data{keys = Keys2, indexes = Idx2}, - S1 = S#state{data = BData1}, - S1; + {call, _, tx_index_single_term, + [_, _, _, NewKeys, Field, Term]}) -> + #state{keys = Keys0, indexes = Idx0} = S, + Keys1 = lists:umerge(NewKeys, Keys0), + Idx1 = model_index(NewKeys, Field, Term, Idx0), + S#state{keys = Keys1, indexes = Idx1}; next_state_data(default_state, default_state, S, _, - {call, _, index_objects2, - [_, _, _Bucket, IntKey, NKeys, Field, Term]}) -> - #state{data = BData0} = S, - #bucket_data{keys = Keys0, indexes = Idx0} = BData0, - TField = to_tfield(Field, Term), - Keys1 = to_key_list(IntKey, NKeys), - Keys2 = lists:umerge(Keys0, Keys1), - TermKeys = to_term_keys(NKeys, IntKey, Term), - Idx2 = orddict:update(TField, update_fterm2_fn(TermKeys), - TermKeys, Idx0), - BData1 = BData0#bucket_data{keys = Keys2, indexes = Idx2}, - S1 = S#state{data = BData1}, - S1; + {call, _, tx_index_multi_term, + [_, _, _, KeyFieldTerms]}) -> + #state{keys = Keys0, indexes = Idx0} = S, + %% Add to list of keys and dedupe. + NewKeys = [K || {K, _, _} <- KeyFieldTerms], + Keys1 = lists:umerge(NewKeys, Keys0), + Idx1 = model_index(KeyFieldTerms, Idx0), + S#state{keys = Keys1, indexes = Idx1}; next_state_data(default_state, default_state, S, _, - {call, _, delete_one, [_, _, _Bucket, Key]}) -> - #state{data = BData0} = S, - #bucket_data{keys = Keys0, indexes = Idx0} = BData0, + {call, _, tx_delete_one, [_, _, _, Key]}) -> + #state{keys = Keys0, indexes = Idx0} = S, Keys1 = lists:delete(Key, Keys0), - Idx1 = delete_key_from_idx(Key, Idx0), - BData1 = BData0#bucket_data{keys = Keys1, indexes = Idx1}, - S#state{data = BData1}; + Idx1 = model_delete_key(Key, Idx0), + S#state{keys = Keys1, indexes = Idx1}; next_state_data(_, _, S, _, _) -> - %% Assume anything not handled leaves the state unchanged. + %% Any other transition leaves state unchanged. S. +%% No precondition checks. Among other things, that means that shrinking may +%% end up issuing deletes to keys that do not exist, which is harmless. +%% Any indexing, deleting or querying command can be issued at any point +%% in the sequence. precondition(_From, _To, _S, {call, _, _, _}) -> true. -postcondition(_, _, S, {call, _, query_range, [_, _, Query]}, {error, Err}) -> +%% Signal a test failure if there is an explicit error from the query or +%% if the results do not match what is in the model. +postcondition(_, _, S, {call, _, tx_query_range, [_, _, Query]}, {error, Err}) -> {state, S, query, Query, error, Err}; -postcondition(_, _, S, {call, _, query_range, [_, _, Query]}, Keys) -> - ExpectedKeys = query_state(S, Query), +postcondition(_, _, S, {call, _, tx_query_range, [_, _, Query]}, Keys) -> + #state{indexes = Idx} = S, + ExpectedKeys = model_query_range(Query, Idx), case lists:usort(Keys) =:= ExpectedKeys of true -> true; false -> {state, S, query, Query, expected, ExpectedKeys, actual, Keys} @@ -282,64 +346,196 @@ postcondition(_, _, _, _Call, _) -> true. %% ====================================================== -%% Internal +%% State transition functions. %% ====================================================== -to_key_list(BaseKey, N) -> - [Key || Key <- lists:seq(BaseKey, BaseKey + N)]. +%% Returns a dictionary that stores a client object per each node +%% and client type. +%% {Node, Type} -> {Type, Client} +tx_create_clients(Nodes) -> + orddict:from_list([{{N, T}, {T, create_client(N, T)}} + || N <- Nodes, T <- ?CLIENT_TYPES]). -update_keys_fn(NewKeys) -> - fun(OldKeys) -> - lists:umerge(OldKeys, NewKeys) - end. +%% Returns a different bucket name each time it's called. +tx_next_bucket() -> + N = ets:update_counter(bucket_table, bucket_number, 1), + NBin = integer_to_binary(N), + <<"bucket", NBin/binary>>. -update_fterm_fn(Term, Keys) -> - fun(TermKeys) -> - orddict:update(Term, update_keys_fn(Keys), Keys, TermKeys) - end. +%% Index a bunch of keys under the same field/term. +tx_index_single_term(Clients, ClientId, Bucket, Keys, Field, Term) -> + Client = get_client(ClientId, Clients), + lager:info("Indexing in ~p under (~p, ~p) using client ~p: ~p", + [Bucket, Field, Term, ClientId, Keys]), + [index_object(Client, Bucket, Key, Field, Term) || Key <- Keys], + ok. -update_fterm2_fn(NewTermKeys) -> - fun(TermKeys) -> - orddict:merge(fun(_K, Keys1, Keys2) -> - lists:umerge(Keys1, Keys2) - end, TermKeys, NewTermKeys) - end. +%% Index a number of keys each under a different term. +tx_index_multi_term(Clients, ClientId, Bucket, KeyFieldTerms) -> + Client = get_client(ClientId, Clients), + lager:info("Indexing in ~p with client ~p: ~p", + [Bucket, ClientId, KeyFieldTerms]), + [index_object(Client, Bucket, Key, Field, Term) + || {Key, Field, Term} <- KeyFieldTerms], + ok. -update_ft_add_fn(Term, Key) -> - fun(OldTermKeys) -> - orddict:update(Term, update_termkey_add_fn(Key), OldTermKeys) +%% Delete a single object and all its associated index entries. +tx_delete_one(Clients, ClientId, Bucket, IntKey) -> + Client = get_client(ClientId, Clients), + lager:info("Deleting key ~p from bucket ~p using ~p", + [IntKey, Bucket, ClientId]), + delete_key(Client, Bucket, IntKey), + ok. + +tx_query_range(Clients, ClientId, Query) -> + Client = get_client(ClientId, Clients), + Keys = lists:sort(query_range(Client, Query, [])), + lager:info("Query ~p, ~p from ~p to ~p, page = ~p, returned ~p keys.", + [Query#query.bucket, Query#query.field, Query#query.start_term, + Query#query.end_term, Query#query.page_size, length(Keys)]), + %% Re-run with page sizes 1 -> 100, verify it's always the same result. + PageChecks = + [begin + Q2 = Query#query{page_size = PSize}, + OKeys = lists:sort(query_range(Client, Q2, [])), + OKeys =:= Keys + end || PSize <- lists:seq(1, 100)], + case lists:all(fun is_true/1, PageChecks) of + true -> + Keys; + false -> + {error, mismatch_when_paged} end. -update_termkey_add_fn(Key) -> - fun(OldKeys) -> - lists:usort([Key|OldKeys]) - end. +%% ====================================================== +%% Client utilities. +%% ====================================================== -to_tfield(FieldName, Term) -> - case is_integer(Term) of - true -> {int, FieldName}; - false -> {bin, FieldName} +create_client(Node, pb) -> + rt:pbc(Node); +create_client(Node, http) -> + rt:httpc(Node). + +get_client(ClientId, Clients) -> + orddict:fetch(ClientId, Clients). + +%% Convert field/term pair to pb client argument format. +pb_field_term(Field, Term) when is_integer(Term) -> + {Field ++ "_int", Term}; +pb_field_term(Field, Term) when is_binary(Term) -> + {Field ++ "_bin", Term}. + +pb_field(Field, Term) when is_integer(Term) -> + {integer_index, Field}; +pb_field(Field, Term) when is_binary(Term) -> + {binary_index, Field}. + +index_object({pb, PB}, Bucket, Key0, Field, Term) -> + Key = to_bin_key(Key0), + FT = pb_field_term(Field, Term), + Obj = + case riakc_pb_socket:get(PB, Bucket, Key) of + {ok, O} -> + %% Existing object, add index tag and de-duplicate. + MD = riakc_obj:get_metadata(O), + IndexMD2 = + case dict:find(<<"index">>, MD) of + {ok, IndexMD} -> + lists:usort([FT | IndexMD]); + error -> + [FT] + end, + MD2 = dict:store(<<"index">>, IndexMD2, MD), + riakc_obj:update_metadata(O, MD2); + {error, notfound} -> + %% New object, just add index tag. + O = riakc_obj:new(Bucket, Key, Key), + MD = dict:from_list([{<<"index">>, [FT]}]), + riakc_obj:update_metadata(O, MD) + end, + ok = riakc_pb_socket:put(PB, Obj), + ok. + +delete_key({pb, PB}, Bucket, IntKey) -> + Key = to_bin_key(IntKey), + ok = riakc_pb_socket:delete(PB, Bucket, Key), + %% Wait until all tombstones have been reaped. + ok = rt:wait_until(fun() -> rt:pbc_really_deleted(PB, Bucket, [Key]) end); +delete_key({http, C}, Bucket, IntKey) -> + Key = to_bin_key(IntKey), + ok = rhc:delete(C, Bucket, Key), + %% Wait until all tombstones have been reaped. + ok = rt:wait_until(fun() -> rt:httpc_really_deleted(C, Bucket, [Key]) end). + +%% Execute range query using a client, fetching multiple pages if necessary. +query_range({pb, PB} = Client, + #query { bucket = Bucket, field = FieldName, + start_term = Start, end_term = End, + page_size = PageSize, continuation = Cont } = Query, + AccKeys) -> + Field = pb_field(FieldName, Start), + case riakc_pb_socket:get_index_range(PB, Bucket, Field, Start, End, + [{max_results, PageSize}, + {continuation, Cont}]) of + {ok, ?INDEX_RESULTS{keys = Keys, continuation = undefined}} -> + AccKeys ++ Keys; + {ok, ?INDEX_RESULTS{keys = Keys, continuation = Cont1}} -> + Query1 = Query#query{continuation = Cont1}, + query_range(Client, Query1, AccKeys ++ Keys) end. -to_term_keys(Count, Key, Term) when is_integer(Term) -> - [{Term + N, [Key]} || N <- lists:seq(0, Count - 1)]; -to_term_keys(Count, Key, Term) when is_binary(Term) -> - [begin - Suffix = iolist_to_binary(io_lib:format("~0..5b", [N])), - {<>, [Key]} - end || N <- lists:seq(0, Count - 1)]. +%% Close all clients, ignore errors. +close_clients(Clients) -> + [catch riakc_pb_socket:stop(Client) || {pb, Client} <- Clients], + ok. -delete_key_from_term_keys(Key, TermKeys) -> - [{Term, lists:delete(Key, Keys)} || {Term, Keys} <- TermKeys]. +%% ====================================================== +%% Model data utilities +%% ====================================================== + +model_index([], Idx) -> + Idx; +model_index([{Keys, Field, Term} | More], Idx) -> + Idx1 = model_index(Keys, Field, Term, Idx), + model_index(More, Idx1). -delete_key_from_idx(Key, Idx) -> +model_index(NewKeys0, Field, Term, Idx) when is_list(NewKeys0) -> + TField = to_tfield(Field, Term), + NewKeys = lists:usort(NewKeys0), + TermKeys1 = + case orddict:find(TField, Idx) of + {ok, TermKeys0} -> + case orddict:find(Term, TermKeys0) of + {ok, Keys0} -> + MergedKeys = lists:umerge(NewKeys, Keys0), + orddict:store(Term, MergedKeys, TermKeys0); + _ -> + orddict:store(Term, NewKeys, TermKeys0) + end; + _ -> + [{Term, NewKeys}] + end, + orddict:store(TField, TermKeys1, Idx); +model_index(NewKey, Field, Term, Idx) -> + model_index([NewKey], Field, Term, Idx). + +model_delete_key(Key, Idx) -> [{Field, delete_key_from_term_keys(Key, TermKeys)} || {Field, TermKeys} <- Idx]. +delete_key_from_term_keys(Key, TermKeys) -> + [{Term, lists:delete(Key, Keys)} || {Term, Keys} <- TermKeys]. + +%% Produces a typed field id. For example, "i1"/43 -> {int, "i1"} +to_tfield(FieldName, Term) -> + case is_integer(Term) of + true -> {int, FieldName}; + false -> {bin, FieldName} + end. + %% Query against the modeled data. -query_state(#state{ data = #bucket_data{indexes = Idx}}, - #query{ field = Field, - start_term = Start, end_term = End }) -> +model_query_range(Query, Idx) -> + #query{ field = Field, start_term = Start, end_term = End } = Query, TField = to_tfield(Field, Start), %% Collect all keys with terms within the given range, ignore others. Scanner = fun({Term, Keys}, Acc) when Term >= Start, End >= Term -> @@ -356,137 +552,22 @@ query_state(#state{ data = #bucket_data{indexes = Idx}}, [to_bin_key(Key) || Key <- IntKeys] end. -pb_field_term(Field, Term) when is_integer(Term) -> - {Field ++ "_int", Term}; -pb_field_term(Field, Term) when is_binary(Term) -> - {Field ++ "_bin", Term}. - -pb_field(Field, Term) when is_integer(Term) -> - {integer_index, Field}; -pb_field(Field, Term) when is_binary(Term) -> - {binary_index, Field}. - %% ====================================================== -%% Transition functions. +%% Internal %% ====================================================== -create_clients(Nodes) -> - dict:from_list([{Node, rt:pbc(Node)} || Node <- Nodes]). -%% Returns a different bucket name each time it's called. -choose_bucket() -> - N = ets:update_counter(bucket_table, bucket_number, 1), - NBin = integer_to_binary(N), - <<"bucket", NBin/binary>>. - -index_objects(Clients, Nodes, Bucket, Key0, NKeys, Field, Term) -> - Node = hd(Nodes), - lager:info("Indexing ~p in ~p starting at key ~p " - "under (~p, ~p) on node ~p", - [NKeys, Bucket, Key0, Field, Term, Node]), - PB = dict:fetch(Node, Clients), - FT = pb_field_term(Field, Term), - [begin - Key = to_bin_key(N), - Obj = - case riakc_pb_socket:get(PB, Bucket, Key) of - {ok, O} -> - MD = riakc_obj:get_metadata(O), - IndexMD2 = - case dict:find(<<"index">>, MD) of - {ok, IndexMD} -> - lists:usort([FT | IndexMD]); - error -> - [FT] - end, - MD2 = dict:store(<<"index">>, IndexMD2, MD), - riakc_obj:update_metadata(O, MD2); - {error, notfound} -> - O = riakc_obj:new(Bucket, Key, Key), - MD = dict:from_list([{<<"index">>, [FT]}]), - riakc_obj:update_metadata(O, MD) - end, - ok = riakc_pb_socket:put(PB, Obj) - end || N <- to_key_list(Key0, NKeys)], - ok. - -index_objects2(Clients, Nodes, Bucket, Key0, NKeys, Field, Term0) -> - Node = hd(Nodes), - lager:info("Indexing (2) ~p in ~p starting at key ~p, term ~p " - "under ~p, on node ~p", - [NKeys, Bucket, Key0, Term0, Field, Node]), - PB = dict:fetch(Node, Clients), - [begin - Key = to_bin_key(N), - FT = pb_field_term(Field, Term), - Obj = - case riakc_pb_socket:get(PB, Bucket, Key) of - {ok, O} -> - MD = riakc_obj:get_metadata(O), - IndexMD2 = - case dict:find(<<"index">>, MD) of - {ok, IndexMD} -> - lists:usort([FT | IndexMD]); - error -> - [FT] - end, - MD2 = dict:store(<<"index">>, IndexMD2, MD), - riakc_obj:update_metadata(O, MD2); - {error, notfound} -> - O = riakc_obj:new(Bucket, Key, Key), - MD = dict:from_list([{<<"index">>, [FT]}]), - riakc_obj:update_metadata(O, MD) - end, - ok = riakc_pb_socket:put(PB, Obj) - end || {Term, [N]} <- to_term_keys(NKeys, Key0, Term0)], - ok. - -delete_one(Clients, Nodes, Bucket, IntKey) -> - Key = to_bin_key(IntKey), - Node = hd(Nodes), - PB = dict:fetch(Node, Clients), - lager:info("Deleting key ~p from bucket ~p", [Key, Bucket]), - ok = riakc_pb_socket:delete(PB, Bucket, Key), - ok = rt:wait_until(fun() -> rt:pbc_really_deleted(PB, Bucket, [Key]) end), - ok. - is_true(true) -> true; is_true(_) -> false. -query_range(Clients, Nodes, Query) -> - Node = hd(Nodes), - PB = dict:fetch(Node, Clients), - Keys = lists:sort(query_range_pb(PB, Query, [])), - lager:info("Query ~p, ~p from ~p to ~p, page = ~p, returned ~p keys.", - [Query#query.bucket, Query#query.field, Query#query.start_term, - Query#query.end_term, Query#query.page_size, length(Keys)]), - %% Re-run with page sizes 1 -> 100, verify it's always the same result. - PageChecks = - [begin - Q2 = Query#query{page_size = PSize}, - OKeys = lists:sort(query_range_pb(PB, Q2, [])), - OKeys =:= Keys - end || PSize <- lists:seq(1, 100)], - case lists:all(fun is_true/1, PageChecks) of - true -> - Keys; - false -> - {error, mismatch_when_paged} - end. +%% Initialize counter used to use a different bucket per run. +init_bucket_counter() -> + ets:new(bucket_table, [named_table, public]), + ets:insert_new(bucket_table, [{bucket_number, 0}]). -query_range_pb(PB, - #query { bucket = Bucket, field = FieldName, - start_term = Start, end_term = End, - page_size = PageSize, continuation = Cont } = Query, - AccKeys) -> - Field = pb_field(FieldName, Start), - case riakc_pb_socket:get_index_range(PB, Bucket, Field, Start, End, - [{max_results, PageSize}, - {continuation, Cont}]) of - {ok, ?INDEX_RESULTS{keys = Keys, continuation = undefined}} -> - AccKeys ++ Keys; - {ok, ?INDEX_RESULTS{keys = Keys, continuation = Cont1}} -> - Query1 = Query#query{continuation = Cont1}, - query_range_pb(PB, Query1, AccKeys ++ Keys) - end. +%% Convert integer object key to binary form. +to_bin_key(N) when is_integer(N) -> + iolist_to_binary(io_lib:format("~5..0b", [N])); +to_bin_key(Key) when is_binary(Key) -> + Key. -endif. From 477d0e78c996790777b4210bc3ab4109b7515310 Mon Sep 17 00:00:00 2001 From: "Engel A. Sanchez" Date: Tue, 11 Aug 2015 23:12:11 -0400 Subject: [PATCH 72/89] Fleshed out but disabled http client support The http client seems to have a lot of problems and works differently than the PB client. Index values are not coming back on a get, even though a PB get returns them, so indexing is not working properly. Binaries are not encoded properly when the query index URL is created, and empty binaries are taken to mean "no value". (cherry picked from commit 1c29f7fb43cbc45753d999a160f9d359445fbf50) --- tests/verify_2i_eqc.erl | 141 +++++++++++++++++++++++++++++----------- 1 file changed, 102 insertions(+), 39 deletions(-) diff --git a/tests/verify_2i_eqc.erl b/tests/verify_2i_eqc.erl index 2c50b472f..ab413d5a3 100644 --- a/tests/verify_2i_eqc.erl +++ b/tests/verify_2i_eqc.erl @@ -76,6 +76,11 @@ -define(MAX_CLUSTER_SIZE, 1). -define(MAX_FIELDS, 1). -define(FIELDS, ["i" ++ integer_to_list(N) || N <- lists:seq(1, ?MAX_FIELDS)]). +%% Enabling the use of http clients requires a lot more work, as things +%% do not work the same as with PB. Binaries are not encoced, empty binaries +%% have a special meaning, it's not inclusive on the end term like PB. +%% Who knows what else. :( +%%-define(CLIENT_TYPES, [pb, http]). -define(CLIENT_TYPES, [pb]). -type index_field() :: {int | bin, binary()}. @@ -170,8 +175,26 @@ gen_int_term() -> %% Generates a random binary. gen_bin_term() -> + %% The riak HTTP interface does not like empty binaries. + %% To enable the use of the http client, which does not encode + %% binaries at freaking all, you would need something like this: + %% iolist_to_binary(http_uri:encode(binary_to_list(B))). + %% You also need to prevent empty binaries, which through http + %% mean "no term" + %%?LET(B, non_empty(binary()), sanitize_binary(B)). binary(). +sanitize_binary(B) -> + B2 = base64:encode(B), + sanitize_binary(B2, <<>>). + +sanitize_binary(<<>>, B) -> + B; +sanitize_binary(<<"=", Rest/binary>>, Out) -> + sanitize_binary(Rest, <>); +sanitize_binary(<>, Out) -> + sanitize_binary(Rest, <>). + %% Generates a list of integer keys without duplicates. gen_key_list() -> ?LET(L, non_empty(list(gen_key())), lists:usort(L)). @@ -335,12 +358,13 @@ precondition(_From, _To, _S, {call, _, _, _}) -> %% if the results do not match what is in the model. postcondition(_, _, S, {call, _, tx_query_range, [_, _, Query]}, {error, Err}) -> {state, S, query, Query, error, Err}; -postcondition(_, _, S, {call, _, tx_query_range, [_, _, Query]}, Keys) -> +postcondition(_, _, S, {call, _, tx_query_range, [_, Client, Query]}, Keys) -> #state{indexes = Idx} = S, ExpectedKeys = model_query_range(Query, Idx), case lists:usort(Keys) =:= ExpectedKeys of true -> true; - false -> {state, S, query, Query, expected, ExpectedKeys, actual, Keys} + false -> {state, S, client, Client, query, Query, + expected, ExpectedKeys, actual, Keys} end; postcondition(_, _, _, _Call, _) -> true. @@ -365,7 +389,7 @@ tx_next_bucket() -> %% Index a bunch of keys under the same field/term. tx_index_single_term(Clients, ClientId, Bucket, Keys, Field, Term) -> Client = get_client(ClientId, Clients), - lager:info("Indexing in ~p under (~p, ~p) using client ~p: ~p", + lager:info("Indexing in ~p under (~p, ~p) using client ~p: ~w", [Bucket, Field, Term, ClientId, Keys]), [index_object(Client, Bucket, Key, Field, Term) || Key <- Keys], ok. @@ -390,9 +414,11 @@ tx_delete_one(Clients, ClientId, Bucket, IntKey) -> tx_query_range(Clients, ClientId, Query) -> Client = get_client(ClientId, Clients), Keys = lists:sort(query_range(Client, Query, [])), - lager:info("Query ~p, ~p from ~p to ~p, page = ~p, returned ~p keys.", + lager:info("Query ~p, ~p from ~p to ~p, page = ~p, using ~p " + "returned ~p keys.", [Query#query.bucket, Query#query.field, Query#query.start_term, - Query#query.end_term, Query#query.page_size, length(Keys)]), + Query#query.end_term, Query#query.page_size, ClientId, + length(Keys)]), %% Re-run with page sizes 1 -> 100, verify it's always the same result. PageChecks = [begin @@ -420,52 +446,74 @@ get_client(ClientId, Clients) -> orddict:fetch(ClientId, Clients). %% Convert field/term pair to pb client argument format. -pb_field_term(Field, Term) when is_integer(Term) -> - {Field ++ "_int", Term}; -pb_field_term(Field, Term) when is_binary(Term) -> - {Field ++ "_bin", Term}. +to_field_id_term(Field, Term) -> + {to_field_id(Field, Term), [Term]}. -pb_field(Field, Term) when is_integer(Term) -> +to_field_id(Field, Term) when is_integer(Term) -> {integer_index, Field}; -pb_field(Field, Term) when is_binary(Term) -> +to_field_id(Field, Term) when is_binary(Term) -> {binary_index, Field}. -index_object({pb, PB}, Bucket, Key0, Field, Term) -> +to_field_id_term_http(Field, Term) -> + {to_field_id_http(Field, Term), [Term]}. + +to_field_id_http(Field, Term) when is_integer(Term) -> + iolist_to_binary([Field, "_int"]); +to_field_id_http(Field, Term) when is_binary(Term) -> + iolist_to_binary([Field, "_bin"]). + +index_object({pb, C}, Bucket, Key0, Field, Term) -> Key = to_bin_key(Key0), - FT = pb_field_term(Field, Term), - Obj = - case riakc_pb_socket:get(PB, Bucket, Key) of - {ok, O} -> - %% Existing object, add index tag and de-duplicate. - MD = riakc_obj:get_metadata(O), - IndexMD2 = - case dict:find(<<"index">>, MD) of - {ok, IndexMD} -> - lists:usort([FT | IndexMD]); - error -> - [FT] - end, - MD2 = dict:store(<<"index">>, IndexMD2, MD), - riakc_obj:update_metadata(O, MD2); - {error, notfound} -> - %% New object, just add index tag. - O = riakc_obj:new(Bucket, Key, Key), - MD = dict:from_list([{<<"index">>, [FT]}]), - riakc_obj:update_metadata(O, MD) - end, - ok = riakc_pb_socket:put(PB, Obj), + FT = to_field_id_term(Field, Term), + Obj0 = case riakc_pb_socket:get(C, Bucket, Key) of + {ok, O} -> + O; + {error, notfound} -> + riakc_obj:new(Bucket, Key, Key) + end, + MD0 = riakc_obj:get_update_metadata(Obj0), + MD1 = riakc_obj:add_secondary_index(MD0, [FT]), + Obj1 = riakc_obj:update_metadata(Obj0, MD1), + ok = riakc_pb_socket:put(C, Obj1, [{dw, 3}]), + ok; +index_object({http, C}, Bucket, Key0, Field, Term) -> + Key = to_bin_key(Key0), + FT = to_field_id_term_http(Field, Term), + Obj0 = case rhc:get(C, Bucket, Key) of + {ok, O} -> + O; + {error, notfound} -> + riakc_obj:new(Bucket, Key, Key) + end, + MD0 = riakc_obj:get_update_metadata(Obj0), + MD1 = riakc_obj:add_secondary_index(MD0, [FT]), + Obj1 = riakc_obj:update_metadata(Obj0, MD1), + ok = rhc:put(C, Obj1, [{dw, 3}]), ok. delete_key({pb, PB}, Bucket, IntKey) -> Key = to_bin_key(IntKey), - ok = riakc_pb_socket:delete(PB, Bucket, Key), + case riakc_pb_socket:get(PB, Bucket, Key) of + {ok, O} -> + ok = riakc_pb_socket:delete_obj(PB, O, [{dw, 3}]); + {error, notfound} -> + ok = riakc_pb_socket:delete(PB, Bucket, Key, [{dw, 3}]) + end, %% Wait until all tombstones have been reaped. - ok = rt:wait_until(fun() -> rt:pbc_really_deleted(PB, Bucket, [Key]) end); + %% TODO: Do we need to reap tombstones for this test? I think now. + %% ok = rt:wait_until(fun() -> rt:pbc_really_deleted(PB, Bucket, [Key]) end); + ok; delete_key({http, C}, Bucket, IntKey) -> Key = to_bin_key(IntKey), - ok = rhc:delete(C, Bucket, Key), + case rhc:get(C, Bucket, Key) of + {ok, O} -> + ok = rhc:delete_obj(C, O, [{dw, 3}]); + {error, notfound} -> + ok = rhc:delete(C, Bucket, Key, [{dw, 3}]) + end, %% Wait until all tombstones have been reaped. - ok = rt:wait_until(fun() -> rt:httpc_really_deleted(C, Bucket, [Key]) end). + %%ok = rt:wait_until(fun() -> rt:httpc_really_deleted(C, Bucket, [Key]) end). + ok. %% Execute range query using a client, fetching multiple pages if necessary. query_range({pb, PB} = Client, @@ -473,7 +521,7 @@ query_range({pb, PB} = Client, start_term = Start, end_term = End, page_size = PageSize, continuation = Cont } = Query, AccKeys) -> - Field = pb_field(FieldName, Start), + Field = to_field_id(FieldName, Start), case riakc_pb_socket:get_index_range(PB, Bucket, Field, Start, End, [{max_results, PageSize}, {continuation, Cont}]) of @@ -482,6 +530,21 @@ query_range({pb, PB} = Client, {ok, ?INDEX_RESULTS{keys = Keys, continuation = Cont1}} -> Query1 = Query#query{continuation = Cont1}, query_range(Client, Query1, AccKeys ++ Keys) + end; +query_range({http, C} = Client, + #query { bucket = Bucket, field = FieldName, + start_term = Start, end_term = End, + page_size = PageSize, continuation = Cont } = Query, + AccKeys) -> + Field = to_field_id(FieldName, Start), + case rhc:get_index(C, Bucket, Field, {Start, End}, + [{max_results, PageSize}, + {continuation, Cont}]) of + {ok, ?INDEX_RESULTS{keys = Keys, continuation = undefined}} -> + AccKeys ++ Keys; + {ok, ?INDEX_RESULTS{keys = Keys, continuation = Cont1}} -> + Query1 = Query#query{continuation = Cont1}, + query_range(Client, Query1, AccKeys ++ Keys) end. %% Close all clients, ignore errors. From 7ead9e02fee688f94e59aa0e8fea989253dcd0fe Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Wed, 24 Feb 2016 11:06:12 -0600 Subject: [PATCH 73/89] Prevent large negative integers as values This is due to a sext encoding bug present in current riak releases. When sext and riak has been updated with a fix, revert this commit. (cherry picked from commit b5a0dd9737935a39c55f544409b7cf9e96ff66a0) --- tests/verify_2i_eqc.erl | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/verify_2i_eqc.erl b/tests/verify_2i_eqc.erl index ab413d5a3..36a5c9456 100644 --- a/tests/verify_2i_eqc.erl +++ b/tests/verify_2i_eqc.erl @@ -73,6 +73,8 @@ -behaviour(riak_test). -export([confirm/0]). +-define(NEG_LIMIT, -1000000). %% Due to sext encoding bug, do not permit negative numbers less than this to be generated. + -define(MAX_CLUSTER_SIZE, 1). -define(MAX_FIELDS, 1). -define(FIELDS, ["i" ++ integer_to_list(N) || N <- lists:seq(1, ?MAX_FIELDS)]). @@ -171,7 +173,24 @@ gen_term() -> %% Generates, with equal likelihood, either a smallish or a largish integer. gen_int_term() -> - oneof([int(), largeint()]). + oneof([int(), gen_large_int()]). + +%% XXX FIXME +%% Ensure that very large negative numbers are not used as values. +%% +%% This is because the version of sext used in riak has an encoding +%% bug for these values. Until we can fix this, this test needs to +%% deal with and work around the sext encoding bug. +%% +%% When that bug is fixed, this commit should be reverted/code +%% deleted. +gen_large_int() -> + ?LET(I, largeint(), no_large_neg_ints(I)). + +no_large_neg_ints(I) when I < ?NEG_LIMIT -> + abs(I); +no_large_neg_ints(I) -> + I. %% Generates a random binary. gen_bin_term() -> From aa757b40c2da4bdda9361c5fafcdb492def61ca4 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Wed, 24 Feb 2016 11:07:16 -0600 Subject: [PATCH 74/89] Always ensure leveldb is the backend for 2i (cherry picked from commit 472fb6447bea41d13ca823048b644a5d4f11d205) --- tests/verify_2i_eqc.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/verify_2i_eqc.erl b/tests/verify_2i_eqc.erl index 36a5c9456..57c4936ed 100644 --- a/tests/verify_2i_eqc.erl +++ b/tests/verify_2i_eqc.erl @@ -115,6 +115,7 @@ confirm() -> TestingTime = rt_config:get(eqc_testing_time, 120), lager:info("Will run in cluster of size ~p for ~p seconds.", [Size, TestingTime]), + rt:set_backend(eleveldb), Nodes = rt:build_cluster(Size), ?assert(eqc:quickcheck( eqc:testing_time(TestingTime, ?MODULE:prop_test(Nodes)))), From 1cc0514e57414287228e0fd46726b32f94914fe0 Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Fri, 29 Apr 2016 15:14:21 -0400 Subject: [PATCH 75/89] Fix merge issues with overload.erl - re-instate w1c tests. --- tests/overload.erl | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/overload.erl b/tests/overload.erl index 96ca04304..b72ef34c8 100644 --- a/tests/overload.erl +++ b/tests/overload.erl @@ -33,8 +33,10 @@ -define(KEY, <<"hotkey">>). -define(NORMAL_TYPE, <<"normal_type">>). -define(CONSISTENT_TYPE, <<"consistent_type">>). +-define(WRITE_ONCE_TYPE, <<"write_once_type">>). -define(NORMAL_BKV, {{?NORMAL_TYPE, ?BUCKET}, ?KEY, <<"test">>}). -define(CONSISTENT_BKV, {{?CONSISTENT_TYPE, ?BUCKET}, ?KEY, <<"test">>}). +-define(WRITE_ONCE_BKV, {{?WRITE_ONCE_TYPE, ?BUCKET}, ?KEY, <<"test">>}). %% This record contains the default values for config settings if they were not set %% in the advanced.config file - because setting something to `undefined` is not the same @@ -83,12 +85,14 @@ confirm() -> ok = create_bucket_type(Nodes, ?NORMAL_TYPE, [{n_val, 3}]), ok = create_bucket_type(Nodes, ?CONSISTENT_TYPE, [{consistent, true}, {n_val, 5}]), + ok = create_bucket_type(Nodes, ?WRITE_ONCE_TYPE, [{write_once, true}, {n_val, 1}]), rt:wait_until(ring_manager_check_fun(hd(Nodes))), Node1 = hd(Nodes), write_once(Node1, ?NORMAL_BKV), write_once(Node1, ?CONSISTENT_BKV), + write_once(Node1, ?WRITE_ONCE_BKV), Tests = [test_no_overload_protection, test_vnode_protection, @@ -99,7 +103,9 @@ confirm() -> ok = erlang:apply(?MODULE, Test, [Nodes, BKV]) end || Test <- Tests, BKV <- [?NORMAL_BKV, - ?CONSISTENT_BKV]], + ?CONSISTENT_BKV, + ?WRITE_ONCE_BKV]], + %% Test cover queries doesn't depend on bucket/keyvalue, just run it once test_cover_queries_overload(Nodes), pass. @@ -163,9 +169,15 @@ test_vnode_protection(Nodes, BKV) -> Pid ! resume, ok. + %% Don't check consistent gets, as they don't use the FSM test_fsm_protection(_, ?CONSISTENT_BKV) -> ok; + +%% Don't check on fast path either. +test_fsm_protection(_, ?WRITE_ONCE_BKV) -> + ok; + test_fsm_protection(Nodes, BKV) -> lager:info("Testing with coordinator protection enabled"), lager:info("Setting FSM limit to ~b", [?THRESHOLD]), From 530a132f88e12dd3e68509e71eaf6e9addc3cbfb Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Thu, 7 Jul 2016 16:06:02 -0400 Subject: [PATCH 76/89] Updated dependencies to be consistent with riak/riak_ee develop-2.2 --- rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 2ecbeec34..19341b5e7 100644 --- a/rebar.config +++ b/rebar.config @@ -11,11 +11,11 @@ {eunit_opts, [verbose]}. {deps, [ - {lager, ".*", {git, "git://github.com/basho/lager", {tag, "2.0.3"}}}, + {lager, ".*", {git, "git://github.com/basho/lager", {tag, "2.2.3"}}}, {getopt, ".*", {git, "git://github.com/jcomellas/getopt", {tag, "v0.4"}}}, {meck, ".*", {git, "git://github.com/basho/meck.git", {tag, "0.8.2"}}}, {mapred_verify, ".*", {git, "git://github.com/basho/mapred_verify", {branch, "master"}}}, - {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {branch, "master"}}}, + {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.1.2"}}}, {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {branch, "master"}}}, {kvc, "1.3.0", {git, "https://github.com/etrepum/kvc", {tag, "v1.3.0"}}}, {druuid, ".*", {git, "git://github.com/kellymclaughlin/druuid.git", {tag, "0.2"}}} From c8cf713f97259ae412693449073f2e0831fc167d Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Fri, 8 Jul 2016 10:06:40 -0400 Subject: [PATCH 77/89] dependency cleanup --- rebar.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rebar.config b/rebar.config index 19341b5e7..72d707572 100644 --- a/rebar.config +++ b/rebar.config @@ -13,10 +13,10 @@ {deps, [ {lager, ".*", {git, "git://github.com/basho/lager", {tag, "2.2.3"}}}, {getopt, ".*", {git, "git://github.com/jcomellas/getopt", {tag, "v0.4"}}}, - {meck, ".*", {git, "git://github.com/basho/meck.git", {tag, "0.8.2"}}}, + {meck, "0.8.2", {git, "git://github.com/basho/meck.git", {tag, "0.8.2"}}}, {mapred_verify, ".*", {git, "git://github.com/basho/mapred_verify", {branch, "master"}}}, - {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.1.2"}}}, - {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {branch, "master"}}}, + {riakc, "2.1.2", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.1.2"}}}, + {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {tag, "2.1.2"}}}, {kvc, "1.3.0", {git, "https://github.com/etrepum/kvc", {tag, "v1.3.0"}}}, {druuid, ".*", {git, "git://github.com/kellymclaughlin/druuid.git", {tag, "0.2"}}} ]}. From 4ec99959f553e35060ef194d0f1837551590b625 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Tue, 26 Jul 2016 16:26:38 -0400 Subject: [PATCH 78/89] Update verify_aae test to verify throttle config --- tests/verify_aae.erl | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tests/verify_aae.erl b/tests/verify_aae.erl index 1d5fa2dda..f81b14c1f 100644 --- a/tests/verify_aae.erl +++ b/tests/verify_aae.erl @@ -43,6 +43,7 @@ % I would hope this would come from the testing framework some day % to use the test in small and large scenarios. -define(DEFAULT_RING_SIZE, 8). +-define(AAE_THROTTLE_LIMITS, [{-1, 0}, {100, 10}]). -define(CFG, [{riak_kv, [ @@ -51,7 +52,8 @@ {anti_entropy_build_limit, {100, 1000}}, {anti_entropy_concurrency, 100}, {anti_entropy_expire, 24 * 60 * 60 * 1000}, % Not for now! - {anti_entropy_tick, 500} + {anti_entropy_tick, 500}, + {aae_throttle_limits, ?AAE_THROTTLE_LIMITS} ]}, {riak_core, [ @@ -65,9 +67,25 @@ confirm() -> Nodes = rt:build_cluster(?NUM_NODES, ?CFG), + verify_throttle_config(Nodes), verify_aae(Nodes), pass. +verify_throttle_config(Nodes) -> + lists:foreach( + fun(Node) -> + ?assert(rpc:call(Node, + riak_kv_entropy_manager, + is_aae_throttle_enabled, + [])), + ?assertMatch(?AAE_THROTTLE_LIMITS, + rpc:call(Node, + riak_kv_entropy_manager, + get_aae_throttle_limits, + [])) + end, + Nodes). + verify_aae(Nodes) -> Node1 = hd(Nodes), % First, recovery without tree rebuilds From 0c9fef2d72b2bc5f4576ee542c0f7f474d89255c Mon Sep 17 00:00:00 2001 From: Zeeshan Lakhani Date: Fri, 29 Apr 2016 17:29:15 -0400 Subject: [PATCH 79/89] - new test for HLL(set) datatypes - update crdt cap and converge tests for hlls - temp rebar updates for riak-client; *will update w/ correct version* upon mergies --- rebar.config | 4 +- tests/test_hll.erl | 217 +++++++++++++++++++++++++++++++ tests/verify_crdt_capability.erl | 11 +- tests/verify_dt_converge.erl | 77 ++++++++--- 4 files changed, 286 insertions(+), 23 deletions(-) create mode 100644 tests/test_hll.erl diff --git a/rebar.config b/rebar.config index 72d707572..20dfe3e39 100644 --- a/rebar.config +++ b/rebar.config @@ -15,8 +15,8 @@ {getopt, ".*", {git, "git://github.com/jcomellas/getopt", {tag, "v0.4"}}}, {meck, "0.8.2", {git, "git://github.com/basho/meck.git", {tag, "0.8.2"}}}, {mapred_verify, ".*", {git, "git://github.com/basho/mapred_verify", {branch, "master"}}}, - {riakc, "2.1.2", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.1.2"}}}, - {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {tag, "2.1.2"}}}, + {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {branch, "feature-zl-hll_datatypes"}}}, + {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {branch, "feature-zl-hll_datatypes"}}}, {kvc, "1.3.0", {git, "https://github.com/etrepum/kvc", {tag, "v1.3.0"}}}, {druuid, ".*", {git, "git://github.com/kellymclaughlin/druuid.git", {tag, "0.2"}}} ]}. diff --git a/tests/test_hll.erl b/tests/test_hll.erl new file mode 100644 index 000000000..dbe108456 --- /dev/null +++ b/tests/test_hll.erl @@ -0,0 +1,217 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2016 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +%%% @doc r_t to test hll datatypes across a riak cluster + +-module(test_hll). + +-export([confirm/0]). + +-include_lib("eunit/include/eunit.hrl"). + +-define(HLL_TYPE1, <<"hlls1">>). +-define(HLL_TYPE2, <<"hlls2">>). +-define(HLL_TYPE3, <<"hlls3">>). +-define(BUCKET1, {?HLL_TYPE1, <<"testbucket1">>}). +-define(BUCKET2, {?HLL_TYPE2, <<"testbucket2">>}). +-define(DEFAULT_P, 14). +-define(SET_P, 16). +-define(BAD_P, 1). +-define(P_SETTING, hll_precision). +-define(KEY, <<"flipit&reverseit">>). +-define(CONFIG, + [ + {riak_core, + [{ring_creation_size, 8}, + {anti_entropy_build_limit, {100, 1000}}, + {anti_entropy_concurrency, 8}, + {handoff_concurrency, 16}] + } + ]). + +confirm() -> + %% Configure cluster. + Nodes = [N1, N2, N3, N4] = rt:build_cluster(4, ?CONFIG), + + NodeRand1A = rt:select_random([N1, N2]), + NodeRand1B = rt:select_random([N1, N2]), + NodeRand2A = rt:select_random([N3, N4]), + NodeRand2B = rt:select_random([N3, N4]), + + lager:info("Create PB/HTTP Clients from first two nodes, and then" + " the second two nodes, as we'll partition Nodes 1 & 2 from" + " Nodes 3 & 4 later"), + + %% Create PB connection. + PBC1 = rt:pbc(NodeRand1A), + PBC2 = rt:pbc(NodeRand2A), + riakc_pb_socket:set_options(PBC1, [queue_if_disconnected]), + riakc_pb_socket:set_options(PBC2, [queue_if_disconnected]), + + %% Create HTTPC connection. + HttpC1 = rt:httpc(NodeRand1B), + HttpC2 = rt:httpc(NodeRand2B), + + ok = rt:create_activate_and_wait_for_bucket_type(Nodes, + ?HLL_TYPE1, + [{datatype, hll}, + {?P_SETTING, ?SET_P}]), + + ok = rt:create_activate_and_wait_for_bucket_type(Nodes, + ?HLL_TYPE2, + [{datatype, hll}]), + + lager:info("Create a bucket-type w/ a HLL datatype and a bad HLL precision" + " - This should throw an error"), + ?assertError({badmatch, {error, [{hll_precision, _}]}}, + rt:create_activate_and_wait_for_bucket_type(Nodes, + ?HLL_TYPE3, + [{datatype, hll}, + {?P_SETTING, + ?BAD_P}])), + + pb_tests(PBC1, PBC2, riakc_pb_socket, ?BUCKET1, Nodes), + http_tests(HttpC1, HttpC2, rhc, ?BUCKET2, Nodes), + + %% Stop PB connections. + riakc_pb_socket:stop(PBC1), + riakc_pb_socket:stop(PBC2), + + pass. + +http_tests(C1, C2, CMod, Bucket, Nodes) -> + lager:info("HTTP CLI TESTS: Create new Hll DT"), + + add_tests(C1, CMod, Bucket), + + HllSet0 = get_hll(C1, CMod, Bucket), + check_precision_and_reduce_test(C1, CMod, Bucket, ?DEFAULT_P, HllSet0), + + partition_write_heal(C1, C2, CMod, Bucket, Nodes), + + HllSet1 = get_hll(C1, CMod, Bucket), + check_precision_and_reduce_invalid_test(C1, CMod, Bucket, ?DEFAULT_P - 1, + HllSet1), + + ok. + +pb_tests(C1, C2, CMod, Bucket, Nodes) -> + lager:info("PB CLI TESTS: Create new Hll DT"), + + add_tests(C1, CMod, Bucket), + + HllSet0 = get_hll(C1, CMod, Bucket), + check_precision_and_reduce_test(C1, CMod, Bucket, ?SET_P, HllSet0), + + partition_write_heal(C1, C2, CMod, Bucket, Nodes), + + HllSet1 = get_hll(C1, CMod, Bucket), + check_precision_and_reduce_invalid_test(C1, CMod, Bucket, ?SET_P - 1, HllSet1), + + ok. + +add_tests(C, CMod, Bucket) -> + S0 = riakc_hll:new(), + + add_element(C, CMod, Bucket, S0, <<"OH">>), + {ok, S1} = CMod:fetch_type(C, Bucket, ?KEY), + ?assertEqual(riakc_hll:value(S1), 1), + + add_elements(C, CMod, Bucket, S1, [<<"C">>, <<"A">>, <<"P">>]), + {ok, S2} = CMod:fetch_type(C, Bucket, ?KEY), + ?assertEqual(riakc_hll:value(S2), 4), + + add_redundant_element(C, CMod, Bucket, S2, <<"OH">>), + {ok, S3} = CMod:fetch_type(C, Bucket, ?KEY), + ?assertEqual(riakc_hll:value(S3), 4). + +partition_write_heal(C1, C2, CMod, Bucket, Nodes) -> + lager:info("Partition cluster in two to force merge."), + [N1, N2, N3, N4] = Nodes, + PartInfo = rt:partition([N1, N2], [N3, N4]), + + try + lager:info("Write to one side of the partition"), + {ok, S0} = CMod:fetch_type(C1, Bucket, ?KEY), + add_element(C1, CMod, Bucket, S0, <<"OH hello there">>), + {ok, S1} = CMod:fetch_type(C1, Bucket, ?KEY), + ?assertEqual(riakc_hll:value(S1), 5), + + lager:info("Write to the other side of the partition"), + {ok, S2} = CMod:fetch_type(C2, Bucket, ?KEY), + add_element(C2, CMod, Bucket, S2, <<"Riak 1.4.eva">>), + {ok, S3} = CMod:fetch_type(C2, Bucket, ?KEY), + ?assertEqual(riakc_hll:value(S3), 5), + + lager:info("Heal") + after + ok = rt:heal(PartInfo) + end, + + ok = rt:wait_until_no_pending_changes(Nodes), + ok = rt:wait_until_transfers_complete(Nodes), + + lager:info("Once healed, check both sides for the correct, merged value"), + {ok, S4} = CMod:fetch_type(C1, Bucket, ?KEY), + ?assertEqual(riakc_hll:value(S4), 6), + {ok, S5} = CMod:fetch_type(C2, Bucket, ?KEY), + ?assertEqual(riakc_hll:value(S5), 6). + +get_hll(C, CMod, Bucket) -> + {ok, Obj} =CMod:get(C, Bucket, ?KEY), + {ok, CRDT} = riak_kv_crdt:from_binary(riakc_obj:get_value(Obj)), + {_, _, _, HllSet} = CRDT, + HllSet. + +add_element(C, CMod, Bucket, S, Elem) -> + lager:info("Add element to HLL DT"), + CMod:update_type( + C, Bucket, ?KEY, + riakc_hll:to_op( + riakc_hll:add_element(Elem, S))). + +add_elements(C, CMod, Bucket, S, Elems) -> + lager:info("Add multiple elements to HLL DT"), + CMod:update_type( + C, Bucket, ?KEY, + riakc_hll:to_op( + riakc_hll:add_elements(Elems, S))). + +add_redundant_element(C, CMod, Bucket, S, Elem) -> + lager:info("Add redundant element to HLL DT by calling" + " add_element/3 again"), + add_element(C, CMod, Bucket, S, Elem). + +check_precision_and_reduce_test(C, CMod, Bucket, ExpP, HllSet) -> + {ok, Props0} = CMod:get_bucket(C, Bucket), + ?assertEqual(proplists:get_value(?P_SETTING, Props0), ExpP), + ?assertEqual(riak_kv_hll:precision(HllSet), ExpP), + ok = CMod:set_bucket(C, Bucket, [{?P_SETTING, ExpP - 1}]), + {ok, Props1} = CMod:get_bucket(C, Bucket), + ?assertEqual(proplists:get_value(?P_SETTING, Props1), ExpP - 1). + +check_precision_and_reduce_invalid_test(C, CMod, Bucket, ExpP, HllSet) -> + lager:info("HLL's can be reduced, but never increased.\n" + " Test to make sure we don't allow invalid values."), + + ?assertEqual(riak_kv_hll:precision(HllSet), ExpP), + {error, _} = CMod:set_bucket(C, Bucket, [{?P_SETTING, ExpP + 1}]), + {ok, Props} = CMod:get_bucket(C, Bucket), + ?assertEqual(proplists:get_value(?P_SETTING, Props), ExpP), + ?assertEqual(riak_kv_hll:precision(HllSet), ExpP). diff --git a/tests/verify_crdt_capability.erl b/tests/verify_crdt_capability.erl index 9b49b5a57..d444af721 100644 --- a/tests/verify_crdt_capability.erl +++ b/tests/verify_crdt_capability.erl @@ -63,8 +63,13 @@ confirm() -> lager:info("Upgrayded!!"), ?assertEqual(ok, rt:wait_until_ready(Current)), ?assertEqual(ok, rt:wait_until_ready(Previous)), - rt:wait_for_service(Previous, riak_kv), - ?assertEqual(ok, rt:wait_until_capability_contains(Current, {riak_kv, crdt}, [riak_dt_pncounter, riak_dt_orswot, riak_dt_map, pncounter])), + rt:wait_for_service(Previous, riak_kv), + ?assertEqual(ok, rt:wait_until_capability_contains(Current, {riak_kv, crdt}, + [riak_dt_pncounter, + riak_dt_orswot, + riak_dt_map, + pncounter, + riak_kv_hll])), ?assertMatch(ok, rhc:counter_incr(PrevHttp, ?BUCKET, ?KEY, 1)), ?assertMatch({ok, 5}, rhc:counter_val(PrevHttp, ?BUCKET, ?KEY)), @@ -86,7 +91,7 @@ confirm() -> ?assertEqual(8, begin {ok, Counter} = riakc_pb_socket:fetch_type(PrevPB1, {<<"default">>, ?BUCKET}, ?KEY), riakc_counter:value(Counter) - end), + end), ?assertEqual(ok, riakc_pb_socket:update_type(PrevPB1, {<<"default">>, ?BUCKET}, ?KEY, gen_counter_op())), ?assertEqual({ok, 9}, riakc_pb_socket:counter_val(PB, ?BUCKET, ?KEY)), diff --git a/tests/verify_dt_converge.erl b/tests/verify_dt_converge.erl index 60c78ab36..db6b67e8c 100644 --- a/tests/verify_dt_converge.erl +++ b/tests/verify_dt_converge.erl @@ -32,9 +32,11 @@ -define(CTYPE, <<"counters">>). -define(STYPE, <<"sets">>). -define(MTYPE, <<"maps">>). +-define(HTYPE, <<"hlls">>). -define(TYPES, [{?CTYPE, counter}, {?STYPE, set}, - {?MTYPE, map}]). + {?MTYPE, map}, + {?HTYPE, hll}]). -define(PB_BUCKET, <<"pbtest">>). -define(HTTP_BUCKET, <<"httptest">>). @@ -58,17 +60,17 @@ confirm() -> %% Do some updates to each type [update_1(Type, ?PB_BUCKET, Client, riakc_pb_socket) || - {Type, Client} <- lists:zip(?TYPES, [P1, P2, P3])], + {Type, Client} <- lists:zip(?TYPES, [P1, P2, P3, P4])], [update_1(Type, ?HTTP_BUCKET, Client, rhc) || - {Type, Client} <- lists:zip(?TYPES, [H1, H2, H3])], + {Type, Client} <- lists:zip(?TYPES, [H1, H2, H3, H4])], %% Check that the updates are stored [check_1(Type, ?PB_BUCKET, Client, riakc_pb_socket) || - {Type, Client} <- lists:zip(?TYPES, [P4, P3, P2])], + {Type, Client} <- lists:zip(?TYPES, [P4, P3, P2, P1])], [check_1(Type, ?HTTP_BUCKET, Client, rhc) || - {Type, Client} <- lists:zip(?TYPES, [H4, H3, H2])], + {Type, Client} <- lists:zip(?TYPES, [H4, H3, H2, H1])], lager:info("Partition cluster in two."), @@ -77,34 +79,34 @@ confirm() -> lager:info("Modify data on side 1"), %% Modify one side [update_2a(Type, ?PB_BUCKET, Client, riakc_pb_socket) || - {Type, Client} <- lists:zip(?TYPES, [P1, P2, P1])], + {Type, Client} <- lists:zip(?TYPES, [P1, P2, P1, P2])], [update_2a(Type, ?HTTP_BUCKET, Client, rhc) || - {Type, Client} <- lists:zip(?TYPES, [H1, H2, H1])], + {Type, Client} <- lists:zip(?TYPES, [H1, H2, H1, H2])], lager:info("Check data is unmodified on side 2"), %% check value on one side is different from other [check_2b(Type, ?PB_BUCKET, Client, riakc_pb_socket) || - {Type, Client} <- lists:zip(?TYPES, [P4, P3, P4])], + {Type, Client} <- lists:zip(?TYPES, [P4, P3, P4, P3])], [check_2b(Type, ?HTTP_BUCKET, Client, rhc) || - {Type, Client} <- lists:zip(?TYPES, [H4, H3, H4])], + {Type, Client} <- lists:zip(?TYPES, [H4, H3, H4, H3])], lager:info("Modify data on side 2"), %% Modify other side [update_3b(Type, ?PB_BUCKET, Client, riakc_pb_socket) || - {Type, Client} <- lists:zip(?TYPES, [P3, P4, P3])], + {Type, Client} <- lists:zip(?TYPES, [P3, P4, P3, P4])], [update_3b(Type, ?HTTP_BUCKET, Client, rhc) || - {Type, Client} <- lists:zip(?TYPES, [H3, H4, H3])], + {Type, Client} <- lists:zip(?TYPES, [H3, H4, H3, H4])], lager:info("Check data is unmodified on side 1"), %% verify values differ [check_3a(Type, ?PB_BUCKET, Client, riakc_pb_socket) || - {Type, Client} <- lists:zip(?TYPES, [P2, P2, P1])], + {Type, Client} <- lists:zip(?TYPES, [P2, P2, P1, P1])], [check_3a(Type, ?HTTP_BUCKET, Client, rhc) || - {Type, Client} <- lists:zip(?TYPES, [H2, H2, H1])], + {Type, Client} <- lists:zip(?TYPES, [H2, H2, H1, H1])], %% heal lager:info("Heal and check merged values"), @@ -138,7 +140,7 @@ create_bucket_types([N1|_]=Nodes, Types) -> [Name, [{datatype, Type}, {allow_mult, true}]]) || {Name, Type} <- Types ], [rt:wait_until(N1, bucket_type_ready_fun(Name)) || {Name, _Type} <- Types], - [ rt:wait_until(N, bucket_type_matches_fun(Types)) || N <- Nodes]. + [rt:wait_until(N, bucket_type_matches_fun(Types)) || N <- Nodes]. bucket_type_ready_fun(Name) -> fun(Node) -> @@ -190,6 +192,13 @@ update_1({BType, map}, Bucket, Client, CMod) -> riakc_counter:increment(10, C) end, M1) end, + {BType, Bucket}, ?KEY, ?MODIFY_OPTS); +update_1({BType, hll}, Bucket, Client, CMod) -> + lager:info("update_1: Updating hyperloglog(set)"), + CMod:modify_type(Client, + fun(S) -> + riakc_hll:add_element(<<"Z">>, S) + end, {BType, Bucket}, ?KEY, ?MODIFY_OPTS). check_1({BType, counter}, Bucket, Client, CMod) -> @@ -202,7 +211,10 @@ check_1({BType, map}, Bucket, Client, CMod) -> lager:info("check_1: Checking map value is correct"), check_value(Client, CMod, {BType, Bucket}, ?KEY, riakc_map, [{{<<"followers">>, counter}, 10}, - {{<<"friends">>, set}, [<<"Russell">>]}]). + {{<<"friends">>, set}, [<<"Russell">>]}]); +check_1({BType, hll}, Bucket, Client, CMod) -> + lager:info("check_1: Checking hll value is correct"), + check_value(Client,CMod,{BType, Bucket},?KEY,riakc_hll,1). update_2a({BType, counter}, Bucket, Client, CMod) -> CMod:modify_type(Client, @@ -231,6 +243,14 @@ update_2a({BType, map}, Bucket, Client, CMod) -> end, M1) end, + {BType, Bucket}, ?KEY, ?MODIFY_OPTS); +update_2a({BType, hll}, Bucket, Client, CMod) -> + CMod:modify_type(Client, + fun(S) -> + riakc_hll:add_element( + <<"DANG">>, + riakc_hll:add_element(<<"Z^2">>, S)) + end, {BType, Bucket}, ?KEY, ?MODIFY_OPTS). check_2b({BType, counter}, Bucket, Client, CMod) -> @@ -243,7 +263,10 @@ check_2b({BType, map},Bucket,Client,CMod) -> lager:info("check_2b: Checking map value is unchanged"), check_value(Client, CMod, {BType, Bucket}, ?KEY, riakc_map, [{{<<"followers">>, counter}, 10}, - {{<<"friends">>, set}, [<<"Russell">>]}]). + {{<<"friends">>, set}, [<<"Russell">>]}]); +check_2b({BType, hll},Bucket,Client,CMod) -> + lager:info("check_2b: Checking hll value is unchanged"), + check_value(Client, CMod, {BType, Bucket}, ?KEY, riakc_hll, 1). update_3b({BType, counter}, Bucket, Client, CMod) -> CMod:modify_type(Client, @@ -273,6 +296,12 @@ update_3b({BType, map},Bucket,Client,CMod) -> end, M1) end, + {BType, Bucket}, ?KEY, ?MODIFY_OPTS); +update_3b({BType, hll}, Bucket, Client, CMod) -> + CMod:modify_type(Client, + fun(S) -> + riakc_hll:add_element(<<"Zedds Dead">>, S) + end, {BType, Bucket}, ?KEY, ?MODIFY_OPTS). check_3a({BType, counter}, Bucket, Client, CMod) -> @@ -287,7 +316,10 @@ check_3a({BType, map}, Bucket, Client, CMod) -> check_value(Client, CMod, {BType, Bucket}, ?KEY, riakc_map, [{{<<"followers">>, counter}, 10}, {{<<"friends">>, set}, [<<"Russell">>, <<"Sam">>]}, - {{<<"verified">>, flag}, true}]). + {{<<"verified">>, flag}, true}]); +check_3a({BType, hll}, Bucket, Client, CMod) -> + lager:info("check_3a: Checking hll value is unchanged"), + check_value(Client,CMod,{BType, Bucket},?KEY,riakc_hll,3). check_4({BType, counter}, Bucket, Client, CMod) -> lager:info("check_4: Checking final merged value of counter"), @@ -311,6 +343,14 @@ check_4({BType, map}, Bucket, Client, CMod) -> {{<<"followers">>, counter}, 10}, {{<<"friends">>, set}, [<<"Sam">>]}, {{<<"verified">>, flag}, true}], + [{pr, 3}, {notfound_ok, false}]); +check_4({BType, hll}, Bucket, Client, CMod) -> + lager:info("check_4: Checking final merged value of hll"), + check_value(Client, + CMod, {BType, Bucket}, + ?KEY, + riakc_hll, + 4, [{pr, 3}, {notfound_ok, false}]). check_value(Client, CMod, Bucket, Key, DTMod, Expected) -> @@ -322,7 +362,8 @@ check_value(Client, CMod, Bucket, Key, DTMod, Expected, Options) -> try Result = CMod:fetch_type(Client, Bucket, Key, Options), - lager:info("Expected ~p~n got ~p~n", [Expected, Result]), + lager:info("Expected ~p~n got ~p~n", [Expected, + Result]), ?assertMatch({ok, _}, Result), {ok, C} = Result, ?assertEqual(true, DTMod:is_type(C)), From 29efef03947aed912838ca767f7a85ea095f56e7 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Thu, 11 Aug 2016 09:35:58 -0400 Subject: [PATCH 80/89] Upload debug logs to Giddyup This can be disabled by setting the `giddyup_post_debug_logs` config value to `false` (it is `true` by default). --- src/riak_test_escript.erl | 33 ++++++++++++++++++++++++++------- src/rt.erl | 3 +++ src/rtdev.erl | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/riak_test_escript.erl b/src/riak_test_escript.erl index e5c7449ef..fc694fb8c 100644 --- a/src/riak_test_escript.erl +++ b/src/riak_test_escript.erl @@ -297,26 +297,45 @@ run_test(Test, Outdir, TestMetaData, Report, HarnessArgs, NumTests) -> 1 -> keep_them_up; _ -> rt:teardown() end, - CoverageFile = rt_cover:maybe_export_coverage(Test, CoverDir, erlang:phash2(TestMetaData)), + CoverageFile = rt_cover:maybe_export_coverage(Test, + CoverDir, + erlang:phash2(TestMetaData)), case Report of undefined -> ok; _ -> - {value, {log, L}, TestResult} = lists:keytake(log, 1, SingleTestResult), + {value, {log, L}, TestResult} = + lists:keytake(log, 1, SingleTestResult), case giddyup:post_result(TestResult) of error -> woops; {ok, Base} -> %% Now push up the artifacts, starting with the test log giddyup:post_artifact(Base, {"riak_test.log", L}), - [ giddyup:post_artifact(Base, File) || File <- rt:get_node_logs() ], - [giddyup:post_artifact(Base, {filename:basename(CoverageFile) ++ ".gz", - zlib:gzip(element(2,file:read_file(CoverageFile)))}) || CoverageFile /= cover_disabled ], - ResultPlusGiddyUp = TestResult ++ [{giddyup_url, list_to_binary(Base)}], - [ rt:post_result(ResultPlusGiddyUp, WebHook) || WebHook <- get_webhooks() ] + [giddyup:post_artifact(Base, File) + || File <- rt:get_node_logs()], + maybe_post_debug_logs(Base), + [giddyup:post_artifact( + Base, + {filename:basename(CoverageFile) ++ ".gz", + zlib:gzip(element(2,file:read_file(CoverageFile)))}) + || CoverageFile /= cover_disabled], + ResultPlusGiddyUp = TestResult ++ + [{giddyup_url, list_to_binary(Base)}], + [rt:post_result(ResultPlusGiddyUp, WebHook) || + WebHook <- get_webhooks()] end end, rt_cover:stop(), [{coverdata, CoverageFile} | SingleTestResult]. +maybe_post_debug_logs(Base) -> + case rt_config:get(giddyup_post_debug_logs, true) of + true -> + [giddyup:post_artifact(Base, File) + || File <- rt:get_node_debug_logs()]; + _ -> + false + end. + get_webhooks() -> Hooks = lists:foldl(fun(E, Acc) -> [parse_webhook(E) | Acc] end, [], diff --git a/src/rt.erl b/src/rt.erl index 32a609ec0..66eebb472 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -1825,6 +1825,9 @@ setup_harness(Test, Args) -> get_node_logs() -> ?HARNESS:get_node_logs(). +get_node_debug_logs() -> + ?HARNESS:get_node_debug_logs(). + %% @doc Performs a search against the log files on `Node' and returns all %% matching lines. -spec search_logs(node(), Pattern::iodata()) -> diff --git a/src/rtdev.erl b/src/rtdev.erl index 678b12994..12ecffbab 100644 --- a/src/rtdev.erl +++ b/src/rtdev.erl @@ -26,6 +26,8 @@ -define(DEVS(N), lists:concat(["dev", N, "@127.0.0.1"])). -define(DEV(N), list_to_atom(?DEVS(N))). -define(PATH, (rt_config:get(rtdev_path))). +-define(DEBUG_LOG_FILE(N), + "dev" ++ integer_to_list(N) ++ "@127.0.0.1-riak-debug.tar.gz"). get_deps() -> lists:flatten(io_lib:format("~s/dev/dev1/lib", [relpath(current)])). @@ -52,6 +54,17 @@ riak_admin_cmd(Path, N, Args) -> ExecName = rt_config:get(exec_name, "riak"), io_lib:format("~s/dev/dev~b/bin/~s-admin ~s", [Path, N, ExecName, ArgStr]). +riak_debug_cmd(Path, N, Args) -> + Quoted = + lists:map(fun(Arg) when is_list(Arg) -> + lists:flatten([$", Arg, $"]); + (_) -> + erlang:error(badarg) + end, Args), + ArgStr = string:join(Quoted, " "), + ExecName = rt_config:get(exec_name, "riak"), + io_lib:format("~s/dev/dev~b/bin/~s-debug ~s", [Path, N, ExecName, ArgStr]). + run_git(Path, Cmd) -> lager:info("Running: ~s", [gitcmd(Path, Cmd)]), {0, Out} = cmd(gitcmd(Path, Cmd)), @@ -765,6 +778,26 @@ get_node_logs() -> {lists:nthtail(RootLen, Filename), Port} end || Filename <- filelib:wildcard(Root ++ "/*/dev/dev*/log/*") ]. +get_node_debug_logs() -> + NodeMap = rt_config:get(rt_nodes), + rt:pmap(fun(Node) -> + get_node_debug_logs(Node) + end, + NodeMap). + +get_node_debug_logs({_Node, NodeNum}) -> + Path = relpath(node_version(NodeNum)), + Args = ["--logs"], + Cmd = riak_debug_cmd(Path, NodeNum, Args), + file:delete(?DEBUG_LOG_FILE(NodeNum)), + {ExitCode, Result} = wait_for_cmd(spawn_cmd(Cmd)), + case ExitCode of + 0 -> + ?DEBUG_LOG_FILE(NodeNum); + _ -> + exit({ExitCode, Result}) + end. + %% @doc Performs a search against the log files on `Node' and returns all %% matching lines. -spec search_logs(node(), Pattern::iodata()) -> From cfccfdd3c1df7c123ce8ee8000dd53dd67d89014 Mon Sep 17 00:00:00 2001 From: Jason Voegele Date: Tue, 16 Aug 2016 15:12:05 -0400 Subject: [PATCH 81/89] Use documented function to delete existing debug log file --- src/rtdev.erl | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/rtdev.erl b/src/rtdev.erl index 12ecffbab..9733a1da4 100644 --- a/src/rtdev.erl +++ b/src/rtdev.erl @@ -786,18 +786,27 @@ get_node_debug_logs() -> NodeMap). get_node_debug_logs({_Node, NodeNum}) -> + DebugLogFile = ?DEBUG_LOG_FILE(NodeNum), + delete_existing_debug_log_file(DebugLogFile), Path = relpath(node_version(NodeNum)), Args = ["--logs"], Cmd = riak_debug_cmd(Path, NodeNum, Args), - file:delete(?DEBUG_LOG_FILE(NodeNum)), {ExitCode, Result} = wait_for_cmd(spawn_cmd(Cmd)), case ExitCode of 0 -> - ?DEBUG_LOG_FILE(NodeNum); + DebugLogFile; _ -> exit({ExitCode, Result}) end. +%% If the debug log file exists from a previous test run it will cause the +%% `riak_debug_cmd' to fail. Therefore, delete the `DebugLogFile' if it exists. +%% Note that by ignoring the return value of `file:delete/1' this function +%% works whether or not the `DebugLogFile' actually exists at the time it is +%% called. +delete_existing_debug_log_file(DebugLogFile) -> + file:delete(DebugLogFile). + %% @doc Performs a search against the log files on `Node' and returns all %% matching lines. -spec search_logs(node(), Pattern::iodata()) -> From 30b0c015157e69c5b104e4a0d145370c843e524b Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Wed, 17 Aug 2016 11:35:50 -0400 Subject: [PATCH 82/89] Update client libraries to use develop-2.2 This pulls in the latest client code, as well as newer riak_pb stuff which is needed for the latest develop-2.2 Riak code. --- rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 72d707572..94200168e 100644 --- a/rebar.config +++ b/rebar.config @@ -15,8 +15,8 @@ {getopt, ".*", {git, "git://github.com/jcomellas/getopt", {tag, "v0.4"}}}, {meck, "0.8.2", {git, "git://github.com/basho/meck.git", {tag, "0.8.2"}}}, {mapred_verify, ".*", {git, "git://github.com/basho/mapred_verify", {branch, "master"}}}, - {riakc, "2.1.2", {git, "git://github.com/basho/riak-erlang-client", {tag, "2.1.2"}}}, - {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {tag, "2.1.2"}}}, + {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {branch, "develop-2.2"}}}, + {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {branch, "develop-2.2"}}}, {kvc, "1.3.0", {git, "https://github.com/etrepum/kvc", {tag, "v1.3.0"}}}, {druuid, ".*", {git, "git://github.com/kellymclaughlin/druuid.git", {tag, "0.2"}}} ]}. From a0ad732190b4a31843f67f7a1831af3e28085d05 Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Fri, 19 Aug 2016 12:26:11 -0400 Subject: [PATCH 83/89] Update dependency versions in prep for merge All the HLL stuff has been merged into the client develop-2.2 branches, so we can go ahead and update these dependencies to match that. --- rebar.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 20dfe3e39..94200168e 100644 --- a/rebar.config +++ b/rebar.config @@ -15,8 +15,8 @@ {getopt, ".*", {git, "git://github.com/jcomellas/getopt", {tag, "v0.4"}}}, {meck, "0.8.2", {git, "git://github.com/basho/meck.git", {tag, "0.8.2"}}}, {mapred_verify, ".*", {git, "git://github.com/basho/mapred_verify", {branch, "master"}}}, - {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {branch, "feature-zl-hll_datatypes"}}}, - {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {branch, "feature-zl-hll_datatypes"}}}, + {riakc, ".*", {git, "git://github.com/basho/riak-erlang-client", {branch, "develop-2.2"}}}, + {riakhttpc, ".*", {git, "git://github.com/basho/riak-erlang-http-client", {branch, "develop-2.2"}}}, {kvc, "1.3.0", {git, "https://github.com/etrepum/kvc", {tag, "v1.3.0"}}}, {druuid, ".*", {git, "git://github.com/kellymclaughlin/druuid.git", {tag, "0.2"}}} ]}. From 17301e8312447db5bbe005800cd58b8abd39fdc0 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Mon, 29 Aug 2016 14:58:08 -0400 Subject: [PATCH 84/89] Fixed yz_extractors test. --- src/rt.erl | 1 + src/yokozuna_rt.erl | 1 + tests/yz_extractors.erl | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rt.erl b/src/rt.erl index 66eebb472..738c49621 100644 --- a/src/rt.erl +++ b/src/rt.erl @@ -756,6 +756,7 @@ wait_for_service(Node, Services) when is_list(Services) -> {badrpc, Error} -> {badrpc, Error}; CurrServices when is_list(CurrServices) -> + lager:info("Waiting for services ~p: current services: ~p", [Services, CurrServices]), lists:all(fun(Service) -> lists:member(Service, CurrServices) end, Services); Res -> Res diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl index 95a54b7ac..fb80ef926 100644 --- a/src/yokozuna_rt.erl +++ b/src/yokozuna_rt.erl @@ -46,6 +46,7 @@ wait_for_index/2, wait_for_schema/2, wait_for_schema/3, + wait_until/2, write_data/5, write_data/6, http/4, diff --git a/tests/yz_extractors.erl b/tests/yz_extractors.erl index d21f0b699..0a03f516f 100644 --- a/tests/yz_extractors.erl +++ b/tests/yz_extractors.erl @@ -401,7 +401,7 @@ test_bad_extraction(Cluster) -> %% Verify the stats. There should be one more index failure, %% but there should be more more "melts" (error threshold failures) %% - yz_rt:wait_until( + yokozuna_rt:wait_until( Cluster, fun(_Node) -> check_error_stats(Cluster, PreviousFailCount, PreviousErrorThresholdCount) From 70ce356665c94d4548857608da050f60eefd058d Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Mon, 29 Aug 2016 19:55:21 +0000 Subject: [PATCH 85/89] Add back call to stop nodes before deleting index directories. Somehow managed to pass on 2.0.7, but shouldn't have (and doesn't in 2.2.0). --- src/yokozuna_rt.erl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl index 95a54b7ac..750bf88a0 100644 --- a/src/yokozuna_rt.erl +++ b/src/yokozuna_rt.erl @@ -243,6 +243,7 @@ brutal_kill_remove_index_dirs(Nodes, IndexName, Services) -> -spec remove_index_dirs([node()], index_name(), [atom()]) -> ok. remove_index_dirs(Nodes, IndexName, Services) -> IndexDirs = get_index_dirs(IndexName, Nodes), + [rt:stop(ANode) || ANode <- Nodes], remove_index_dirs2(Nodes, IndexDirs, Services), ok. From 61c1ea801cb2352644ce6c572e67aac8d02403a3 Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Tue, 30 Aug 2016 01:30:40 +0000 Subject: [PATCH 86/89] Wait for CRDT search results, rather than failing after the first attempt. --- src/yokozuna_rt.erl | 8 ++++ tests/yz_crdt.erl | 90 ++++++++++++++++++++++++--------------------- 2 files changed, 57 insertions(+), 41 deletions(-) diff --git a/src/yokozuna_rt.erl b/src/yokozuna_rt.erl index 750bf88a0..3a9ed096a 100644 --- a/src/yokozuna_rt.erl +++ b/src/yokozuna_rt.erl @@ -26,6 +26,7 @@ check_exists/2, clear_trees/1, commit/2, + drain_solrqs/1, expire_trees/1, gen_keys/1, host_entries/1, @@ -454,6 +455,13 @@ commit(Nodes, Index) -> rpc:multicall(Nodes, yz_solr, commit, [Index]), ok. +-spec drain_solrqs(node() | cluster()) -> ok. +drain_solrqs(Cluster) when is_list(Cluster) -> + [drain_solrqs(Node) || Node <- Cluster]; +drain_solrqs(Node) -> + rpc:call(Node, yz_solrq_drain_mgr, drain, []), + ok. + -spec override_schema(pid(), [node()], index_name(), schema_name(), string()) -> {ok, [node()]}. override_schema(Pid, Cluster, Index, Schema, RawUpdate) -> diff --git a/tests/yz_crdt.erl b/tests/yz_crdt.erl index 1a103815e..44e37f01a 100644 --- a/tests/yz_crdt.erl +++ b/tests/yz_crdt.erl @@ -59,59 +59,67 @@ confirm() -> ?KEY, riakc_map:to_op(Map2)), + yokozuna_rt:drain_solrqs(Nodes), yokozuna_rt:commit(Nodes, ?INDEX), - %% Perform simple queries, check for register, set fields. - {ok, {search_results, Results1a, _, _}} = riakc_pb_socket:search( + ok = rt:wait_until( + fun() -> + validate_search_results(Pid) + end), + %% Stop PB connection. + riakc_pb_socket:stop(Pid), + + pass. + +validate_search_results(Pid) -> + try + {ok, {search_results, Results1a, _, _}} = riakc_pb_socket:search( Pid, ?INDEX, <<"name_register:Chris*">>), - lager:info("Search name_register:Chris*: ~p~n", [Results1a]), - ?assertEqual(length(Results1a), 1), - ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results1a)), - list_to_binary(?KEY)), - ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results1a)), - <<"thing">>), - - {ok, {search_results, Results2a, _, _}} = riakc_pb_socket:search( + lager:info("Search name_register:Chris*: ~p~n", [Results1a]), + ?assertEqual(length(Results1a), 1), + ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results1a)), + list_to_binary(?KEY)), + ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results1a)), + <<"thing">>), + + {ok, {search_results, Results2a, _, _}} = riakc_pb_socket:search( Pid, ?INDEX, <<"interests_set:thing*">>), - lager:info("Search interests_set:thing*: ~p~n", [Results2a]), - ?assertEqual(length(Results2a), 1), - ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results2a)), - list_to_binary(?KEY)), - ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results2a)), - <<"thing">>), - - {ok, {search_results, Results3a, _, _}} = riakc_pb_socket:search( + lager:info("Search interests_set:thing*: ~p~n", [Results2a]), + ?assertEqual(length(Results2a), 1), + ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results2a)), + list_to_binary(?KEY)), + ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results2a)), + <<"thing">>), + + {ok, {search_results, Results3a, _, _}} = riakc_pb_socket:search( Pid, ?INDEX, <<"_yz_rb:testbucket">>), - lager:info("Search testbucket: ~p~n", [Results3a]), - ?assertEqual(length(Results3a), 1), - ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results3a)), - list_to_binary(?KEY)), - ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results3a)), - <<"thing">>), - - %% Redo queries and check if results are equal - {ok, {search_results, Results1b, _, _}} = riakc_pb_socket:search( + lager:info("Search testbucket: ~p~n", [Results3a]), + ?assertEqual(length(Results3a), 1), + ?assertEqual(?GET(<<"name_register">>, ?GET(?INDEX, Results3a)), + list_to_binary(?KEY)), + ?assertEqual(?GET(<<"interests_set">>, ?GET(?INDEX, Results3a)), + <<"thing">>), + + %% Redo queries and check if results are equal + {ok, {search_results, Results1b, _, _}} = riakc_pb_socket:search( Pid, ?INDEX, <<"name_register:Chris*">>), - ?assertEqual(number_of_fields(Results1a), - number_of_fields(Results1b)), + ?assertEqual(number_of_fields(Results1a), + number_of_fields(Results1b)), - {ok, {search_results, Results2b, _, _}} = riakc_pb_socket:search( + {ok, {search_results, Results2b, _, _}} = riakc_pb_socket:search( Pid, ?INDEX, <<"interests_set:thing*">>), - ?assertEqual(number_of_fields(Results2a), - number_of_fields(Results2b)), + ?assertEqual(number_of_fields(Results2a), + number_of_fields(Results2b)), - {ok, {search_results, Results3b, _, _}} = riakc_pb_socket:search( + {ok, {search_results, Results3b, _, _}} = riakc_pb_socket:search( Pid, ?INDEX, <<"_yz_rb:testbucket">>), ?assertEqual(number_of_fields(Results3a), number_of_fields(Results3b)), - - %% Stop PB connection. - riakc_pb_socket:stop(Pid), - - %% Clean cluster. - rt:clean_cluster(Nodes), - - pass. + true + catch Err:Reason -> + lager:info("Waiting for CRDT search results to converge. Error was ~p.", [{Err, Reason}]), + false + end. %% @private number_of_fields(Resp) -> From baf7653440f0298c69b793675ae430bd0c78e296 Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Wed, 15 Jun 2016 17:06:47 +0000 Subject: [PATCH 87/89] Update pipe_verify_handoff_blocking to provoke vnode_down errors. Use intercepts to provoke an otherwise-occasional error where a vnode has been transferred after the vnode proxy has gotten its pid but before the work can be sent. Will fail until fix to `riak_pipe_vnode:queue_work_wait` is merged. See basho/riak_pipe#106 for the fix. --- .../riak_core_vnode_master_intercepts.erl | 42 +++++++++++++++++++ tests/pipe_verify_handoff_blocking.erl | 32 +++++++------- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/intercepts/riak_core_vnode_master_intercepts.erl b/intercepts/riak_core_vnode_master_intercepts.erl index 87e28a29b..0902bc7fb 100644 --- a/intercepts/riak_core_vnode_master_intercepts.erl +++ b/intercepts/riak_core_vnode_master_intercepts.erl @@ -28,6 +28,19 @@ forwardable :: boolean(), opts = [] :: list()}). +-record(fitting, +{ + pid :: pid(), + ref :: reference(), + chashfun, + nval +}). + +-record(cmd_enqueue, {fitting :: #fitting{}, + input :: term(), + timeout, + usedpreflist}). + -define(M, riak_core_vnode_master_orig). @@ -52,4 +65,33 @@ stop_vnode_after_bloom_fold_request_succeeds(IndexNode, Req, Sender, VMaster) -> ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster) end; false -> ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster) + end. + +stop_pipe_vnode_after_request_sent(IndexNode, Req, Sender, VMaster) -> + case Req of + #cmd_enqueue{} = _Req -> + %% ?I_INFO("Intercepting riak_core_vnode_master:command_returning_vnode"), + random:seed(os:timestamp()), + case random:uniform(20) of + 5 -> + %% Simulate what happens when a VNode completes handoff between command_returning_vnode + %% and the fold attempting to start - other attempts to intercept and slow + %% certain parts of Riak to invoke the particular race condition were unsuccessful + ?I_INFO("Replaced VNode with spawned function in command_returning_vnode"), + Runner = self(), + VNodePid = spawn(fun() -> + Runner ! go, + exit(normal) + end), + receive + go -> ok + end, + %% Still need to send the work + ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster), + {ok, VNodePid}; + _ -> + ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster) + end; + _ -> + ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster) end. \ No newline at end of file diff --git a/tests/pipe_verify_handoff_blocking.erl b/tests/pipe_verify_handoff_blocking.erl index 4c48bdabc..b1d314f02 100644 --- a/tests/pipe_verify_handoff_blocking.erl +++ b/tests/pipe_verify_handoff_blocking.erl @@ -43,17 +43,6 @@ %% archive), but we don't want them to process so many inputs that %% they consume their blocking queues before handing off. -%% Please Note: Under rare circumstances, this test may fail with a -%% "{badmatch,{error,[{vnode_down,noproc}]}}' error. This is not a -%% failure of this test but rather a side effect of a race condition -%% in riak_core_vnode_proxy. It manifests due to the fact that the -%% test is attempting to send a command to a vnode that is in fact -%% down, however monitor only works by issuing a command and getting -%% a PID. In some instances, get_vnode_pid fails because vnode shutdown -%% is queued up in the mailbox before monitor node. Unfortunately, the -%% fix would require a fundamental shift in the architecture of -%% riak_core, which at the time of this writing is not feasible for -%% this rare failure case. -module(pipe_verify_handoff_blocking). -export([ @@ -80,9 +69,12 @@ confirm() -> Inputs = lists:seq(1, 20), lager:info("Start ~b nodes", [?NODE_COUNT]), - NodeDefs = lists:duplicate(?NODE_COUNT, {current, default}), + Config = [{riak_core, [{ring_creation_size, 8}, + {vnode_management_timer, 1000}, + {handoff_concurrency, 100}, + {vnode_inactivity_timeout, 1000}]}], Services = [riak_pipe], - [Primary,Secondary] = Nodes = rt:deploy_nodes(NodeDefs, Services), + [Primary,Secondary] = Nodes = rt:deploy_nodes(?NODE_COUNT, Config, Services), %% Ensure each node owns 100% of it's own ring [?assertEqual([Node], rt:owners_according_to(Node)) || Node <- Nodes], @@ -114,6 +106,7 @@ confirm() -> lager:info("Unpause workers"), Runner ! go, + set_up_vnode_crashing_intercept(Primary), ok = rt:wait_until_transfers_complete(Nodes), @@ -121,7 +114,10 @@ confirm() -> %% if we make it this far, then no filler ever saw the vnode_down %% error message; otherwise badmatches in queue_filler/4 will have - %% halted the test + %% halted the test. Note that we should not see this test fail + %% with `{error,[{vnode_down,noproc}]}}` errors, as the `noproc` case + %% should be handled similarly to the `normal` exit case in + %% `riak_pipe_vnode:queue_work_wait` _Status2 = pipe_status(Primary, Pipe), @@ -139,7 +135,13 @@ confirm() -> lager:info("~s: PASS", [atom_to_list(?MODULE)]), pass. -%%% queue filling +set_up_vnode_crashing_intercept(Primary) -> + lager:info("Add intercept to kill vnode before calling the wait function"), + rt_intercept:add(Primary, {riak_core_vnode_master, + [{{command_return_vnode, 4}, + stop_pipe_vnode_after_request_sent}]}). + +%% queue filling %% @doc fill pipe vnode queues by repeatedly sending each input in the %% input list until the queue reports timeout. From 486ea7502faafb0e101d5da2e6a66b76a60129d8 Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Fri, 17 Jun 2016 13:16:31 -0400 Subject: [PATCH 88/89] WIP --- .../riak_core_vnode_master_intercepts.erl | 61 +++++++------------ tests/pipe_verify_handoff_blocking.erl | 17 ++++-- 2 files changed, 33 insertions(+), 45 deletions(-) diff --git a/intercepts/riak_core_vnode_master_intercepts.erl b/intercepts/riak_core_vnode_master_intercepts.erl index 0902bc7fb..45bd3253d 100644 --- a/intercepts/riak_core_vnode_master_intercepts.erl +++ b/intercepts/riak_core_vnode_master_intercepts.erl @@ -28,19 +28,6 @@ forwardable :: boolean(), opts = [] :: list()}). --record(fitting, -{ - pid :: pid(), - ref :: reference(), - chashfun, - nval -}). - --record(cmd_enqueue, {fitting :: #fitting{}, - input :: term(), - timeout, - usedpreflist}). - -define(M, riak_core_vnode_master_orig). @@ -67,31 +54,25 @@ stop_vnode_after_bloom_fold_request_succeeds(IndexNode, Req, Sender, VMaster) -> false -> ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster) end. -stop_pipe_vnode_after_request_sent(IndexNode, Req, Sender, VMaster) -> - case Req of - #cmd_enqueue{} = _Req -> - %% ?I_INFO("Intercepting riak_core_vnode_master:command_returning_vnode"), - random:seed(os:timestamp()), - case random:uniform(20) of - 5 -> - %% Simulate what happens when a VNode completes handoff between command_returning_vnode - %% and the fold attempting to start - other attempts to intercept and slow - %% certain parts of Riak to invoke the particular race condition were unsuccessful - ?I_INFO("Replaced VNode with spawned function in command_returning_vnode"), - Runner = self(), - VNodePid = spawn(fun() -> - Runner ! go, - exit(normal) - end), - receive - go -> ok - end, - %% Still need to send the work - ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster), - {ok, VNodePid}; - _ -> - ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster) - end; +return_dead_process_pid_from_get_vnode_pid(Node, Index, VNodeMod = riak_pipe_vnode) -> + %% ?I_INFO("Intercepting riak_core_vnode_master:get_vnode_pid"), + random:seed(os:timestamp()), + case random:uniform(100) of + 7 -> + %% Simulate what happens when a VNode completes handoff between get_vnode_pid + %% and the fold attempting to start - other attempts to intercept and slow + %% certain parts of Riak to invoke the particular race condition were unsuccessful + ?I_INFO("Replaced VNode with spawned function in get_vnode_pid"), + VNodePid = spawn(fun() -> + ok + end), + MonRef = erlang:monitor(VNodePid), + receive + {'DOWN', MonRef, process, VNodePid, _Reason} -> ok + end, + {ok, VNodePid}; _ -> - ?M:command_return_vnode_orig(IndexNode, Req, Sender, VMaster) - end. \ No newline at end of file + ?M:get_vnode_pid_orig(Node, Index, VNodeMod) + end; +return_dead_process_pid_from_get_vnode_pid(Node, Index, VNodeMod) -> + ?M:get_vnode_pid_orig(Node, Index, VNodeMod). \ No newline at end of file diff --git a/tests/pipe_verify_handoff_blocking.erl b/tests/pipe_verify_handoff_blocking.erl index b1d314f02..d9f4def3d 100644 --- a/tests/pipe_verify_handoff_blocking.erl +++ b/tests/pipe_verify_handoff_blocking.erl @@ -110,6 +110,7 @@ confirm() -> ok = rt:wait_until_transfers_complete(Nodes), + lager:info("Check input count"), FillerInputCount = stop_fillers(Fillers), %% if we make it this far, then no filler ever saw the vnode_down @@ -118,7 +119,7 @@ confirm() -> %% with `{error,[{vnode_down,noproc}]}}` errors, as the `noproc` case %% should be handled similarly to the `normal` exit case in %% `riak_pipe_vnode:queue_work_wait` - + lager:info("Check pipe status"), _Status2 = pipe_status(Primary, Pipe), lager:info("Send eoi and collect results"), @@ -138,8 +139,8 @@ confirm() -> set_up_vnode_crashing_intercept(Primary) -> lager:info("Add intercept to kill vnode before calling the wait function"), rt_intercept:add(Primary, {riak_core_vnode_master, - [{{command_return_vnode, 4}, - stop_pipe_vnode_after_request_sent}]}). + [{{get_vnode_pid, 3}, + return_dead_process_pid_from_get_vnode_pid}]}). %% queue filling @@ -177,8 +178,14 @@ queue_filler(Node, Pipe, Inputs, Count) -> {stop, Owner} -> Owner ! {done, Count} after 0 -> {{value, I}, Q} = queue:out(Inputs), - ok = rpc:call(Node, riak_pipe, queue_work, [Pipe, I]), - queue_filler(Node, Pipe, queue:in(I, Q), Count+1) + case rpc:call(Node, riak_pipe, queue_work, [Pipe, I], 40000) of + ok -> + lager:info("Received response from queue_work"), + queue_filler(Node, Pipe, queue:in(I, Q), Count+1); + _ -> + lager:info("Timed out waiting for response from queue_work"), + queue_filler(Node, Pipe, queue:in(I, Q), Count+1) + end end. %% @doc tell all fillers to stop and collect and sum their send counts From 54a5e052f9521fd338187d8afdc6c4972997cd7c Mon Sep 17 00:00:00 2001 From: Doug Rohrer Date: Wed, 31 Aug 2016 07:41:40 -0400 Subject: [PATCH 89/89] WIP getting this test to fail again in the right way --- .../riak_core_vnode_manager_intercepts.erl | 48 +++++++++++++++++++ tests/pipe_verify_handoff_blocking.erl | 12 ++--- 2 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 intercepts/riak_core_vnode_manager_intercepts.erl diff --git a/intercepts/riak_core_vnode_manager_intercepts.erl b/intercepts/riak_core_vnode_manager_intercepts.erl new file mode 100644 index 000000000..e96fdb10a --- /dev/null +++ b/intercepts/riak_core_vnode_manager_intercepts.erl @@ -0,0 +1,48 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2015 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%%------------------------------------------------------------------- + +-module(riak_core_vnode_manager_intercepts). +-include("intercept.hrl"). +%% API +-compile(export_all). +-define(M, riak_core_vnode_manager_orig). + +return_dead_process_pid_from_get_vnode_pid(Index, VNodeMod = riak_pipe_vnode) -> + %% ?I_INFO("Intercepting riak_core_vnode_master:get_vnode_pid"), + random:seed(os:timestamp()), + case random:uniform(100) of + 7 -> + %% Simulate what happens when a VNode completes handoff between get_vnode_pid + %% and the fold attempting to start - other attempts to intercept and slow + %% certain parts of Riak to invoke the particular race condition were unsuccessful + ?I_INFO("Replaced VNode with spawned function in get_vnode_pid"), + VNodePid = spawn(fun() -> + ok + end), + MonRef = erlang:monitor(process, VNodePid), + receive + {'DOWN', MonRef, process, VNodePid, _Reason} -> ok + end, + {ok, VNodePid}; + _ -> + ?M:get_vnode_pid_orig(Index, VNodeMod) + end; +return_dead_process_pid_from_get_vnode_pid(Index, VNodeMod) -> + ?M:get_vnode_pid_orig(Index, VNodeMod). \ No newline at end of file diff --git a/tests/pipe_verify_handoff_blocking.erl b/tests/pipe_verify_handoff_blocking.erl index d9f4def3d..6840bd214 100644 --- a/tests/pipe_verify_handoff_blocking.erl +++ b/tests/pipe_verify_handoff_blocking.erl @@ -177,15 +177,9 @@ queue_filler(Node, Pipe, Inputs, Count) -> receive {stop, Owner} -> Owner ! {done, Count} after 0 -> - {{value, I}, Q} = queue:out(Inputs), - case rpc:call(Node, riak_pipe, queue_work, [Pipe, I], 40000) of - ok -> - lager:info("Received response from queue_work"), - queue_filler(Node, Pipe, queue:in(I, Q), Count+1); - _ -> - lager:info("Timed out waiting for response from queue_work"), - queue_filler(Node, Pipe, queue:in(I, Q), Count+1) - end + {{value, I}, Q} = queue:out(Inputs), + ok = rpc:call(Node, riak_pipe, queue_work, [Pipe, I]), + queue_filler(Node, Pipe, queue:in(I, Q), Count+1) end. %% @doc tell all fillers to stop and collect and sum their send counts