diff --git a/README.md b/README.md index de9680283..13932f5a4 100644 --- a/README.md +++ b/README.md @@ -3,163 +3,37 @@ Welcome to the exciting world of `riak_test`. ## What is Riak Test? -`riak_test` is a system for testing Riak clusters. Tests are written -in Erlang, and can interact with the cluster using distributed Erlang. +`riak_test` is a system for testing Riak clusters. Tests are written in Erlang, and can interact with the cluster using distributed Erlang. ### How does it work? -`riak_test` runs tests in a sandbox, typically `$HOME/rt/riak`. The sanbox -uses `git` to reset back to a clean state after tests are run. The +`riak_test` runs tests in a sandbox, typically `$HOME/rt/riak`. The sandbox uses `git` to reset back to a clean state after tests are run. The contents of `$HOME/rt/riak` might look something like this: ``` $ ls $HOME/rt/riak -current riak-1.4.12 riak-2.0.2 riak-2.0.4 riak-2.0.6 +current previous ``` -Inside each of these directories is a `dev` folder, typically -created with your normal `make [stage]devrel`. So how does +Inside each of these directories is a `dev` folder, typically created with your normal `make devrel`. So how does this sandbox get populated to begin with? -You'll create another directory that will contain full builds -of different version of Riak for your platform. Typically this directory -has been `~/test-releases` but it can be called anything and be anywhere -that you'd like. The `dev/` directory from each of these -releases will be copied into the sandbox (`$HOME/rt/riak`). -There are helper scripts in `bin/` which will -help you get both `~/test-releases` and `$HOME/rt/riak` all set up. A full -tutorial for using them exists further down in this README. - -There is one folder in `$HOME/rt/riak` that does not come from -`~/test-releases`: `current`. The `current` folder can refer -to any version of Riak, but is typically used for something -like the `master` branch, a feature branch, or a release candidate. -The `$HOME/rt/riak/current` dev release gets populated from a devrel of Riak -that can come from anywhere, but is usually your 'normal' git checkout -of Riak. The `bin/rtdev-current.sh` can be run from within that folder -to copy `dev/` into `$HOME/rt/riak/current`. - -Once you have everything set up (again, instructions for this are below), -you'll want to run and write tests. This repository also holds code for -an Erlang application called `riak_test`. The actual tests exist in -the `test/` directory. +You'll create another directory that will contain full builds of different version of Riak for your platform. Typically this directory +has been `~/test-releases` but it can be called anything and be anywhere that you'd like. The `dev/` directory from each of these releases will be copied into the sandbox (`$HOME/rt/riak`). -## Bootstrapping Your Test Environment +There are helper scripts in `bin/` which will help you get both `~/test-releases` and `$HOME/rt/riak` all set up. -Running tests against a -development version of Riak is just one of the things that you can do -with riak_test. You can also test things involving upgrading from -previous versions of Riak. Together, we'll get your test environment -up and running. Scripts to help in this process are located in the -`bin` directory of this project. - -### rtdev-all.sh - -This script is for the lazy. It performs all of the setup steps described -in the other scripts, including installing the current "master" branch from -Github into "current". The releases will be built in your current working -directory, so create an empty one in a place you'd like to store these -builds for posterity, so that you don't have to rebuild them if your -installation path (`$HOME/rt/riak` by the way this script installs it) gets into -a bad state. - -If you do want to restore your `$HOME/rt/riak` folder to factory condition, see -`rtdev-setup-releases.sh` and if you want to change the current riak under -test, see `rtdev-current.sh`. - -### rtdev-build-releases.sh - -The first one that we want to look at is `rtdev-build-releases.sh`. If -left unchanged, this script is going to do the following: - -1. Download the source for the past three major Riak versions (e.g. - 1.4.12, 2.0.6 and 2.1.2) -1. Build the proper version of Erlang that release was built with, - using kerl (which it will also download) -1. Build those releases of Riak. - -You'll want to run this script from an empty directory. Also, you might be -thinking that you already have all the required versions of erlang. Great! You -should set and export the following environment variables prior to running this -and other `riak_test` scripts: - -Here, kerl is configured to use "$HOME/.kerl/installs" as the installation -directory for erlang builds. - -```bash -export R15B01="$HOME/.kerl/installs/erlang-R15B01" -export R16B02="$HOME/.kerl/installs/erlang-R16B02" -export CURRENT_OTP="$R16B02" -``` -**Kerlveat**: If you want kerl to build erlangs with serious 64-bit -Macintosh action, you'll need a `~/.kerlrc` file that looks like this: +Once you have everything set up (again, instructions for this are below), you'll want to run and write tests. This repository also holds code for +an Erlang application called `riak_test`. The actual tests exist in the `test/` directory. -``` -KERL_CONFIGURE_OPTIONS="--disable-hipe --enable-smp-support --enable-threads --enable-kernel-poll --enable-darwin-64bit --without-odbc" -``` +## Bootstrapping Your Test Environment -The script will check that all these paths exist. If even one of them -is missing, it will prompt you to install kerl, even if you already -have kerl. If you say no, the script quits. If you say yes, or all of -your erlang paths check out, then go get a cup of coffee, you'll be -building for a little while. - -### rtdev-setup-releases.sh - -The `rtdev-setup-releases.sh` will get the releases you just built -into a local git repository. Run this script from the -same directory that you just built all of your releases into. -By default this script initializes the repository into `$HOME/rt/riak` but -you can override [`$RT_DEST_DIR`](https://github.com/basho/riak_test/blob/master/bin/rtdev-setup-releases.sh#L11). - -### rtdev-current.sh - -`rtdev-current.sh` is where it gets interesting. You need to run that -from the Riak source folder you're wanting to test as the current -version of Riak. Also, make sure that you've already run `make devrel` -or `make stagedevrel` before you run `rtdev-current.sh`. Like setting up -releases you can override [`$RT_DEST_DIR`](https://github.com/basho/riak_test/blob/master/bin/rtdev-current.sh#L6) -so all your riak builds are in one place. Also you can override the tag -of the current version pulled by setting [`$RT_CURRENT_TAG`](https://github.com/basho/riak_test/blob/master/bin/rtdev-current.sh#L7) -to a release number, e.g. `2.0.0`. It will automatically be prefixed with -the repo name, e.g. `riak_ee-2.0.0`. To use `riak_ee` instead of `riak` set [`$RT_USE_EE`](https://github.com/basho/riak_test/blob/master/bin/rtdev-setup-releases.sh#L23) -to any non-empty string. - -### rtdev-install.sh - -`rtdev-install.sh` is exactly the same as `rtdev-current.sh`, however, -you can use arbitrary names like `riak-2.1.2` instead of just `current`. -The single argument supplied to this script is that directory name. - -#### reset-current-env.sh - -`reset-current-env.sh` resets test environments setup using `rtdev-current.sh` -using the following process: - - 1. Delete the current stagedevrel/devrel environment - 1. `make stagedevrel` for the Riak release being tested (current default is 2.0, - overidden with the `-v` flag). When the `-c` option is specified, - `make devclean` will be executed before rebuilding. - 1. Execute `rtdev-current.sh` for the Riak release being tested - 1. Rebuild the current riak_test branch. When the `-c` option is specified, - 'make clean' will be executed before rebuilding. - -This script is intended to provide to cover the common test environment -reset method -- not cover all possible testing configurations/scenarios. -As such, it makes the following assumptions regarding the environment and -test operation: - - * Riak source trees will be symlinked into the riak_test root directory - as `riak-`. - * The script will be located in a sub-directory of . It - can be executed from any directory, but it uses the script location to - determine the riak_test home directory. +[For simple setup instructions follow this guide](doc/SIMPLE_SETUP.md). ### Config file. -Now that you've got your releases all ready and gitified, you'll need -to tell riak_test about them. The method of choice is to create a +Now that you've got your releases all ready and gitified, you'll need to tell riak_test about them. The method of choice is to create a `~/.riak_test.config` that looks something like this: ```erlang @@ -188,11 +62,7 @@ to tell riak_test about them. The method of choice is to create a ]}. ``` -The `default` section of the config file will be overridden by the config -name you specify. For example, running the command below will use an -`rt_retry_delay` of 500 and an `rt_max_wait_time` of 180000. If your -defaults contain every option you need, you can run riak_test without -the `-c` argument. +The `default` section of the config file will be overridden by the config name you specify. For example, running the command below will use an `rt_retry_delay` of 500 and an `rt_max_wait_time` of 180000. If your defaults contain every option you need, you can run riak_test without the `-c` argument. Some configuration parameters: @@ -253,13 +123,12 @@ This is an example test result JSON message posted to a webhook: "giddyup_url": "http://giddyup.basho.com/test_results/53" } ``` Notice that the giddyup URL is not the page for the test result, but a resource from which you can GET information about the test in JSON. + ### Running riak_test for the first time -Run a test! After running `make` from the root of your `riak_test` -clone just `./riak_test -c rtdev -t verify_build_cluster` +Run a test! After running `make` from the root of your `riak_test` clone just `./riak_test -c rtdev -t verify_build_cluster` -Did that work? Great, try something harder: `./riak_test -c -rtdev -t loaded_upgrade` +Did that work? Great, try something harder: `./riak_test -c rtdev -t loaded_upgrade` diff --git a/doc/SIMPLE_SETUP.md b/doc/SIMPLE_SETUP.md index 4af0be18a..e334a1a9c 100644 --- a/doc/SIMPLE_SETUP.md +++ b/doc/SIMPLE_SETUP.md @@ -1,79 +1,50 @@ # Simple Setup Process -This intended as a simplified setup set-by-step guide (in comparison to the [README](../README.md)). +This intended as a simplified setup step-by-step guide to running `riak_test` tests on a fresh machine. ## Decide Paths You need to decide on destinations for the following: -- A runtime for Riak instances running tests (e.g. "~/rt/riak") +- A runtime for Riak instances running tests (e.g. `~/rt/riak`) -- A destination to install yokozuna tests should you wish to run yokozuna tests (e.g. "\~/test_sw/yokozuna") and basho_bench (e.g. "\~/test_sw/basho_bench") +- A destination in which to build riak releases to be tested (e.g. `~/test_build/current/riak` and `~test_build/previous/riak`) -- A destination in which to build riak releases to be tested (e.g. "~/test_build/riak") +- A location for the riak_test software itself, where the tests will be built and test scripts will be run and logs will be stored (e.g. `~/riak_test`) -- A location for the riak_test software itself, where the tests will be built and test scripts will be run and logs will be stored (e.g. "~/riak_test") +There will also need to be a test configuration file (`.riak_test.config`), which must be in the root of the home directory of the user running tests. -There will also need to be a test configuration file (".riak_test.config"), which must be in the root of the home directory of the user running tests. +## Prerequisites + +Erlang must be installed and running on the machine. A simple way of achieving this is via [kerl](https://github.com/kerl/kerl). All parts should be built and the tests run using the same Erlang version: riak_test, current riak and previous riak. ## Clone Repos -Clone riak into your build area i.e.: +Clone riak into your build area, and switch to the tag or branch you wish to test i.e.: +```bash +cd ~/test_build/current +git clone https://github.com/basho/riak +cd riak +git checkout riak-3.2.3 ``` -cd ~/test_build + +```bash +cd ~/test_build/previous git clone https://github.com/basho/riak +cd riak +git checkout riak-3.2.1 ``` -Clone riak_test into your test software location, checkout the test branch you intend to use, and make riak_test i.e.: +Clone riak_test into your test software location, checkout the test branch you intend to use, and `make all` i.e.: -``` +```bash cd ~ git clone https://github.com/basho/riak_test cd riak_test -git checkout develop-2.9 -make all -``` - -Clone yokozuna into your test software location, checkout the test branch you intend to use, and make test i.e.: - -``` -cd ~/test_sw -git clone https://github.com/basho/yokozuna -cd yokozuna -git checkout develop-2.9 +git checkout openriak-3.4 make all -make test -``` - -Also within yokozuna make the yokozuna bench test scripts (which are called within some of the yokozuna tests) - ``` -cd ~/test_sw/yokozuna/misc/bench -../../rebar get-deps -../../rebar compile -``` - -Clone basho_bench into your test software location, checkout the test branch you intend to use, and make i.e.: - -``` -cd ~/test_sw -git clone https://github.com/basho/basho_bench -cd basho_bench -git checkout develop-2.9 -make all -``` - -## Setup Test Runtime - -Initialise git in the test runtime environment i.e.: - -``` -cd ~/riak_test -./bin/rtdev-setup-releases.sh -``` - -This script will make `~/rt/riak` the path to setup releases. If this is to be overridden, then it ca be do so by override [$RT_DEST_DIR](../bin/rtdev-setup-releases.sh#L11) ## Setup initial test Configuration @@ -81,96 +52,134 @@ Create a `~/.riak_test.config` file with the following sample configuration: ``` {default, [ + {rt_max_wait_time, 600000}, {rt_retry_delay, 1000}, {rt_harness, rtdev}, {rt_scratch_dir, "/tmp/riak_test_scratch"}, - {spam_dir, "~/riak_test/search-corpus/spam.0"}, - {platform, "osx-64"} + {spam_dir, "/Users/sgtwilko/dbtest/OpenRiak/riak_test/search-corpus/spam.0"}, + {platform, "osx-64"}, + {conn_fail_time, 60000}, + {organisation, nhse}, + {deps, ["./_build/default/lib/"]} + ]}. {rtdev, [ + {rt_project, "riak"}, - {basho_bench, "~/test_sw/basho_bench"}, - {yz_dir, "~/test_sw/yokozuna"}, - {test_paths, ["~/test_sw/yokozuna/riak_test/ebin"]}, - {rtdev_path, [{root, "~/rt/riak"}, - {current, "~/rt/riak/current"}, - {previous, "~/rt/riak/previous"}, - {legacy, "~/rt/riak/riak-2.0.5"}, - {"2.0.5", "~/rt/riak/riak-2.0.5"} - ]} + {rt_default_config, [{riak_kv, [{handoff_deletes, true}]}]}, + {test_paths, ["_build/test/lib/riak_test/tests"]}, + { + rtdev_path, + [ + {root, "/Users/sgtwilko/rt/riak"}, + {current, "/Users/sgtwilko/rt/riak/current"}, + {previous, "/Users/sgtwilko/rt/riak/previous"} + ] + } ]}. ``` -The `spam_dir` will need to point at the location of the riak_test software. At that location there will be a tarred/zipped file containing test data, which you will need to untar/unzip to populate the spam directory. +The `spam_dir` will need to point at the location of the riak_test software. At that location there will be a tarred/zipped file containing test data, which you will need to untar/unzip to populate the spam directory. This is only relevant to a single test - `partition_repair`. Some tests may also require a version of OpenSSL to be available within the PATH. The platform shout be set to `osx-64` or `linux`. -When testing yokozuna using the `group.sh` script, the yokozuna riak_tests are not copied as part of the initialisation of the script and so need to be found through setting the path in `test_paths`. There are three yokozuna tests that require a specific reference to the "2.0.5" build, hence the last line in the `rtdev_path` list. +There are issues with using relative references for some paths, so use fully-qualified file paths were possible. -## Build each Test version +## Setup Test Runtime -Within the test_build are, it is necessary to build each release and copy them over to the run time environment. +Initialise git in the test runtime environment i.e.: +```bash +cd ~/riak_test +./bin/rtdev-setup-releases.sh ``` -# to make develop-2.9 'current' -git checkout develop-2.9 +This script will make `~/rt/riak` the path to setup releases, so make sure this is the same path used in the `.riak_test.config` file. If another path is used, then the path used by the script can be overridden by changing [$RT_DEST_DIR](../bin/rtdev-setup-releases.sh#L11). The test folders are reset back to default each time using git (and this is what the setup script will initiate). + +## Build each Test version + +Within the test_build are, it is necessary to build each release and copy them over to the run time environment using the install script. + +Assuming each riak folder is currently set to the correct release or branch: + +```bash +cd ~/test_build/current make devclean; make devrel ~/riak_test/bin/rtdev-install.sh current -# to make Riak 2.2.6 'previous' -git checkout riak-2.2.6 +cd ~/test_build/previous make devclean; make devrel ~/riak_test/bin/rtdev-install.sh previous - -# to make Riak 2.0.5 'legacy' -git checkout riak-2.0.5 -make devclean; make devrel -~/riak_test/bin/rtdev-install.sh legacy ``` -Each time you change a branch for testing, re-run the `make devclean; make devrel` and the `rtdev-install.sh` script. - -If you intend to test cluster upgrade tests that include replication, then you will need to use the enterprise release of Riak for nay release prior to `2.2.5`. - -## Install JDK - -To test yokozuna, it will be necessary to install a JDK. The install instructions on Yokozuna github recommend "Java 1.6 or later, Oracle 7u25 is recommended". Tests should run successfully if the retired [Java 7 is used](https://java.com/en/download/faq/java_7.xml), but please be aware of the security implications of running this software. - -## Redbug support +Each time you change a branch for testing, re-run the `make devclean; make devrel` and the `rtdev-install.sh` script to update the current or previous test destination as appropriate. When changing to a tag, the tag will be made with the correct `rebar.lock` file, however this will not necessarily be the case with a branch. If looking to test a branch rather than a tag, always remove the `rebar.lock` file from the root of the riak folder before running `make devclean; make devrel`. -tba - further work required to get redbug tests working. +The riak `make devrel` process does not make a full copy of the riak installation, using symbolic links instead. So it is important not to modify these riak folders even after running the install script - as tests may be impacted. ## Run Tests -There are two primary ways of running tests: +There are two primary ways of running tests: as a group; or running a single test in isolation. ### Run a test 'group' There are pre-defined groups of tests under the `groups` folder in `riak_test`, or you could define your own. The group is a list of test names, and the test script will run each of these tests in alphabetical order. -``` -# To run the 'kv_all' group with the bitcask backend +```bash +# To run the 'kv_all' group with the bitcask backend cd ~/riak_test ./group.sh -g kv_all -c rtdev -b bitcask -# To run the '2i_all' group with the leveled backend +# To run the '2i_all' group with the leveled backend cd ~/riak_test ./group.sh -g 2i_all -c rtdev -b leveled ``` The `-c` reference must refer to a stanza in your `~/.riak_test.config`. +Test groups will ignore failures, and continue with other tests in the group. At the end of all tests, a report of pass/fail by test is printed to screen, and to a log file. + +The following test groups and backends are tested as part of a release: + +``` +kv_all (leveled) +nextgenrepl (leveled) +core_all (leveled) +datatypes_all (leveled) +repl_all (leveled) +2i_all (leveled) +ensemble (leveled) +admin_all (leveled) +mapred_all (leveled) +pipe_all (leveled) +rtc_all (leveled) +kv_all (bitcask) +nextgenrepl (bitcask) +bitcask_only (bitcask) +smoke (eleveldb) +eleveldb_only (eleveldb) +``` + +Test groups `kv_all`, `nextgenrepl`, `repl_all` and `smoke` will each take multiple hours to run. Most of the remaining test groups will run within an hour. + +The `smoke` test group is intended to give good general coverage of test scenarios. The `vape` group is a lightweight version of the `smoke` group. Note that a 2i capable backend is required for both `smoke` and `vape`. + +The tests contained in each group are listed within the group file in the groups folder for riak_test (e.g. `groups/kv_all` for the kv_all tests). Tests are run in alphabetic order (ignoring the order tests are listed in the group). + ### Run an individual test Tests can be run individually, with an additional facility that failing tests will be paused at the point they fail, if they fail. This allows you to inspect the state of the cluster (within `~/rtdev/riak`) at the failure point. -``` +```bash # e.g. run verify_crdt_capability with leveled backend ./riak_test -c rtdev -t verify_crdt_capability -b leveled - -# e.g. run an individual yokozuna test -./riak_test -c rtdev -t ~/riak_test_deps/yokozuna/riak_test/ebin/yz_security ``` + +Individual tests will abort on failure, but leave the riak instances running so that you can attach to them via `riak remote_console` and examine the state at the point of the failure. The logs for each node will be available in the `~/rt/riak/current/dev/dev{n}/riak/log` path. + +When updating a test, run `make all` before re-running the test. + +Each test is an individual file within the `tests` folder. Only tests that are within a group included in the release set are confirmed to pass, other tests may have undetected failures unrelated to any recent riak changes. For failing tests, including intermittent failures, then please troubleshoot and raise an issue on [OpenRiak riak_test github](https://github.com/OpenRiak/riak_test/issues). + +Any test changes in riak_test should also be verified using `./rebar3 as check do dialyzer`. \ No newline at end of file diff --git a/tests/verify_2i_aae.erl b/tests/verify_2i_aae.erl index 49574d067..d3ef2166a 100644 --- a/tests/verify_2i_aae.erl +++ b/tests/verify_2i_aae.erl @@ -46,7 +46,6 @@ confirm() -> [{{index_specs, 1}, skippable_index_specs}, {{diff_index_specs, 2}, skippable_diff_index_specs}]}), ?LOG_INFO("Installed intercepts to corrupt index specs on node ~0p", [Node1]), - %%rpc:call(Node1, lager, set_loglevel, [lager_console_backend, debug]), PBC = rt:pbc(Node1), NumItems = ?NUM_ITEMS, NumDel = ?NUM_DELETES, @@ -177,8 +176,9 @@ run_2i_repair(Node1) -> MaxWaitTime = rt_config:get(rt_max_wait_time), receive {'DOWN', Mon, _, _, Status} -> - ?LOG_INFO("Status: ~0p", [Status]), - Status + ?LOG_INFO("Status: ~p", [Status]), + timer:sleep(1000), + Status after MaxWaitTime -> ?LOG_ERROR("Timed out (~wms) waiting for 2i AAE repair process", [MaxWaitTime]), @@ -194,6 +194,7 @@ to_key(N) -> put_obj(PBC, Bucket, N, IN, Index) -> K = to_key(N), + timer:sleep(1), Obj = case riakc_pb_socket:get(PBC, Bucket, K) of {ok, ExistingObj} -> @@ -208,6 +209,7 @@ put_obj(PBC, Bucket, N, IN, Index) -> del_obj(PBC, Bucket, N) -> K = to_key(N), + timer:sleep(1), case riakc_pb_socket:get(PBC, Bucket, K) of {ok, ExistingObj} -> ?assertMatch(ok, riakc_pb_socket:delete_obj(PBC, ExistingObj)); @@ -218,13 +220,19 @@ del_obj(PBC, Bucket, N) -> assert_range_query(Pid, Bucket, Expected0, Index, StartValue, EndValue) -> ?LOG_INFO("Searching Index ~0p/~0p for ~0p-~0p", [Bucket, Index, StartValue, EndValue]), - {ok, ?INDEX_RESULTS{terms=Keys}} = riakc_pb_socket:get_index_range(Pid, Bucket, Index, StartValue, EndValue, [{return_terms, true}]), - Actual = case Keys of - undefined -> - []; - _ -> - lists:sort(Keys) - end, + {ok, ?INDEX_RESULTS{terms=Keys}} = + riakc_pb_socket:get_index_range( + Pid, Bucket, Index, StartValue, EndValue, [{return_terms, true}]), + Actual = + case Keys of + undefined -> + []; + _ -> + lists:sort(Keys) + end, Expected = lists:sort(Expected0), - ?assertEqual({Bucket, Expected}, {Bucket, Actual}), + MissingKeys = Expected -- Actual, + AdditionalKeys = Actual -- Expected, + ?assertEqual({Bucket, []}, {Bucket, MissingKeys}), + ?assertEqual({Bucket, []}, {Bucket, AdditionalKeys}), ?LOG_INFO("Yay! ~b (actual) == ~b (expected)", [length(Actual), length(Expected)]). diff --git a/tests/verify_dt_converge.erl b/tests/verify_dt_converge.erl index a325127b9..c23d92571 100644 --- a/tests/verify_dt_converge.erl +++ b/tests/verify_dt_converge.erl @@ -52,18 +52,30 @@ -define(MODIFY_OPTS, [create]). confirm() -> - Config = [ - {riak_kv, [ - {handoff_concurrency, 16}, - {anti_entropy, {off, []}} - ]}, - {riak_core, [ - {ring_creation_size, 16} - ]} - ], + Config = + [ + { + riak_kv, + [ + {handoff_concurrency, 16}, + {anti_entropy, {off, []}} + ] + }, + { + riak_core, + [ + {ring_creation_size, 16}, + {vnode_management_timer, 2000}, + {vnode_inactivity_timeout, 4000}, + {handoff_concurrency, 16} + ] + } + ], [N1, N2, N3, N4] = Nodes = rt:build_cluster(4, Config), + rt:wait_until_transfers_complete(Nodes), + create_bucket_types(Nodes, ?TYPES), [P01, P02, P03, P04] = create_pb_clients(Nodes), @@ -277,13 +289,20 @@ update_2a({BType, hll}, Bucket, Client, CMod) -> end, {BType, Bucket}, ?KEY, ?MODIFY_OPTS); update_2a({BType, gset}, Bucket, Client, CMod) -> - CMod:modify_type(Client, - fun(S) -> - riakc_gset:add_element( - <<"DANG">>, - riakc_gset:add_element(<<"Z^2">>, S)) - end, - {BType, Bucket}, ?KEY, ?MODIFY_OPTS). + R = + CMod:modify_type( + Client, + fun(S) -> + riakc_gset:add_element( + <<"DANG">>, + riakc_gset:add_element(<<"Z^2">>, S)) + end, + {BType, Bucket}, + ?KEY, + ?MODIFY_OPTS + ), + ?LOG_INFO("Update 2a for GSET with CMod ~p result ~p", [CMod, R]), + R. check_2b({BType, counter}, Bucket, Client, CMod) -> @@ -415,24 +434,26 @@ check_value(Client, CMod, Bucket, Key, DTMod, Expected) -> [{pr, 1}, {r,2}, {notfound_ok, true}, {timeout, 5000}]). check_value(Client, CMod, Bucket, Key, DTMod, Expected, Options) -> - rt:wait_until(fun() -> - try - Result = CMod:fetch_type(Client, Bucket, Key, - Options), - ?LOG_INFO("Expected ~0p got ~0p", [Expected, - Result]), - ?assertMatch({ok, _}, Result), - {ok, C} = Result, - ?assertEqual(true, DTMod:is_type(C)), - ?assertEqual(Expected, DTMod:value(C)), - true - catch - Type:Error -> - ?LOG_DEBUG("check_value(~0p,~0p,~0p,~0p,~0p) " - "failed: ~0p:~0p", [Client, Bucket, - Key, DTMod, - Expected, Type, - Error]), - false - end - end). + rt:wait_until( + fun() -> + try + Result = CMod:fetch_type(Client, Bucket, Key, Options), + ?LOG_INFO( + "Expected ~p~n got ~p~n", + [Expected, Result]), + ?assertMatch({ok, _}, Result), + {ok, C} = Result, + ?assertEqual(true, DTMod:is_type(C)), + ?assertEqual(Expected, DTMod:value(C)), + true + catch + Type:Error -> + ?LOG_DEBUG( + "check_value(~p,~p,~p,~p,~p) failed: ~p:~p", + [Client, Bucket, Key, DTMod, Expected, Type, Error]), + false + end + end, + 10, + 1000 + ).