Merge pull request #1284 from basho/feature/verify-retry-logic-to-get…

…-blocks

Add a test to verify a retry logic after getting the block fails with nval=1

Reviewed-by: shino

riak_test/intercepts/intercept.hrl

@@ -0,0 +1,23 @@
+%% -------------------------------------------------------------------
+%%
+%% Copyright (c) 2015 Basho Technologies, Inc.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%%-------------------------------------------------------------------
+
+-define(I_TAG(S), "INTERCEPT: " ++ S).
+-define(I_INFO(Msg), error_logger:info_msg(?I_TAG(Msg))).
+-define(I_INFO(Msg, Args), error_logger:info_msg(?I_TAG(Msg), Args)).

riak_test/intercepts/riak_cs_block_server_intercepts.erl

@@ -20,7 +20,18 @@
 
 -module(riak_cs_block_server_intercepts).
 -compile(export_all).
+-include("intercept.hrl").
 -define(M, riak_cs_block_server_orig).
 
 get_block_local_timeout(_RcPid, _FullBucket, _FullKey, _GetOptions, _Timeout, _StatsKey) ->
     {error, timeout}.
+
+get_block_local_insufficient_vnode_at_nval1(RcPid, FullBucket, FullKey, GetOptions, Timeout, StatsKey) ->
+    case proplists:get_value(n_val, GetOptions) of
+        1 ->
+            ?I_INFO("riak_cs_block_server:get_block_local/6 returns insufficient_vnodes"),
+            {error, <<"{insufficient_vnodes,0,need,1}">>};
+        N ->
+            ?I_INFO("riak_cs_block_server:get_block_local/6 forwards original code with n_val=~p", [N]),
+            ?M:get_block_local_orig(RcPid, FullBucket, FullKey, GetOptions, Timeout, StatsKey)
+    end.

riak_test/tests/object_get_test.erl

@@ -66,6 +66,12 @@ non_mp_get_cases(UserConfig) ->
     basic_get_test_case(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock, UserConfig),
     basic_get_test_case(?TEST_BUCKET, ?KEY_MULTIPLE_BLOCK, MultipleBlock, UserConfig),
 
+    %% GET after nval=1 GET failure
+    rt_intercept:add(rtcs:cs_node(1), {riak_cs_block_server, [{{get_block_local, 6}, get_block_local_insufficient_vnode_at_nval1}]}),
+    Res = erlcloud_s3:get_object(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, UserConfig),
+    ?assertEqual(SingleBlock, proplists:get_value(content, Res)),
+    rt_intercept:clean(rtcs:cs_node(1), riak_cs_block_server),
+
     %% Range GET for single-block object test cases
     [range_get_test_case(?TEST_BUCKET, ?KEY_SINGLE_BLOCK, SingleBlock,
                          Range, UserConfig)