Allow metadata attempts to be retried

Note that this does not plumb anything into the session via
config vars.  I think that should be a separate pull request.

botocore/credentials.py

@@ -36,6 +36,11 @@
 )
 
 
+class _RetriesExceededError(Exception):
+    """Internal exception used when the number of retries are exceeded."""
+    pass
+
+
 class Credentials(object):
     """
     Holds the credentials needed to authenticate requests.  In addition
@@ -58,33 +63,50 @@ def __init__(self, access_key=None, secret_key=None, token=None):
 
 
 def retrieve_iam_role_credentials(url=METADATA_SECURITY_CREDENTIALS_URL,
-                                  timeout=None):
+                                  timeout=None, num_retries=0):
     if timeout is None:
         timeout = DEFAULT_METADATA_SERVICE_TIMEOUT
     d = {}
     try:
-        r = requests.get(url, timeout=timeout)
-        if r.status_code == 200 and r.content:
+        r = _get_request(url, timeout, num_retries)
+        if r.content:
             fields = r.content.decode('utf-8').split('\n')
             for field in fields:
                 if field.endswith('/'):
-                    d[field[0:-1]] = retrieve_iam_role_credentials(url + field)
+                    d[field[0:-1]] = retrieve_iam_role_credentials(
+                        url + field, timeout, num_retries)
                 else:
-                    val = requests.get(url + field,
-                                       timeout=timeout).content.decode('utf-8')
+                    val = _get_request(
+                        url + field,
+                        timeout=timeout,
+                        num_retries=num_retries).content.decode('utf-8')
                     if val[0] == '{':
                         val = json.loads(val)
                     d[field] = val
         else:
             logger.debug("Metadata service returned non 200 status code "
                          "of %s for url: %s, content body: %s",
                          r.status_code, url, r.content)
-    except (requests.Timeout, requests.ConnectionError) as e:
-        logger.debug("Caught exception wil trying to retrieve credentials "
-                     "from metadata service: %s", e, exc_info=True)
+    except _RetriesExceededError:
+        logger.debug("Max number of retries exceeded (%s) when "
+                     "attempting to retrieve data from metadata service.",
+                     num_retries)
     return d
 
 
+def _get_request(url, timeout, num_retries):
+    for i in range(num_retries + 1):
+        try:
+            response = requests.get(url, timeout=timeout)
+        except (requests.Timeout, requests.ConnectionError) as e:
+            logger.debug("Caught exception wil trying to retrieve credentials "
+                        "from metadata service: %s", e, exc_info=True)
+        else:
+            if response.status_code == 200:
+                return response
+    raise _RetriesExceededError()
+
+
 def search_iam_role(**kwargs):
     credentials = None
     metadata = retrieve_iam_role_credentials()

tests/unit/test_credentials.py

@@ -133,11 +133,11 @@ class IamRoleTest(BaseEnvVar):
     def setUp(self):
         super(IamRoleTest, self).setUp()
         self.session = botocore.session.get_session(env_vars=TESTENVVARS)
+        self.environ['BOTO_CONFIG'] = ''
 
     @mock.patch('botocore.credentials.retrieve_iam_role_credentials')
     def test_iam_role(self, retriever):
         retriever.return_value = metadata
-        self.environ['BOTO_CONFIG'] = ''
         credentials = self.session.get_credentials()
         self.assertEqual(credentials.method, 'iam-role')
         self.assertEqual(credentials.access_key, 'foo')
@@ -154,7 +154,6 @@ def test_empty_boto_config_is_ignored(self, retriever):
 
     @mock.patch('botocore.vendored.requests.get')
     def test_get_credentials_with_metadata_mock(self, get):
-        self.environ['BOTO_CONFIG'] = ''
         first = mock.Mock()
         first.status_code = 200
         first.content = 'foobar'.encode('utf-8')
@@ -169,7 +168,6 @@ def test_get_credentials_with_metadata_mock(self, get):
 
     @mock.patch('botocore.vendored.requests.get')
     def test_timeout_argument_forwarded_to_requests(self, get):
-        self.environ['BOTO_CONFIG'] = ''
         first = mock.Mock()
         first.status_code = 200
         first.content = 'foobar'.encode('utf-8')
@@ -184,13 +182,33 @@ def test_timeout_argument_forwarded_to_requests(self, get):
 
     @mock.patch('botocore.vendored.requests.get')
     def test_request_timeout_occurs(self, get):
-        self.environ['BOTO_CONFIG'] = ''
         first = mock.Mock()
         first.side_effect = ConnectionError
 
         d = credentials.retrieve_iam_role_credentials(timeout=10)
         self.assertEqual(d, {})
 
+    @mock.patch('botocore.vendored.requests.get')
+    def test_retry_errors(self, get):
+        # First attempt we get a connection error.
+        first = mock.Mock()
+        first.side_effect = ConnectionError
+
+        # Next attempt we get a response with the foobar key.
+        second = mock.Mock()
+        second.status_code = 200
+        second.content = 'foobar'.encode('utf-8')
+
+        # Next attempt we get a response with the foobar creds.
+        third = mock.Mock()
+        third.status_code = 200
+        third.content = json.dumps(metadata['foobar']).encode('utf-8')
+        get.side_effect = [first, second, third]
+
+        retrieved = credentials.retrieve_iam_role_credentials(
+            num_retries=1)
+        self.assertEqual(retrieved['foobar']['AccessKeyId'], 'foo')
+
 
 if __name__ == "__main__":
     unittest.main()