Fix: Force delay (take 2) between docker get calls

Docker has introduced a 429 error return code
if there site is called to quickly.

Retry after 60 seconds after every 429 return code,
up to 19 times. Then fail.

Issue-ID: RELENG-3711

Signed-off-by: Bengt Thuree <bthuree@linuxfoundation.org>
Change-Id: I51d23fa9a2da1dc7cc30afceae4f35942bb42182

diff --git a/lftools/nexus/release_docker_hub.py b/lftools/nexus/release_docker_hub.py
index 813c926..81fd783 100644 (file)
--- a/lftools/nexus/release_docker_hub.py
+++ b/lftools/nexus/release_docker_hub.py
@@ -45,8 +45,8 @@ import multiprocessing
 import os
 import re
 import socket
+import time
 from multiprocessing.dummy import Pool as ThreadPool
-from time import sleep
 
 import docker
 import requests
@@ -306,8 +306,16 @@ class DockerTagClass(TagClass):
         while retries < 20:
             try:
                 r = _request_get(self._docker_base + "/" + combined_repo_name + "/tags")
-                sleep(0.5)
-                break
+                if r.status_code == 429:
+                    # Docker returns 429 if we access it too fast too many times.
+                    # If it happends, delay 60 seconds, and try again, up to 19 times.
+                    log.debug(
+                        "Too many docker gets too fast, wait 1 min: {}, repo {}".format(retries, combined_repo_name)
+                    )
+                    time.sleep(60)
+                    retries = retries + 1
+                else:
+                    break
             except requests.HTTPError as excinfo:
                 log.debug("Fetching Docker Hub tags. {}".format(excinfo))
                 retries = retries + 1

diff --git a/tests/test_release_docker_hub.py b/tests/test_release_docker_hub.py
index 7daa81a..578229d 100644 (file)
--- a/tests/test_release_docker_hub.py
+++ b/tests/test_release_docker_hub.py
@@ -103,6 +103,40 @@ def test_docker_tag_class(responses):
     assert len(test_tags.invalid) == len(answer_invalid_tags)
 
 
+def test_docker_tag_err_429_class(responses, mocker):
+    """Test DockerTagClass"""
+    mocker.patch("time.sleep", return_value=None)
+    org = "onap"
+    repo = "base-sdc-sanity"
+    repo_from_file = False
+    url = "https://registry.hub.docker.com/v1/repositories/onap/base-sdc-sanity/tags"
+    answer = """[{"layer": "", "name": "latest"},
+        {"layer": "", "name": "1.3.0"},
+        {"layer": "", "name": "1.3.1"},
+        {"layer": "", "name": "1.4.0"},
+        {"layer": "", "name": "1.4.1"},
+        {"layer": "", "name": "v1.0.0"}]
+    """
+    answer_429 = '{"detail": "Rate limit exceeded", "error": false}"'
+    answer_valid_tags = ["1.3.0", "1.3.1", "1.4.0", "1.4.1"]
+    answer_invalid_tags = ["latest", "v1.0.0"]
+    rdh.initialize(org)
+
+    # Do 19 failed and next one should work
+    for k in range(19):
+        responses.add(responses.GET, url, body=answer_429, status=429)
+    responses.add(responses.GET, url, body=answer, status=200)
+    test_tags = rdh.DockerTagClass(org, repo, repo_from_file)
+    assert len(test_tags.valid) == len(answer_valid_tags)
+    assert len(test_tags.invalid) == len(answer_invalid_tags)
+
+    # Do 20 failed, and all failes
+    for k in range(20):
+        responses.add(responses.GET, url, body=answer_429, status=429)
+    with pytest.raises(requests.HTTPError):
+        test_tags = rdh.DockerTagClass(org, repo, repo_from_file)
+
+
 def test_tag_class_repository_exist():
     """Test TagClass"""
     org = "onap"