From: Bengt Thuree Date: Thu, 13 Jun 2019 00:51:35 +0000 (+1000) Subject: lftools deploy archives handle dup patterns/file X-Git-Tag: v0.25.0~1 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F49%2F15849%2F10;p=releng%2Flftools.git lftools deploy archives handle dup patterns/file If the pattern is not properly done, the resulting file list might contain duplicated files. This fix will remove the duplicated patterns, as well as duplicated matched files. This fix should fix the OSError exception in mkdir Change-Id: Ib7ac21ecdec2fdefe84b710a90666b70d966be12 Signed-off-by: Bengt Thuree Signed-off-by: Anil Belur --- diff --git a/lftools/deploy.py b/lftools/deploy.py index 0797338e..0addcc7e 100644 --- a/lftools/deploy.py +++ b/lftools/deploy.py @@ -144,6 +144,20 @@ def _get_node_from_xml(xml_data, tag_name): return childnode.firstChild.data +def _remove_duplicates_and_sort(lst): + # Remove duplicates from list, and sort it + no_dups_lst = list(dict.fromkeys(lst)) + no_dups_lst.sort() + + duplicated_list = [] + for i in range(len(no_dups_lst)): + if (lst.count(no_dups_lst[i]) > 1): + duplicated_list.append(no_dups_lst[i]) + log.debug("duplicates : {}".format(duplicated_list)) + + return no_dups_lst + + def copy_archives(workspace, pattern=None): """Copy files matching PATTERN in a WORKSPACE to the current directory. @@ -188,8 +202,10 @@ def copy_archives(workspace, pattern=None): if pattern is None: return + no_dups_pattern = _remove_duplicates_and_sort(pattern) + paths = [] - for p in pattern: + for p in no_dups_pattern: if p == '': # Skip empty patterns as they are invalid continue @@ -197,7 +213,9 @@ def copy_archives(workspace, pattern=None): paths.extend(glob2.glob(search, recursive=True)) log.debug('Files found: {}'.format(paths)) - for src in paths: + no_dups_paths = _remove_duplicates_and_sort(paths) + + for src in no_dups_paths: if len(os.path.basename(src)) > 255: log.warn('Filename {} is over 255 characters. Skipping...'.format( os.path.basename(src))) @@ -208,7 +226,7 @@ def copy_archives(workspace, pattern=None): try: shutil.move(src, dest) except IOError as e: # Switch to FileNotFoundError when Python 2 support is dropped. - log.debug(e) + log.debug("Missing path, will create it {}".format(os.path.dirname(dest))) os.makedirs(os.path.dirname(dest)) shutil.move(src, dest) diff --git a/releasenotes/notes/copy_archive_with_pattern_causes_OSError-c194d1960d322c51.yaml b/releasenotes/notes/copy_archive_with_pattern_causes_OSError-c194d1960d322c51.yaml new file mode 100644 index 00000000..0910f77d --- /dev/null +++ b/releasenotes/notes/copy_archive_with_pattern_causes_OSError-c194d1960d322c51.yaml @@ -0,0 +1,17 @@ +--- +fixes: + - | + Fix OSError in lftools deploy archives due to pattern + + If the pattern is not properly done, the resulting file list might + contain duplicated files. + + This fix will remove the duplicated patterns, as well as the + duplicated matched files. + + This fix should fix the following crash + 08:24:05 File "/home/jenkins/.local/lib/python2.7/site-packages/lftools/deploy.py", line 204, in copy_archives + 08:24:05 os.makedirs(os.path.dirname(dest)) + 08:24:05 File "/usr/lib64/python2.7/os.py", line 157, in makedirs + 08:24:05 mkdir(name, mode) + 08:24:05 OSError: [Errno 17] File exists: '/tmp/lftools-da.m80YHz/features/benchmark/odl-benchmark-api/target/surefire-reports' diff --git a/tests/fixtures/deploy/workspace-patternfile/abc.log b/tests/fixtures/deploy/workspace-patternfile/abc.log new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/abc.log @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace-patternfile/archives/test.log b/tests/fixtures/deploy/workspace-patternfile/archives/test.log new file mode 100644 index 00000000..b3589bf7 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/archives/test.log @@ -0,0 +1 @@ +This is a test log. diff --git a/tests/fixtures/deploy/workspace-patternfile/dir1/abc.log b/tests/fixtures/deploy/workspace-patternfile/dir1/abc.log new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/dir1/abc.log @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace-patternfile/dir1/abc.txt b/tests/fixtures/deploy/workspace-patternfile/dir1/abc.txt new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/dir1/abc.txt @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_12.log b/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_12.log new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_12.log @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_13.log b/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_13.log new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_13.log @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace-patternfile/dir2/abc.log b/tests/fixtures/deploy/workspace-patternfile/dir2/abc.log new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/dir2/abc.log @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace-patternfile/dir2/abc2.txt b/tests/fixtures/deploy/workspace-patternfile/dir2/abc2.txt new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/dir2/abc2.txt @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_12.log b/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_12.log new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_12.log @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_13.log b/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_13.log new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_13.log @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/test_deploy.py b/tests/test_deploy.py index 453f6e4e..d53b6bc6 100644 --- a/tests/test_deploy.py +++ b/tests/test_deploy.py @@ -153,6 +153,82 @@ def test_deploy_archive3(datafiles): assert workspace_dir in str(excinfo.value) +@pytest.mark.datafiles( + os.path.join(FIXTURE_DIR, 'deploy'), + ) +def test_deploy_archive4(cli_runner, datafiles, responses): + """Test deploy_archives() command when using duplicated patterns.""" + os.chdir(str(datafiles)) + workspace_dir = os.path.join(str(datafiles), 'workspace-patternfile') + pattern=["**/*.log", "**/hs_err_*.log", "**/target/**/feature.xml", "**/target/failsafe-reports/failsafe-summary.xml", "**/target/surefire-reports/*-output.txt", "**/target/surefire-reports/*-output.txt", "**/target/failsafe-reports/failsafe-summary.xml"] + result = deploy_sys.copy_archives(workspace_dir, pattern) + assert result is None + + +def test_remove_duplicates_and_sort(): + test_lst = [[["file1"], + ["file1"]], + + [["file1", "file2"], + ["file1", "file2"]], + + [["file2", "file3", "file5", "file1", "file4"], + ["file1", "file2", "file3", "file4", "file5"]], + + [["file2", "file3", "file2", "file3", "file4"], + ["file2", "file3", "file4"]], + + + [["**/*.log", + "**/hs_err_*.log", + "**/target/**/feature.xml", + "**/target/failsafe-reports/failsafe-summary.xml", + "**/target/surefire-reports/*-output.txt", + "**/target/surefire-reports/*-output.txt", + "**/target/failsafe-reports/failsafe-summary.xml"], + + ["**/*.log", + "**/hs_err_*.log", + "**/target/**/feature.xml", + "**/target/failsafe-reports/failsafe-summary.xml", + "**/target/surefire-reports/*-output.txt"]], + + [['/workspace-patternfile/abc.log', + '/workspace-patternfile/dir1/hs_err_13.log', + '/workspace-patternfile/dir1/hs_err_12.log', + '/workspace-patternfile/dir1/abc.log', + '/workspace-patternfile/dir2/hs_err_13.log', + '/workspace-patternfile/dir2/hs_err_12.log', + '/workspace-patternfile/dir2/abc.log', + '/workspace-patternfile/dir1/hs_err_13.log', + '/workspace-patternfile/dir1/hs_err_12.log', + '/workspace-patternfile/dir2/hs_err_13.log', + '/workspace-patternfile/dir2/hs_err_12.log', + '/workspace-patternfile/target/dir1/feature.xml', + '/workspace-patternfile/target/dir2/feature.xml', + '/workspace-patternfile/target/surefire-reports/abc1-output.txt', + '/workspace-patternfile/target/surefire-reports/abc2-output.txt', + '/workspace-patternfile/target/surefire-reports/abc1-output.txt', + '/workspace-patternfile/target/surefire-reports/abc2-output.txt'], + + ['/workspace-patternfile/abc.log', + '/workspace-patternfile/dir1/abc.log', + '/workspace-patternfile/dir1/hs_err_12.log', + '/workspace-patternfile/dir1/hs_err_13.log', + '/workspace-patternfile/dir2/abc.log', + '/workspace-patternfile/dir2/hs_err_12.log', + '/workspace-patternfile/dir2/hs_err_13.log', + '/workspace-patternfile/target/dir1/feature.xml', + '/workspace-patternfile/target/dir2/feature.xml', + '/workspace-patternfile/target/surefire-reports/abc1-output.txt', + '/workspace-patternfile/target/surefire-reports/abc2-output.txt']] + + ] + + for tst in test_lst: + assert deploy_sys._remove_duplicates_and_sort(tst[0]) == tst[1] + + @pytest.mark.datafiles( os.path.join(FIXTURE_DIR, 'deploy'), )