lftools deploy archives handle dup patterns/file

If the pattern is not properly done, the resulting file list might
contain duplicated files.

This fix will remove the duplicated patterns, as well as
duplicated matched files.

This fix should fix the OSError exception in mkdir

Change-Id: Ib7ac21ecdec2fdefe84b710a90666b70d966be12
Signed-off-by: Bengt Thuree <bthuree@linuxfoundation.org>
Signed-off-by: Anil Belur <abelur@linuxfoundation.org>

diff --git a/lftools/deploy.py b/lftools/deploy.py
index 0797338..0addcc7 100644 (file)
--- a/lftools/deploy.py
+++ b/lftools/deploy.py
@@ -144,6 +144,20 @@ def _get_node_from_xml(xml_data, tag_name):
     return childnode.firstChild.data
 
 
+def _remove_duplicates_and_sort(lst):
+    # Remove duplicates from list, and sort it
+    no_dups_lst = list(dict.fromkeys(lst))
+    no_dups_lst.sort()
+
+    duplicated_list = []
+    for i in range(len(no_dups_lst)):
+        if (lst.count(no_dups_lst[i]) > 1):
+            duplicated_list.append(no_dups_lst[i])
+    log.debug("duplicates  : {}".format(duplicated_list))
+
+    return no_dups_lst
+
+
 def copy_archives(workspace, pattern=None):
     """Copy files matching PATTERN in a WORKSPACE to the current directory.
 
@@ -188,8 +202,10 @@ def copy_archives(workspace, pattern=None):
     if pattern is None:
         return
 
+    no_dups_pattern = _remove_duplicates_and_sort(pattern)
+
     paths = []
-    for p in pattern:
+    for p in no_dups_pattern:
         if p == '':  # Skip empty patterns as they are invalid
             continue
 
@@ -197,7 +213,9 @@ def copy_archives(workspace, pattern=None):
         paths.extend(glob2.glob(search, recursive=True))
     log.debug('Files found: {}'.format(paths))
 
-    for src in paths:
+    no_dups_paths = _remove_duplicates_and_sort(paths)
+
+    for src in no_dups_paths:
         if len(os.path.basename(src)) > 255:
             log.warn('Filename {} is over 255 characters. Skipping...'.format(
                 os.path.basename(src)))
@@ -208,7 +226,7 @@ def copy_archives(workspace, pattern=None):
         try:
             shutil.move(src, dest)
         except IOError as e:  # Switch to FileNotFoundError when Python 2 support is dropped.
-            log.debug(e)
+            log.debug("Missing path, will create it {}".format(os.path.dirname(dest)))
             os.makedirs(os.path.dirname(dest))
             shutil.move(src, dest)
 

diff --git a/releasenotes/notes/copy_archive_with_pattern_causes_OSError-c194d1960d322c51.yaml b/releasenotes/notes/copy_archive_with_pattern_causes_OSError-c194d1960d322c51.yaml
new file mode 100644 (file)
index 0000000..0910f77
--- /dev/null
+++ b/releasenotes/notes/copy_archive_with_pattern_causes_OSError-c194d1960d322c51.yaml
@@ -0,0 +1,17 @@
+---
+fixes:
+  - |
+    Fix OSError in lftools deploy archives due to pattern
+
+    If the pattern is not properly done, the resulting file list might
+    contain duplicated files.
+
+    This fix will remove the duplicated patterns, as well as the
+    duplicated matched files.
+
+    This fix should fix the following crash
+    08:24:05   File "/home/jenkins/.local/lib/python2.7/site-packages/lftools/deploy.py", line 204, in copy_archives
+    08:24:05     os.makedirs(os.path.dirname(dest))
+    08:24:05   File "/usr/lib64/python2.7/os.py", line 157, in makedirs
+    08:24:05     mkdir(name, mode)
+    08:24:05 OSError: [Errno 17] File exists: '/tmp/lftools-da.m80YHz/features/benchmark/odl-benchmark-api/target/surefire-reports'

diff --git a/tests/fixtures/deploy/workspace-patternfile/abc.log b/tests/fixtures/deploy/workspace-patternfile/abc.log
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/abc.log
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace-patternfile/archives/test.log b/tests/fixtures/deploy/workspace-patternfile/archives/test.log
new file mode 100644 (file)
index 0000000..b3589bf
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/archives/test.log
@@ -0,0 +1 @@
+This is a test log.

diff --git a/tests/fixtures/deploy/workspace-patternfile/dir1/abc.log b/tests/fixtures/deploy/workspace-patternfile/dir1/abc.log
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/dir1/abc.log
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace-patternfile/dir1/abc.txt b/tests/fixtures/deploy/workspace-patternfile/dir1/abc.txt
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/dir1/abc.txt
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_12.log b/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_12.log
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_12.log
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_13.log b/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_13.log
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/dir1/hs_err_13.log
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace-patternfile/dir2/abc.log b/tests/fixtures/deploy/workspace-patternfile/dir2/abc.log
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/dir2/abc.log
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace-patternfile/dir2/abc2.txt b/tests/fixtures/deploy/workspace-patternfile/dir2/abc2.txt
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/dir2/abc2.txt
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_12.log b/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_12.log
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_12.log
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_13.log b/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_13.log
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace-patternfile/dir2/hs_err_13.log
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/test_deploy.py b/tests/test_deploy.py
index 453f6e4..d53b6bc 100644 (file)
--- a/tests/test_deploy.py
+++ b/tests/test_deploy.py
@@ -153,6 +153,82 @@ def test_deploy_archive3(datafiles):
     assert workspace_dir in str(excinfo.value)
 
 
+@pytest.mark.datafiles(
+    os.path.join(FIXTURE_DIR, 'deploy'),
+    )
+def test_deploy_archive4(cli_runner, datafiles, responses):
+    """Test deploy_archives() command when using duplicated patterns."""
+    os.chdir(str(datafiles))
+    workspace_dir = os.path.join(str(datafiles), 'workspace-patternfile')
+    pattern=["**/*.log", "**/hs_err_*.log", "**/target/**/feature.xml", "**/target/failsafe-reports/failsafe-summary.xml", "**/target/surefire-reports/*-output.txt", "**/target/surefire-reports/*-output.txt", "**/target/failsafe-reports/failsafe-summary.xml"]
+    result = deploy_sys.copy_archives(workspace_dir, pattern)
+    assert result is None
+
+
+def test_remove_duplicates_and_sort():
+    test_lst = [[["file1"],
+                 ["file1"]],
+
+                [["file1", "file2"],
+                 ["file1", "file2"]],
+
+                [["file2", "file3", "file5", "file1", "file4"],
+                 ["file1", "file2", "file3", "file4", "file5"]],
+
+                [["file2", "file3", "file2", "file3", "file4"],
+                 ["file2", "file3", "file4"]],
+
+
+                [["**/*.log",
+                "**/hs_err_*.log",
+                "**/target/**/feature.xml",
+                "**/target/failsafe-reports/failsafe-summary.xml",
+                "**/target/surefire-reports/*-output.txt",
+                "**/target/surefire-reports/*-output.txt",
+                "**/target/failsafe-reports/failsafe-summary.xml"],
+
+                ["**/*.log",
+                "**/hs_err_*.log",
+                "**/target/**/feature.xml",
+                "**/target/failsafe-reports/failsafe-summary.xml",
+                "**/target/surefire-reports/*-output.txt"]],
+
+                [['/workspace-patternfile/abc.log',
+                  '/workspace-patternfile/dir1/hs_err_13.log',
+                  '/workspace-patternfile/dir1/hs_err_12.log',
+                  '/workspace-patternfile/dir1/abc.log',
+                  '/workspace-patternfile/dir2/hs_err_13.log',
+                  '/workspace-patternfile/dir2/hs_err_12.log',
+                  '/workspace-patternfile/dir2/abc.log',
+                  '/workspace-patternfile/dir1/hs_err_13.log',
+                  '/workspace-patternfile/dir1/hs_err_12.log',
+                  '/workspace-patternfile/dir2/hs_err_13.log',
+                  '/workspace-patternfile/dir2/hs_err_12.log',
+                  '/workspace-patternfile/target/dir1/feature.xml',
+                  '/workspace-patternfile/target/dir2/feature.xml',
+                  '/workspace-patternfile/target/surefire-reports/abc1-output.txt',
+                  '/workspace-patternfile/target/surefire-reports/abc2-output.txt',
+                  '/workspace-patternfile/target/surefire-reports/abc1-output.txt',
+                  '/workspace-patternfile/target/surefire-reports/abc2-output.txt'],
+
+                 ['/workspace-patternfile/abc.log',
+                  '/workspace-patternfile/dir1/abc.log',
+                  '/workspace-patternfile/dir1/hs_err_12.log',
+                  '/workspace-patternfile/dir1/hs_err_13.log',
+                  '/workspace-patternfile/dir2/abc.log',
+                  '/workspace-patternfile/dir2/hs_err_12.log',
+                  '/workspace-patternfile/dir2/hs_err_13.log',
+                  '/workspace-patternfile/target/dir1/feature.xml',
+                  '/workspace-patternfile/target/dir2/feature.xml',
+                  '/workspace-patternfile/target/surefire-reports/abc1-output.txt',
+                  '/workspace-patternfile/target/surefire-reports/abc2-output.txt']]
+
+              ]
+
+    for tst in test_lst:
+        assert deploy_sys._remove_duplicates_and_sort(tst[0]) == tst[1]
+
+
 @pytest.mark.datafiles(
     os.path.join(FIXTURE_DIR, 'deploy'),
     )