Refactor copy_archives deploy function to Python

Rewrites the copy_archives() bash script to Python for
better portability on Windows systems.

Issue: RELENG-1375
Change-Id: I264c29f9a3b86415fd879ef9dce38714105309f0
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/docs/commands/deploy.rst b/docs/commands/deploy.rst
index 4a6d8f5..e741118 100644 (file)
--- a/docs/commands/deploy.rst
+++ b/docs/commands/deploy.rst
@@ -15,6 +15,11 @@ archives
 
 .. program-output:: lftools deploy archives --help
 
+copy-archives
+-------------
+
+.. program-output:: lftools deploy copy-archives --help
+
 logs
 ----
 

diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py
index b2dc1fc..31b9f9b 100644 (file)
--- a/lftools/cli/deploy.py
+++ b/lftools/cli/deploy.py
@@ -17,6 +17,8 @@ import sys
 
 import click
 
+import lftools.deploy as deploy_sys
+
 
 @click.group()
 @click.pass_context
@@ -48,6 +50,22 @@ def archives(ctx, nexus_url, nexus_path, workspace, pattern):
     sys.exit(status)
 
 
+@click.command(name='copy-archives')
+@click.argument('workspace', envvar='WORKSPACE')
+@click.argument('pattern', nargs=-1, default=None, required=False)
+@click.pass_context
+def copy_archives(ctx, workspace, pattern):
+    """Copy files for archiving.
+
+    Arguments:
+
+        workspace: Typically a Jenkins WORKSPACE to copy files from.
+        pattern: Space-separated list of Unix style glob patterns of files to
+            copy for archiving. (default: false)
+    """
+    deploy_sys.copy_archives(workspace, pattern)
+
+
 @click.command()
 @click.argument('nexus-url', envvar='NEXUS_URL')
 @click.argument('nexus-repo-id')
@@ -277,6 +295,7 @@ def nexus_zip(ctx, nexus_url, nexus_repo, nexus_path, deploy_zip):
 
 
 deploy.add_command(archives)
+deploy.add_command(copy_archives)
 deploy.add_command(file)
 deploy.add_command(logs)
 deploy.add_command(maven_file)

diff --git a/lftools/deploy.py b/lftools/deploy.py
new file mode 100644 (file)
index 0000000..453cb95
--- /dev/null
+++ b/lftools/deploy.py
@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Library of functions for deploying artifacts to Nexus."""
+
+import logging
+import os
+import shutil
+
+import glob2  # Switch to glob when Python < 3.5 support is dropped
+
+log = logging.getLogger(__name__)
+
+
+def copy_archives(workspace, pattern=None):
+    """Copy files matching PATTERN in a WORKSPACE to the current directory.
+
+    The best way to use this function is to cd into the directory you wish to
+    store the files first before calling the function.
+
+    :params:
+
+        :arg str pattern: Space-separated list of Unix style glob patterns.
+            (default: None)
+    """
+    archives_dir = os.path.join(workspace, 'archives')
+    dest_dir = os.getcwd()
+
+    log.debug('Copying files from {} with pattern \'{}\' to {}.'.format(
+        workspace, pattern, dest_dir))
+    for file_or_dir in os.listdir(archives_dir):
+        f = os.path.join(archives_dir, file_or_dir)
+        try:
+            log.debug('Moving {}'.format(f))
+            shutil.move(f, dest_dir)
+        except shutil.Error as e:
+            log.warn(e)
+
+    paths = []
+    for p in pattern:
+        search = os.path.join(workspace, p)
+        paths.extend(glob2.glob(search, recursive=True))
+    log.debug('Files found: {}'.format(paths))
+
+    for src in paths:
+        if len(os.path.basename(src)) > 255:
+            log.warn('Filename {} is over 255 characters. Skipping...'.format(
+                os.path.basename(src)))
+
+        dest = os.path.join(dest_dir, src[len(workspace)+1:])
+        log.debug('{} -> {}'.format(src, dest))
+
+        try:
+            shutil.move(src, dest)
+        except IOError as e:  # Switch to FileNotFoundError when Python 2 support is dropped.
+            log.debug(e)
+            os.makedirs(os.path.dirname(dest))
+            shutil.move(src, dest)

diff --git a/releasenotes/notes/refactor-copy-archives-b5e7ee75fc7bf271.yaml b/releasenotes/notes/refactor-copy-archives-b5e7ee75fc7bf271.yaml
new file mode 100644 (file)
index 0000000..f5f15d8
--- /dev/null
+++ b/releasenotes/notes/refactor-copy-archives-b5e7ee75fc7bf271.yaml
@@ -0,0 +1,10 @@
+---
+features:
+  - |
+    The shell/deploy file's copy_archives() function has been reimplemented in
+    pure Python for better portability to Windows systems.
+deprecations:
+  - |
+    The shell/deploy script's copy_archives() function is now deprecated and
+    will be removed in a later version. We recommend migrating to the lftools
+    pure Python implementation of this function.

diff --git a/requirements.txt b/requirements.txt
index 1f76aec..e447727 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
 click
+glob2  # Needed for Python < 3.5 recursive glob support
 pyyaml
 requests~=2.18.0
 ruamel.yaml

diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt
new file mode 100644 (file)
index 0000000..e529ce5
--- /dev/null
+++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt
@@ -0,0 +1,6 @@
+[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized
+[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized

diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt
new file mode 100644 (file)
index 0000000..80ba15a
--- /dev/null
+++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt
@@ -0,0 +1,2 @@
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertProvider - aaa Certificate Service Initalized
+[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - ctlTest.jks is created

diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt
new file mode 100644 (file)
index 0000000..97bfffa
--- /dev/null
+++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt
@@ -0,0 +1,7 @@
+[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized
+[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized

diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt
new file mode 100644 (file)
index 0000000..13003b6
--- /dev/null
+++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt
@@ -0,0 +1 @@
+[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created

diff --git a/tests/fixtures/deploy/workspace/abc.txt b/tests/fixtures/deploy/workspace/abc.txt
new file mode 100644 (file)
index 0000000..b0883f3
--- /dev/null
+++ b/tests/fixtures/deploy/workspace/abc.txt
@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

diff --git a/tests/fixtures/deploy/workspace/archives/test.log b/tests/fixtures/deploy/workspace/archives/test.log
new file mode 100644 (file)
index 0000000..b3589bf
--- /dev/null
+++ b/tests/fixtures/deploy/workspace/archives/test.log
@@ -0,0 +1 @@
+This is a test log.

diff --git a/tests/fixtures/deploy/workspace/dependencies.log b/tests/fixtures/deploy/workspace/dependencies.log
new file mode 100644 (file)
index 0000000..80ccd5e
--- /dev/null
+++ b/tests/fixtures/deploy/workspace/dependencies.log
@@ -0,0 +1,17 @@
+aaa:controller,infrautils,mdsal,odlparent,yangtools
+bgpcep:controller,infrautils,mdsal,netconf,odlparent,yangtools
+controller:mdsal,odlparent,yangtools
+coe:controller,mdsal,netconf,odlparent
+daexim:controller,infrautils,mdsal,netconf,odlparent,yangtools
+genius:controller,daexim,infrautils,mdsal,netconf,odlparent,openflowplugin,ovsdb,serviceutils,yangtools
+infrautils:odlparent
+integration/distribution:aaa,integration,odlparent
+lispflowmapping:controller,mdsal,netconf,neutron,odlparent
+mdsal:odlparent,yangtools
+netconf:aaa,controller,mdsal,odlparent,yangtools
+netvirt:aaa,coe,controller,genius,infrautils,mdsal,netconf,neutron,odlparent,openflowplugin,ovsdb,serviceutils,sfc,yangtools
+neutron:aaa,controller,infrautils,mdsal,netconf,odlparent,ovsdb,yangtools
+openflowplugin:controller,infrautils,mdsal,netconf,odlparent,serviceutils,yangtools
+ovsdb:aaa,controller,infrautils,mdsal,netconf,odlparent,yangtools
+serviceutils:controller,infrautils,mdsal,odlparent,yangtools
+sfc:aaa,controller,genius,infrautils,lispflowmapping,mdsal,netconf,odlparent,openflowplugin,ovsdb,serviceutils

diff --git a/tests/test_deploy.py b/tests/test_deploy.py
new file mode 100644 (file)
index 0000000..9e34635
--- /dev/null
+++ b/tests/test_deploy.py
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+"""Test deploy command."""
+
+import os
+
+import pytest
+
+from lftools import cli
+
+FIXTURE_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'fixtures',
+    )
+
+
+@pytest.mark.datafiles(
+    os.path.join(FIXTURE_DIR, 'deploy'),
+    )
+def test_copy_archive_dir(cli_runner, datafiles):
+    """Test copy_archives() command to ensure archives dir is copied."""
+    os.chdir(str(datafiles))
+    workspace_dir = os.path.join(str(datafiles), 'workspace')
+    stage_dir = str(datafiles.mkdir("stage_archive"))
+
+    os.chdir(stage_dir)
+    result = cli_runner.invoke(
+        cli.cli,
+        ['--debug', 'deploy', 'copy-archives', workspace_dir],
+        obj={})
+    assert result.exit_code == 0
+
+    assert os.path.exists(os.path.join(stage_dir, 'test.log'))
+
+@pytest.mark.datafiles(
+    os.path.join(FIXTURE_DIR, 'deploy'),
+    )
+def test_copy_archive_pattern(cli_runner, datafiles):
+    """Test copy_archives() command to ensure glob patterns are copied."""
+    os.chdir(str(datafiles))
+    workspace_dir = os.path.join(str(datafiles), 'workspace')
+    stage_dir = str(datafiles.mkdir("stage_archive"))
+
+    os.chdir(stage_dir)
+    result = cli_runner.invoke(
+        cli.cli,
+        ['--debug', 'deploy', 'copy-archives', workspace_dir, '**/*.txt'],
+        obj={})
+    assert result.exit_code == 0
+
+    assert os.path.exists(os.path.join(stage_dir, 'test.log'))
+    assert os.path.exists(os.path.join(stage_dir, 'abc.txt'))
+    assert not os.path.exists(os.path.join(stage_dir, 'dependencies.log'))
+    assert os.path.exists(os.path.join(
+        stage_dir, 'aaa', 'aaa-cert', 'target', 'surefire-reports',
+        'org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt'))