From a889de0e5c9891e58bb99cc1d2e6dbff4e125885 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Wed, 31 Oct 2018 10:38:27 -0400 Subject: [PATCH] Refactor copy_archives deploy function to Python Rewrites the copy_archives() bash script to Python for better portability on Windows systems. Issue: RELENG-1375 Change-Id: I264c29f9a3b86415fd879ef9dce38714105309f0 Signed-off-by: Thanh Ha --- docs/commands/deploy.rst | 5 ++ lftools/cli/deploy.py | 19 +++++++ lftools/deploy.py | 64 ++++++++++++++++++++++ .../refactor-copy-archives-b5e7ee75fc7bf271.yaml | 10 ++++ requirements.txt | 1 + ...a.cert.test.AaaCertMdsalProviderTest-output.txt | 6 ++ ...ht.aaa.cert.test.AaaCertProviderTest-output.txt | 2 + ....cert.test.AaaCertRpcServiceImplTest-output.txt | 7 +++ ...aaa.cert.test.KeyStoresDataUtilsTest-output.txt | 1 + tests/fixtures/deploy/workspace/abc.txt | 1 + tests/fixtures/deploy/workspace/archives/test.log | 1 + tests/fixtures/deploy/workspace/dependencies.log | 17 ++++++ tests/test_deploy.py | 63 +++++++++++++++++++++ 13 files changed, 197 insertions(+) create mode 100644 lftools/deploy.py create mode 100644 releasenotes/notes/refactor-copy-archives-b5e7ee75fc7bf271.yaml create mode 100644 tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt create mode 100644 tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt create mode 100644 tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt create mode 100644 tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt create mode 100644 tests/fixtures/deploy/workspace/abc.txt create mode 100644 tests/fixtures/deploy/workspace/archives/test.log create mode 100644 tests/fixtures/deploy/workspace/dependencies.log create mode 100644 tests/test_deploy.py diff --git a/docs/commands/deploy.rst b/docs/commands/deploy.rst index 4a6d8f5c..e741118e 100644 --- a/docs/commands/deploy.rst +++ b/docs/commands/deploy.rst @@ -15,6 +15,11 @@ archives .. program-output:: lftools deploy archives --help +copy-archives +------------- + +.. program-output:: lftools deploy copy-archives --help + logs ---- diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py index b2dc1fc1..31b9f9b0 100644 --- a/lftools/cli/deploy.py +++ b/lftools/cli/deploy.py @@ -17,6 +17,8 @@ import sys import click +import lftools.deploy as deploy_sys + @click.group() @click.pass_context @@ -48,6 +50,22 @@ def archives(ctx, nexus_url, nexus_path, workspace, pattern): sys.exit(status) +@click.command(name='copy-archives') +@click.argument('workspace', envvar='WORKSPACE') +@click.argument('pattern', nargs=-1, default=None, required=False) +@click.pass_context +def copy_archives(ctx, workspace, pattern): + """Copy files for archiving. + + Arguments: + + workspace: Typically a Jenkins WORKSPACE to copy files from. + pattern: Space-separated list of Unix style glob patterns of files to + copy for archiving. (default: false) + """ + deploy_sys.copy_archives(workspace, pattern) + + @click.command() @click.argument('nexus-url', envvar='NEXUS_URL') @click.argument('nexus-repo-id') @@ -277,6 +295,7 @@ def nexus_zip(ctx, nexus_url, nexus_repo, nexus_path, deploy_zip): deploy.add_command(archives) +deploy.add_command(copy_archives) deploy.add_command(file) deploy.add_command(logs) deploy.add_command(maven_file) diff --git a/lftools/deploy.py b/lftools/deploy.py new file mode 100644 index 00000000..453cb952 --- /dev/null +++ b/lftools/deploy.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Library of functions for deploying artifacts to Nexus.""" + +import logging +import os +import shutil + +import glob2 # Switch to glob when Python < 3.5 support is dropped + +log = logging.getLogger(__name__) + + +def copy_archives(workspace, pattern=None): + """Copy files matching PATTERN in a WORKSPACE to the current directory. + + The best way to use this function is to cd into the directory you wish to + store the files first before calling the function. + + :params: + + :arg str pattern: Space-separated list of Unix style glob patterns. + (default: None) + """ + archives_dir = os.path.join(workspace, 'archives') + dest_dir = os.getcwd() + + log.debug('Copying files from {} with pattern \'{}\' to {}.'.format( + workspace, pattern, dest_dir)) + for file_or_dir in os.listdir(archives_dir): + f = os.path.join(archives_dir, file_or_dir) + try: + log.debug('Moving {}'.format(f)) + shutil.move(f, dest_dir) + except shutil.Error as e: + log.warn(e) + + paths = [] + for p in pattern: + search = os.path.join(workspace, p) + paths.extend(glob2.glob(search, recursive=True)) + log.debug('Files found: {}'.format(paths)) + + for src in paths: + if len(os.path.basename(src)) > 255: + log.warn('Filename {} is over 255 characters. Skipping...'.format( + os.path.basename(src))) + + dest = os.path.join(dest_dir, src[len(workspace)+1:]) + log.debug('{} -> {}'.format(src, dest)) + + try: + shutil.move(src, dest) + except IOError as e: # Switch to FileNotFoundError when Python 2 support is dropped. + log.debug(e) + os.makedirs(os.path.dirname(dest)) + shutil.move(src, dest) diff --git a/releasenotes/notes/refactor-copy-archives-b5e7ee75fc7bf271.yaml b/releasenotes/notes/refactor-copy-archives-b5e7ee75fc7bf271.yaml new file mode 100644 index 00000000..f5f15d84 --- /dev/null +++ b/releasenotes/notes/refactor-copy-archives-b5e7ee75fc7bf271.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + The shell/deploy file's copy_archives() function has been reimplemented in + pure Python for better portability to Windows systems. +deprecations: + - | + The shell/deploy script's copy_archives() function is now deprecated and + will be removed in a later version. We recommend migrating to the lftools + pure Python implementation of this function. diff --git a/requirements.txt b/requirements.txt index 1f76aecd..e447727d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ click +glob2 # Needed for Python < 3.5 recursive glob support pyyaml requests~=2.18.0 ruamel.yaml diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt new file mode 100644 index 00000000..e529ce52 --- /dev/null +++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt @@ -0,0 +1,6 @@ +[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized +[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt new file mode 100644 index 00000000..80ba15ad --- /dev/null +++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertProviderTest-output.txt @@ -0,0 +1,2 @@ +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertProvider - aaa Certificate Service Initalized +[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - ctlTest.jks is created diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt new file mode 100644 index 00000000..97bfffab --- /dev/null +++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.AaaCertRpcServiceImplTest-output.txt @@ -0,0 +1,7 @@ +[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertMdsalProvider - AaaCertMdsalProvider Initialized +[main] INFO org.opendaylight.aaa.cert.impl.AaaCertRpcServiceImpl - AaaCert Rpc Service has been initialized diff --git a/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt new file mode 100644 index 00000000..13003b64 --- /dev/null +++ b/tests/fixtures/deploy/workspace/aaa/aaa-cert/target/surefire-reports/org.opendaylight.aaa.cert.test.KeyStoresDataUtilsTest-output.txt @@ -0,0 +1 @@ +[main] INFO org.opendaylight.aaa.cert.impl.ODLKeyTool - odlTest.jks is created diff --git a/tests/fixtures/deploy/workspace/abc.txt b/tests/fixtures/deploy/workspace/abc.txt new file mode 100644 index 00000000..b0883f38 --- /dev/null +++ b/tests/fixtures/deploy/workspace/abc.txt @@ -0,0 +1 @@ +abcdefghijklmnopqrstuvwxyz diff --git a/tests/fixtures/deploy/workspace/archives/test.log b/tests/fixtures/deploy/workspace/archives/test.log new file mode 100644 index 00000000..b3589bf7 --- /dev/null +++ b/tests/fixtures/deploy/workspace/archives/test.log @@ -0,0 +1 @@ +This is a test log. diff --git a/tests/fixtures/deploy/workspace/dependencies.log b/tests/fixtures/deploy/workspace/dependencies.log new file mode 100644 index 00000000..80ccd5ed --- /dev/null +++ b/tests/fixtures/deploy/workspace/dependencies.log @@ -0,0 +1,17 @@ +aaa:controller,infrautils,mdsal,odlparent,yangtools +bgpcep:controller,infrautils,mdsal,netconf,odlparent,yangtools +controller:mdsal,odlparent,yangtools +coe:controller,mdsal,netconf,odlparent +daexim:controller,infrautils,mdsal,netconf,odlparent,yangtools +genius:controller,daexim,infrautils,mdsal,netconf,odlparent,openflowplugin,ovsdb,serviceutils,yangtools +infrautils:odlparent +integration/distribution:aaa,integration,odlparent +lispflowmapping:controller,mdsal,netconf,neutron,odlparent +mdsal:odlparent,yangtools +netconf:aaa,controller,mdsal,odlparent,yangtools +netvirt:aaa,coe,controller,genius,infrautils,mdsal,netconf,neutron,odlparent,openflowplugin,ovsdb,serviceutils,sfc,yangtools +neutron:aaa,controller,infrautils,mdsal,netconf,odlparent,ovsdb,yangtools +openflowplugin:controller,infrautils,mdsal,netconf,odlparent,serviceutils,yangtools +ovsdb:aaa,controller,infrautils,mdsal,netconf,odlparent,yangtools +serviceutils:controller,infrautils,mdsal,odlparent,yangtools +sfc:aaa,controller,genius,infrautils,lispflowmapping,mdsal,netconf,odlparent,openflowplugin,ovsdb,serviceutils diff --git a/tests/test_deploy.py b/tests/test_deploy.py new file mode 100644 index 00000000..9e34635f --- /dev/null +++ b/tests/test_deploy.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Test deploy command.""" + +import os + +import pytest + +from lftools import cli + +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + 'fixtures', + ) + + +@pytest.mark.datafiles( + os.path.join(FIXTURE_DIR, 'deploy'), + ) +def test_copy_archive_dir(cli_runner, datafiles): + """Test copy_archives() command to ensure archives dir is copied.""" + os.chdir(str(datafiles)) + workspace_dir = os.path.join(str(datafiles), 'workspace') + stage_dir = str(datafiles.mkdir("stage_archive")) + + os.chdir(stage_dir) + result = cli_runner.invoke( + cli.cli, + ['--debug', 'deploy', 'copy-archives', workspace_dir], + obj={}) + assert result.exit_code == 0 + + assert os.path.exists(os.path.join(stage_dir, 'test.log')) + +@pytest.mark.datafiles( + os.path.join(FIXTURE_DIR, 'deploy'), + ) +def test_copy_archive_pattern(cli_runner, datafiles): + """Test copy_archives() command to ensure glob patterns are copied.""" + os.chdir(str(datafiles)) + workspace_dir = os.path.join(str(datafiles), 'workspace') + stage_dir = str(datafiles.mkdir("stage_archive")) + + os.chdir(stage_dir) + result = cli_runner.invoke( + cli.cli, + ['--debug', 'deploy', 'copy-archives', workspace_dir, '**/*.txt'], + obj={}) + assert result.exit_code == 0 + + assert os.path.exists(os.path.join(stage_dir, 'test.log')) + assert os.path.exists(os.path.join(stage_dir, 'abc.txt')) + assert not os.path.exists(os.path.join(stage_dir, 'dependencies.log')) + assert os.path.exists(os.path.join( + stage_dir, 'aaa', 'aaa-cert', 'target', 'surefire-reports', + 'org.opendaylight.aaa.cert.test.AaaCertMdsalProviderTest-output.txt')) -- 2.16.6