Refactor deploy-logs cmd to Python

Issue: RELENG-1377
Change-Id: Ibf58a92fe8800ad77d8be32a123da603c6a36c56
Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>

diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py
index 43129dc..520ac3f 100644 (file)
--- a/lftools/cli/deploy.py
+++ b/lftools/cli/deploy.py
@@ -135,11 +135,15 @@ def logs(ctx, nexus_url, nexus_path, build_url):
     for log archiving.
 
     To use this script the Nexus server must have a site repository configured
-    with the name "logs" as this is a hardcoded path. Also this script uses
-    ~/.netrc for it's authentication which must be provided.
+    with the name "logs" as this is a hardcoded path.
     """
-    status = subprocess.call(['deploy', 'logs', nexus_url, nexus_path, build_url])
-    sys.exit(status)
+    try:
+        deploy_sys.deploy_logs(nexus_url, nexus_path, build_url)
+    except HTTPError as e:
+        log.error(str(e))
+        sys.exit(1)
+
+    log.info('Logs upload complete.')
 
 
 @click.command(name='maven-file')

diff --git a/lftools/deploy.py b/lftools/deploy.py
index eceb398..098d48f 100644 (file)
--- a/lftools/deploy.py
+++ b/lftools/deploy.py
@@ -14,6 +14,7 @@ import logging
 import os
 import re
 import shutil
+import subprocess
 import sys
 import tempfile
 
@@ -23,6 +24,28 @@ import requests
 log = logging.getLogger(__name__)
 
 
+def _compress_text(dir):
+    """Compress all text files in directory."""
+    save_dir = os.getcwd()
+    os.chdir(dir)
+
+    compress_types = [
+        '**/*.log',
+        '**/*.txt',
+    ]
+    paths = []
+    for _type in compress_types:
+        search = os.path.join(dir, _type)
+        paths.extend(glob2.glob(search, recursive=True))
+
+    for _file in paths:
+        with open(_file, 'rb') as src, gzip.open('{}.gz'.format(_file), 'wb') as dest:
+            shutil.copyfileobj(src, dest)
+            os.remove(_file)
+
+    os.chdir(save_dir)
+
+
 def _format_url(url):
     """Ensure url starts with http and trim trailing '/'s."""
     start_pattern = re.compile('^(http|https)://')
@@ -138,20 +161,7 @@ def deploy_archives(nexus_url, nexus_path, workspace, pattern=None):
     log.debug('workspace: {}, work_dir: {}'.format(workspace, work_dir))
 
     copy_archives(workspace, pattern)
-
-    compress_types = [
-        '**/*.log',
-        '**/*.txt',
-    ]
-    paths = []
-    for _type in compress_types:
-        search = os.path.join(work_dir, _type)
-        paths.extend(glob2.glob(search, recursive=True))
-
-    for _file in paths:
-        with open(_file, 'rb') as src, gzip.open('{}.gz'.format(_file), 'wb') as dest:
-            shutil.copyfileobj(src, dest)
-            os.remove(_file)
+    _compress_text(work_dir)
 
     archives_zip = shutil.make_archive(
         '{}/archives'.format(workspace), 'zip')
@@ -169,3 +179,92 @@ def deploy_archives(nexus_url, nexus_path, workspace, pattern=None):
                 r.status_code, r.content))
 
     shutil.rmtree(work_dir)
+
+
+def deploy_logs(nexus_url, nexus_path, build_url):
+    """Deploy logs to a Nexus site repository named logs.
+
+    Fetches logs and system information and pushes them to Nexus
+    for log archiving.
+    Requirements:
+
+    To use this API a Nexus server must have a site repository configured
+    with the name "logs" as this is a hardcoded path.
+
+    Parameters:
+
+        :nexus_url: URL of Nexus server. Eg: https://nexus.opendaylight.org
+        :nexus_path: Path on nexus logs repo to place the logs. Eg:
+            $SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER
+        :build_url: URL of the Jenkins build. Jenkins typically provides this
+                    via the $BUILD_URL environment variable.
+    """
+    nexus_url = _format_url(nexus_url)
+    work_dir = tempfile.mkdtemp(prefix='lftools-dl.')
+    os.chdir(work_dir)
+    log.debug('work_dir: {}'.format(work_dir))
+
+    build_details = open('_build-details.log', 'w+')
+    build_details.write('build-url: {}'.format(build_url))
+
+    with open('_sys-info.log', 'w+') as sysinfo_log:
+        sys_cmds = []
+
+        log.debug('Platform: {}'.format(sys.platform))
+        if sys.platform == "linux" or sys.platform == "linux2":
+            sys_cmds = [
+                ['uname', '-a'],
+                ['lscpu'],
+                ['nproc'],
+                ['df', '-h'],
+                ['free', '-m'],
+                ['ip', 'addr'],
+                ['sar', '-b', '-r', '-n', 'DEV'],
+                ['sar', '-P', 'ALL'],
+            ]
+
+        for c in sys_cmds:
+            try:
+                output = subprocess.check_output(c).decode('utf-8')
+            except OSError:  # TODO: Switch to FileNotFoundError when Python < 3.5 support is dropped.
+                log.debug('Command not found: {}'.format(c))
+                continue
+
+            output = '---> {}:\n{}\n'.format(' '.join(c), output)
+            sysinfo_log.write(output)
+            log.info(output)
+
+    build_details.close()
+
+    # Magic string used to trim console logs at the appropriate level during wget
+    MAGIC_STRING = "-----END_OF_BUILD-----"
+    log.info(MAGIC_STRING)
+
+    resp = requests.get('{}/consoleText'.format(_format_url(build_url)))
+    with open('console.log', 'w+') as f:
+        f.write(resp.text.split(MAGIC_STRING)[0])
+
+    resp = requests.get('{}/timestamps?time=HH:mm:ss&appendLog'.format(_format_url(build_url)))
+    with open('console-timestamp.log', 'w+') as f:
+        f.write(resp.text.split(MAGIC_STRING)[0])
+
+    _compress_text(work_dir)
+
+    console_zip = tempfile.NamedTemporaryFile(prefix='lftools-dl', delete=True)
+    log.debug('console-zip: {}'.format(console_zip.name))
+    shutil.make_archive(console_zip.name, 'zip', work_dir)
+    log.debug('listdir: {}'.format(os.listdir(work_dir)))
+
+    # TODO: Update to use I58ea1d7703b626f791dcd74e63251c4f3261ca7d once it's available.
+    upload_files = {'upload_file': open('{}.zip'.format(console_zip.name), 'rb')}
+    url = '{}/service/local/repositories/logs/content-compressed/{}'.format(
+        nexus_url, nexus_path)
+    r = requests.post(url, files=upload_files)
+    log.debug('{}: {}'.format(r.status_code, r.text))
+    if r.status_code != 201:
+        raise requests.exceptions.HTTPError(
+            'Failed to upload to Nexus with status code {}.\n{}'.format(
+                r.status_code, r.content))
+
+    console_zip.close()
+    shutil.rmtree(work_dir)

diff --git a/releasenotes/notes/refactor-deploy-logs-8631ffcf7eb7cad2.yaml b/releasenotes/notes/refactor-deploy-logs-8631ffcf7eb7cad2.yaml
new file mode 100644 (file)
index 0000000..cb29d13
--- /dev/null
+++ b/releasenotes/notes/refactor-deploy-logs-8631ffcf7eb7cad2.yaml
@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Refactored deploy_logs() function from shell/deploy to pure Python to
+    be more portable with Windows systems.
+deprecations:
+  - |
+    shell/deploy script's deploy_logs() function is now deprecated and will
+    be removed in a future release.

diff --git a/tests/test_deploy.py b/tests/test_deploy.py
index a3d5b4e..5458502 100644 (file)
--- a/tests/test_deploy.py
+++ b/tests/test_deploy.py
@@ -116,6 +116,28 @@ def test_deploy_archive(cli_runner, datafiles, responses):
         obj={})
     assert result.exit_code == 1
 
+@pytest.mark.datafiles(
+    os.path.join(FIXTURE_DIR, 'deploy'),
+    )
+def test_deploy_logs(cli_runner, datafiles, responses):
+    """Test deploy_logs() command for expected upload cases."""
+    os.chdir(str(datafiles))
+    workspace_dir = os.path.join(str(datafiles), 'workspace')
+
+    # Test successful upload
+    build_url = 'https://jenkins.example.org/job/builder-check-poms/204'
+    nexus_url = 'https://nexus.example.org/service/local/repositories/logs/content-compressed'
+    responses.add(responses.GET, '{}/consoleText'.format(build_url),
+                  status=201)
+    responses.add(responses.GET, '{}/timestamps?time=HH:mm:ss&appendLog'.format(build_url),
+                  body='This is a console timestamped log.', status=201)
+    responses.add(responses.POST, '{}/test/log/upload'.format(nexus_url), status=201)
+    result = cli_runner.invoke(
+        cli.cli,
+        ['--debug', 'deploy', 'logs', 'https://nexus.example.org', 'test/log/upload', build_url],
+        obj={})
+    assert result.exit_code == 0
+
 def mocked_log_error(msg1=None, msg2=None):
     """Mock local_log_error_and_exit function.
     This function is modified to simply raise an Exception.