From: Thanh Ha Date: Mon, 6 Mar 2017 22:59:38 +0000 (-0500) Subject: Add deploy command to lftools X-Git-Tag: v0.1.0~1^2 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F40%2F4340%2F4;p=releng%2Flftools.git Add deploy command to lftools Provides a generic script that can be used to deploy artifacts to a Nexus site repository. Currently it is hardcoded to use a site repo named "logs" but a future update can be implemented to allow this repo to be configurable if necessary. Provides 3 deploy commands: archives) Deploys everything that exists in $WORKSPACE/archives if it exists. This needs to be provided by the job. Designed to run at the end of a Jenkins build. logs) Deploys system information and build console logs. The script will handle all this. Designed to run at the end of a Jenkins build. files) An arbitrary list of files passed by the user will be deployed to the logs server into an archives directory. Designed to be used inside of scripts at any time when the job needs to deploy anything to the server. This is an alternative to having to place all files inside an archives directory. (Unimplemented, will be implemented in a future patch) Change-Id: Ie48f8c65d89fcdce2cc2619fa4bc3408b1e4d709 Signed-off-by: Thanh Ha --- diff --git a/lftools/cli/__init__.py b/lftools/cli/__init__.py index 69c84ad0..d1b56cdf 100644 --- a/lftools/cli/__init__.py +++ b/lftools/cli/__init__.py @@ -13,6 +13,7 @@ __author__ = 'Thanh Ha' import click +from lftools.cli.deploy import deploy from lftools.cli.nexus import nexus from lftools.cli.version import version from lftools.openstack.cmd import openstack @@ -26,6 +27,7 @@ def cli(ctx): pass +cli.add_command(deploy) cli.add_command(openstack) cli.add_command(nexus) cli.add_command(version) diff --git a/lftools/cli/deploy.py b/lftools/cli/deploy.py new file mode 100644 index 00000000..cde4fbb3 --- /dev/null +++ b/lftools/cli/deploy.py @@ -0,0 +1,68 @@ +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +"""Script to deploy files to a Nexus sites repository.""" + +__author__ = 'Thanh Ha' + + +import subprocess + +import click + + +@click.group() +@click.pass_context +def deploy(ctx): + """Deploy files to a Nexus sites repository.""" + pass + + +@click.command() +@click.argument('nexus-url', envvar='NEXUS_URL') +@click.argument('nexus-path', envvar='NEXUS_PATH') +@click.argument('workspace', envvar='WORKSPACE') +@click.option('-p', '--pattern', default='') +@click.pass_context +def archives(ctx, nexus_url, nexus_path, workspace, pattern): + """Archive files to a Nexus site repository. + + Provides 2 ways to archive files: + + \b + 1) globstar pattern provided by the user. + 2) $WORKSPACE/archives directory provided by the user. + + To use this script the Nexus server must have a site repository configured + with the name "logs" as this is a hardcoded path. Also this script uses + ~/.netrc for it's authentication which must be provided. + """ + subprocess.call(['deploy', 'archives', nexus_url, nexus_path, workspace, pattern]) + + +@click.command() +@click.argument('nexus-url', envvar='NEXUS_URL') +@click.argument('nexus-path', envvar='NEXUS_PATH') +@click.argument('build-url', envvar='BUILD_URL') +@click.pass_context +def logs(ctx, nexus_url, nexus_path, build_url): + """Deploy logs to a Nexus site repository. + + This script fetches logs and system information and pushes them to Nexus + for log archiving. + + To use this script the Nexus server must have a site repository configured + with the name "logs" as this is a hardcoded path. Also this script uses + ~/.netrc for it's authentication which must be provided. + """ + subprocess.call(['deploy', 'logs', nexus_url, nexus_path, build_url]) + + +deploy.add_command(archives) +deploy.add_command(logs) diff --git a/setup.py b/setup.py index e8401877..13d96d28 100644 --- a/setup.py +++ b/setup.py @@ -40,6 +40,7 @@ setup( lftools=lftools.cli:main ''', scripts=[ + 'shell/deploy', 'shell/version', ], ) diff --git a/shell/deploy b/shell/deploy new file mode 100755 index 00000000..9ec165b7 --- /dev/null +++ b/shell/deploy @@ -0,0 +1,226 @@ +#!/bin/bash + +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2016, 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script handles sending files to a Nexus site repo for archiving. Mainly +# useful for storing logs for example on logs.opendaylight.org. + +copy_archives() { + # Find all files matching $pattern in a $workspace and copies it to the + # current directory. The best way to use this function is to cd into the + # directory you wish to store the files first before calling the function. + # + # To use this script the Nexus server must have a site repository configured + # with the name "logs" as this is a hardcoded path. Also this script uses + # ~/.netrc for it's authentication which must be provided. + # + # Usage: copy_archives + # + # : Directory in which to search, typically in Jenkins this is + # $WORKSPACE + # : Globstar pattern that is space separated of files that should + # be archived. (optional) + local workspace="$1" + local archive_pattern="${2:-}" + local dest_dir + dest_dir="$(pwd)" + + pushd "$workspace" + + # First copy all archives provided by user if any + if [ "$(ls -A "$workspace/archives/")" ]; then + mv "$workspace/archives/"* "$dest_dir" + fi + + # Copy all files matching pattern provided by user + if [ ! -z "$archive_pattern" ]; then + shopt -s globstar # Enable globstar to copy archives + for pattern in $archive_pattern; do + [[ -e $pattern ]] || continue # handle the case of no files to archive + echo "Archiving $pattern" >> "$workspace/archives.log" + dir=$(dirname "$pattern") + mkdir -p "$dest_dir/$dir" + mv "$pattern" "$dest_dir/$pattern" + done + shopt -u globstar # Disable globstar once archives are copied + fi + popd +} + +deploy() { + # Entry point for the deploy command. + subcommand=$1; shift + + case "$subcommand" in + archives ) + echo "Deploying archives..." + deploy_archives "$@" + exit 0 + ;; + files ) + echo "Deploying files..." + echo "ERROR: Unimplemented." + exit 1 + ;; + logs ) + echo "Deploying logs..." + deploy_logs "$@" + exit 0 + ;; + * ) + echo "Invalid command: $subcommand" 1>&2 + exit 1 + ;; + esac +} + +deploy_archives() { + # Archive files provided by the user to a Nexus site repository named logs. + # + # Provides 2 ways to archive files: + # 1) globstar pattern provided by the user. + # 2) $WORKSPACE/archives directory provided by the user. + # + # To use this script the Nexus server must have a site repository configured + # with the name "logs" as this is a hardcoded path. Also this script uses + # ~/.netrc for it's authentication which must be provided. + # + # Usage: deploy_archives + # + # : URL of Nexus server. Eg: https://nexus.opendaylight.org + # : Path on nexus logs repo to place the logs. Eg: + # $SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER + # : Directory in which to search, typically in Jenkins this is + # $WORKSPACE + # : Globstar pattern that is space separated of files that should + # be archived. (optional) + + if [ -z "$3" ]; then + echo "Missing required arguments." + exit 1 + fi + + local nexus_url="$1" + local nexus_path="$2" + # Workspace of where to search for files to archive. + local workspace="$3" + # Pattern to archive (globstar allowed). Recommended to double quote the + # input so that the full pattern can be passed into the function. + local archive_pattern="${4:-}" + + tmpdir=$(mktemp -d) + pushd "$tmpdir" + + ################### + # BEGIN ARCHIVING # + ################### + touch "$workspace/archives.log" + copy_archives "$workspace" "$archive_pattern" + + # Find and gzip any 'text' files + find "$tmpdir" -type f -print0 \ + | xargs -0r file \ + | grep --extended-regexp --regexp ':.*text.*' \ + | cut -d: -f1 \ + | xargs -d'\n' -r gzip + + if [ "$(ls -A)" ]; then + zip -qr "$workspace/archives.zip" . >> "$workspace/archives.log" + du -sh "$workspace/archives.zip" + gzip --force "$workspace/archives.log" + + echo "Pushing archives.log.gz" + curl --netrc --upload-file "$workspace/archives.log.gz" \ + "${nexus_url}/content/repositories/logs/${nexus_path}/archives.log.gz" + + echo "Pushing archives.zip" + curl --netrc --upload-file "$workspace/archives.zip" \ + "${nexus_url}/service/local/repositories/logs/content-compressed/${nexus_path}" + else + echo "Nothing to archive." + fi + + popd + rm -rf "$tmpdir" +} + +deploy_logs() { + # Deploy logs to a Nexus site repository named logs. + # + # This script fetches logs and system information and pushes them to Nexus + # for log archiving. + # + # To use this script the Nexus server must have a site repository configured + # with the name "logs" as this is a hardcoded path. Also this script uses + # ~/.netrc for it's authentication which must be provided. + # + # Usage: deploy + # + # : URL of Nexus server. Eg: https://nexus.opendaylight.org + # : Path on nexus logs repo to place the logs. Eg: + # $SILO/$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER + # : URL of the Jenkins build. Jenkins typicallyi provides this + # via the $BUILD_URL environment variable. + + if [ -z "$3" ]; then + echo "Missing required arguments." + echo "Usage: deploy " + exit 1 + fi + + local nexus_url="$1" + local nexus_path="$2" + local build_url="$3" + + tmpdir=$(mktemp -d) + pushd "$tmpdir" + + touch "_build-details.log" + { + echo "build-url: ${build_url}" + } 2>&1 | tee -a "_build-details.log" + env | grep -v PASSWORD | sort > "_build-enviroment-variables.log" + + # Print system info before collecting logs + touch "_sys-info.log" + { + echo -e "---> uname -a:\n $(uname -a) \n" + echo -e "---> lscpu:\n $(lscpu) \n" + echo -e "---> nproc:\n $(nproc) \n" + echo -e "---> df -h:\n $(df -h) \n" + echo -e "---> free -m:\n $(free -m) \n" + echo -e "---> ip addr:\n $(/sbin/ip addr) \n" + } 2>&1 | tee -a "_sys-info.log" + + # Magic string used to trim console logs at the appropriate level during wget + MAGIC_STRING="-----END_OF_BUILD-----" + echo "$MAGIC_STRING" + + wget --no-verbose -O "console.log" "${build_url}consoleText" + wget --no-verbose -O "console-timestamp.log" "${build_url}/timestamps?time=HH:mm:ss&appendLog" + sed -i "/^$MAGIC_STRING$/q" "console.log" + sed -i "/^.*$MAGIC_STRING$/q" "console-timestamp.log" + + gzip -- *.log + zip -r console-logs.zip -- *.log.gz + + curl --netrc --upload-file console-logs.zip \ + "${nexus_url}/service/local/repositories/logs/content-compressed/${nexus_path}" + + popd + rm -rf "$tmpdir" +} + +# Only run the script if it is being called directly and not sourced. +if [[ "${BASH_SOURCE[0]}" == "${0}" ]] +then + deploy "$@" +fi diff --git a/shell/deploy-archives b/shell/deploy-archives deleted file mode 100644 index 9a6bcb49..00000000 --- a/shell/deploy-archives +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/bash - -set +e # Do not affect the build result if some part of archiving fails. - -# Print out git status at the end of the build before we archive if $WORKSPACE -# is a git repo. -if [ -d "$WORKSPACE/.git" ]; then - echo "" - echo "----------> Git Status Report" - git status -fi - -echo "" -echo "----------> Archiving build to logs server" -# Configure wget to not print download status when we download logs or when -# Jenkins is installing Maven (To be clear this is the Jenkins Maven plugin -# using a shell script itself that we are unable to modify directly to affect -# wget). -echo "verbose=off" > ~/.wgetrc - -ARCHIVES_DIR="$JENKINS_HOSTNAME/$JOB_NAME/$BUILD_NUMBER" -[ "$LOGS_SERVER" ] || LOGS_SERVER="https://logs.opendaylight.org" -[ "$LOGS_REPO_URL" ] || LOGS_REPO_URL="https://nexus.opendaylight.org/service/local/repositories/logs" - -echo "Build logs: $LOGS_SERVER/$SILO/$ARCHIVES_DIR" - -mkdir .archives -cd .archives/ || exit 404 - -cat > deploy-archives.xml < - 4.0.0 - logs - logs - 1.0.0 - pom - - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - - true - - - - org.sonatype.plugins - maven-upload-plugin - 0.0.1 - - - publish-site - deploy - - upload-file - - - opendaylight-log-archives - $LOGS_REPO_URL/content-compressed - archives.zip - $SILO - - - - - - - -EOF - -mkdir -p "$ARCHIVES_DIR" -mkdir -p "$WORKSPACE/archives" -if [ ! -z "$ARCHIVE_ARTIFACTS" ]; then - pushd "$WORKSPACE" - shopt -s globstar # Enable globstar to copy archives - for f in $ARCHIVE_ARTIFACTS; do - [[ -e $f ]] || continue # handle the case of no files to archive - echo "Archiving $f" - dir=$(dirname "$f") - mkdir -p "$WORKSPACE/archives/$dir" - mv "$f" "$WORKSPACE/archives/$f" - done - shopt -u globstar # Disable globstar once archives are copied - popd -fi - - -# Ignore logging if archives doesn't exist -mv "$WORKSPACE/archives/" "$ARCHIVES_DIR" > /dev/null 2>&1 -touch "$ARCHIVES_DIR/_build-details.txt" -echo "build-url: ${BUILD_URL}" >> "$ARCHIVES_DIR/_build-details.txt" -env | grep -v PASSWORD > "$ARCHIVES_DIR/_build-enviroment-variables.txt" - -# capture system info -touch "$ARCHIVES_DIR/_sys-info.txt" -{ - echo -e "uname -a:\n $(uname -a) \n" - echo -e "df -h:\n $(df -h) \n" - echo -e "free -m:\n $(free -m) \n" - echo -e "nproc:\n $(nproc) \n" - echo -e "lscpu:\n $(lscpu) \n" - echo -e "ip addr:\n $(/sbin/ip addr) \n" -} 2>&1 | tee -a "$ARCHIVES_DIR/_sys-info.txt" - -# Magic string used to trim console logs at the appropriate level during wget -echo "-----END_OF_BUILD-----" -wget -O "$ARCHIVES_DIR/console.log" "${BUILD_URL}consoleText" -wget -O "$ARCHIVES_DIR/console-timestamp.log" "${BUILD_URL}/timestamps?time=HH:mm:ss&appendLog" -sed -i '/^-----END_OF_BUILD-----$/,$d' "$ARCHIVES_DIR/console.log" -sed -i '/^.*-----END_OF_BUILD-----$/,$d' "$ARCHIVES_DIR/console-timestamp.log" - -gzip "$ARCHIVES_DIR"/*.txt "$ARCHIVES_DIR"/*.log -# find and gzip any 'text' files -find "$ARCHIVES_DIR" -type f -print0 \ - | xargs -0r file \ - | egrep -e ':.*text.*' \ - | cut -d: -f1 \ - | xargs -d'\n' -r gzip - -zip -r archives.zip "$JENKINS_HOSTNAME/" > "$ARCHIVES_DIR/_archives-zip.log" -du -sh archives.zip