From 52980f24cf898a614c39d3a3c6674ccd4622aee0 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Fri, 26 May 2017 13:01:01 -0400 Subject: [PATCH] Add python tox verify job This is a generic verify job that can be used to test anything that uses Tox to run tests. This is mostly a port of OpenDaylight's version of the same job. Will be useful initially to test lftools and global-jjb but we plan to migrate all of the OpenDaylight Python / Tox jobs over to this template as well. Change-Id: I10417db8fc78a599b794087d3f9f729658214359 Signed-off-by: Thanh Ha --- README.md | 43 +++++++++++++++++++++--- jjb/lf-macros.yaml | 17 ++++++++++ jjb/lf-python-jobs.yaml | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ shell/tox-install.sh | 17 ++++++++++ shell/tox-run.sh | 15 +++++++++ 5 files changed, 177 insertions(+), 4 deletions(-) create mode 100644 jjb/lf-python-jobs.yaml create mode 100644 shell/tox-install.sh create mode 100644 shell/tox-run.sh diff --git a/README.md b/README.md index 01388fc4..6c684173 100644 --- a/README.md +++ b/README.md @@ -124,18 +124,53 @@ ci-management.yaml: Required parameters: -**project**: is the project repo as defined in Gerrit. +**project**: is the project repo as defined in source control. **project-name**: is a custom name to call the job in Jenkins. **build-node**: is the name of the builder to use when building (Jenkins label). Optional parameters: **branch**: is the git branch to build from. -**git-url**: is used to override the GIT_URL environment variable. Should not - be necessary in most cases. Mainly used by LF projects such as - global-jjb and lftools. **jjb-version**: is the version of JJB to install in the build minion. +## Deploying Python jobs + +We provide the following Python jobs templates: + +### {project-name}-tox-verify-{stream} + +This job can be used to call python-tox to run builds and tests. The most common +usage of this job is to run the Coala linter against projects. + +``` +- project: + name: builder + jobs: + - '{project-name}-tox-verify-{stream}' + + project-name: builder + project: releng/builder + build-node: centos7-java-builder-2c-4g + stream: master +``` + +Required parameters: + +**project**: is the project repo as defined in source control. +**project-name**: is a custom name to call the job in Jenkins. +**build-node**: is the name of the builder to use when building (Jenkins label). +**stream**: typically `master` or matching whatever branch is being built. This + is a useful keywords to map a release codename to a branch. For + example OpenDaylight uses this to map stream=carbon to + branch=stable/carbon. + +Optional parameters: + +**branch**: is the git branch to build from. +**jjb-version**: is the version of JJB to install in the build minion. +**tox-dir**: directory containing tox.ini file (default: '') +**tox-envs**: tox environments to run (default: '') + ## Archiving logs in Jobs There are 2 ways supported for archiving log information: diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index ee0bc313..310df5d7 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -109,6 +109,23 @@ Note that Gerrit will override this parameter automatically if a job is triggered by Gerrit. +# Useful parameters when working with TOX +# https://tox.readthedocs.io/ +- parameter: + name: lf-infra-tox-parameters + parameters: + - string: + name: TOX_DIR + default: '{tox-dir}' + description: | + Path to directory containing tox.ini file. + - string: + name: TOX_ENVS + default: '{tox-envs}' + description: | + Tox environments to run build against. + Example: docs,py2,py3 + # Set an env var for shell scripts to be able to call the dynamically installed # maven without having to calculate the path themselves - parameter: diff --git a/jjb/lf-python-jobs.yaml b/jjb/lf-python-jobs.yaml new file mode 100644 index 00000000..530d05b5 --- /dev/null +++ b/jjb/lf-python-jobs.yaml @@ -0,0 +1,89 @@ +--- +################# +# Job Templates # +################# + +- job-template: + # Python projects typically use tox to run testing. + name: '{project-name}-tox-verify-{stream}' + + # Required Variables: + # branch: git branch (default: master) + # tox-dir: directory containing the project's tox.ini relative to + # the workspace. Empty works if tox.ini is at project root. + + project-type: freestyle + node: '{build-node}' + concurrent: true + + ###################### + # Default parameters # + ###################### + + branch: master + build-timeout: 10 + git-url: '$GIT_URL/$GERRIT_PROJECT' + submodule-recursive: true + tox-dir: '' + tox-envs: '' + + ##################### + # Job Configuration # + ##################### + + properties: + - lf-infra-properties: + build-days-to-keep: 7 + + parameters: + - lf-infra-parameters: + project: '{project}' + branch: '{branch}' + - lf-infra-tox-parameters: + tox-dir: '{tox-dir}' + tox-envs: '{tox-envs}' + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: '{jenkins-ssh-credential}' + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: gerrit + + wrappers: + - lf-infra-wrappers: + build-timeout: '{build-timeout}' + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: + - patchset-created-event: + exclude-drafts: false + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: recheck$ + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + + builders: + - shell: !include-raw-escape: + # Workaround issue where the tox run later breaks the lftools virtualenv. + # Without running the install first the run in the publisher will fail + # due to missing lftools because it gets installed into a tox venv. + - ../shell/lftools-install.sh + - shell: !include-raw-escape: + - ../shell/tox-install.sh + - ../shell/tox-run.sh + + publishers: + - lf-infra-publish diff --git a/shell/tox-install.sh b/shell/tox-install.sh new file mode 100644 index 00000000..09b9ab27 --- /dev/null +++ b/shell/tox-install.sh @@ -0,0 +1,17 @@ +#!/bin/bash +echo "---> tox-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +virtualenv --quiet "$WORKSPACE/.virtualenvs/tox" +# shellcheck source=./.virtualenvs/tox/bin/activate disable=SC1091 +source "$WORKSPACE/.virtualenvs/tox/bin/activate" +PYTHON="$WORKSPACE/.virtualenvs/tox/bin/python" +$PYTHON -m pip install --quiet --upgrade pip +$PYTHON -m pip install --quiet --upgrade pipdeptree +$PYTHON -m pip install --quiet --upgrade tox argparse + +echo "----> Pip Dependency Tree" +pipdeptree diff --git a/shell/tox-run.sh b/shell/tox-run.sh new file mode 100644 index 00000000..6560094c --- /dev/null +++ b/shell/tox-run.sh @@ -0,0 +1,15 @@ +#!/bin/bash +echo "---> tox-install.sh" + +# Ensure we fail the job if any steps fail. +# DO NOT set -u as virtualenv's activate script has unbound variables +set -e -o pipefail + +cd "$WORKSPACE/$TOX_DIR" + +if [ -n "$TOX_ENVS" ]; +then + tox -e "$TOX_ENVS" +else + tox +fi -- 2.16.6