Required parameters:
-**project**: is the project repo as defined in Gerrit.
+**project**: is the project repo as defined in source control.
**project-name**: is a custom name to call the job in Jenkins.
**build-node**: is the name of the builder to use when building (Jenkins label).
**branch**: is the git branch to build from.
**jjb-version**: is the version of JJB to install in the build minion.
+## Deploying Python jobs
+
+We provide the following Python jobs templates:
+
+### {project-name}-tox-verify-{stream}
+
+This job can be used to call python-tox to run builds and tests. The most common
+usage of this job is to run the Coala linter against projects.
+
+```
+- project:
+ name: builder
+ jobs:
+ - '{project-name}-tox-verify-{stream}'
+
+ project-name: builder
+ project: releng/builder
+ build-node: centos7-java-builder-2c-4g
+ stream: master
+```
+
+Required parameters:
+
+**project**: is the project repo as defined in source control.
+**project-name**: is a custom name to call the job in Jenkins.
+**build-node**: is the name of the builder to use when building (Jenkins label).
+**stream**: typically `master` or matching whatever branch is being built. This
+ is a useful keywords to map a release codename to a branch. For
+ example OpenDaylight uses this to map stream=carbon to
+ branch=stable/carbon.
+
+Optional parameters:
+
+**branch**: is the git branch to build from.
+**jjb-version**: is the version of JJB to install in the build minion.
+**tox-dir**: directory containing tox.ini file (default: '')
+**tox-envs**: tox environments to run (default: '')
+
## Archiving logs in Jobs
There are 2 ways supported for archiving log information:
- ../shell/jjb-install.sh
- ../shell/jjb-verify-job.sh
- ../shell/jjb-check-unicode.sh
- - gpg-verify-git-signature
+ - lf-infra-gpg-verify-git-signature
- parameter:
name: lf-infra-jjb-parameters
# yamllint disable-line rule:key-duplicates
<<: *lf_jjb_merge_builders
+ ######################
+ # Default parameters #
+ ######################
+
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+
+ #####################
+ # Job Configuration #
+ #####################
+
scm:
- lf-infra-gerrit-scm:
- jenkins-ssh-credential: '{jenkins-ssh-credential}'
+ git-url: '{git-url}'
refspec: ''
branch: '{branch}'
submodule-recursive: '{submodule-recursive}'
choosing-strategy: default
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
triggers:
- gerrit:
# yamllint disable-line rule:key-duplicates
<<: *lf_jjb_verify_builders
+ ######################
+ # Default parameters #
+ ######################
+
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+
+ #####################
+ # Job Configuration #
+ #####################
+
scm:
- lf-infra-gerrit-scm:
+ git-url: '{git-url}'
refspec: '$GERRIT_REFSPEC'
branch: '$GERRIT_BRANCH'
submodule-recursive: '{submodule-recursive}'
---
+# BUILDERS
- builder:
- name: create-netrc
+ name: lf-infra-create-netrc
# Macro to create a ~/.netrc file from a Maven settings.xml
# Parameters:
# {server-id} The id of a server as defined in settings.xml
- shell: !include-raw-escape: ../shell/create-netrc.sh
- builder:
- name: gpg-verify-git-signature
+ name: lf-infra-gpg-verify-git-signature
# Verify gpg signature of the latest commit message in $WORKSPACE
#
# This command assumes that $WORKSPACE is a git repo.
files:
- file-id: 'jenkins-log-archives-settings'
variable: 'SETTINGS_FILE'
- - create-netrc:
+ - lf-infra-create-netrc:
server-id: logs
- shell: !include-raw:
- ../shell/lftools-install.sh
- description-setter:
regexp: '^Build logs: .*'
+- builder:
+ name: lf-jacoco-nojava-workaround
+ builders:
+ - shell: 'mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes'
+
+# call maven-target builder with a goal of --version to force Jenkins to
+# install the needed maven version
+- builder:
+ name: lf-maven-install
+ builders:
+ - maven-target:
+ maven-version: '{maven-version}'
+ goals: '--version'
+
+# Push a global settings and user settings maven files
+- builder:
+ name: lf-provide-maven-settings
+ builders:
+ - config-file-provider:
+ files:
+ - file-id: '{global-settings-file}'
+ variable: 'GLOBAL_SETTINGS_FILE'
+ - file-id: '{settings-file}'
+ variable: 'SETTINGS_FILE'
+
+
+# PARAMETERS
+# Standard parameters used in the LF CI environments. Gerrit variables are not
+# used by GitHub projects, but defining them isn't harmful
- parameter:
name: lf-infra-parameters
parameters:
Note that Gerrit will override this parameter automatically if a
job is triggered by Gerrit.
+# Useful parameters when working with TOX
+# https://tox.readthedocs.io/
+- parameter:
+ name: lf-infra-tox-parameters
+ parameters:
+ - string:
+ name: TOX_DIR
+ default: '{tox-dir}'
+ description: |
+ Path to directory containing tox.ini file.
+ - string:
+ name: TOX_ENVS
+ default: '{tox-envs}'
+ description: |
+ Tox environments to run build against.
+ Example: docs,py2,py3
+
+# Set an env var for shell scripts to be able to call the dynamically installed
+# maven without having to calculate the path themselves
+- parameter:
+ name: lf-maven-exec
+ parameters:
+ - string:
+ name: MVN
+ # yamllint disable-line rule:line-length
+ default: '/w/tools/hudson.tasks.Maven_MavenInstallation/{maven-version}/bin/mvn'
+ description: 'Maven selector to be used by shell scripts'
+
+
+# PROPERTIES
- property:
name: lf-infra-properties
properties:
# Do not allow artifacts to be stored in Jenkins.
artifact-num-to-keep: 0
+# PUBLISHERS
- publisher:
name: lf-infra-publish
# lf-infra macro to finish up a build.
- '**/*.jenkins-trigger'
fail-build: false
+# SCM
- scm:
name: lf-infra-gerrit-scm
scm:
- git:
credentials-id: '{jenkins-ssh-credential}'
- url: '$GIT_URL/$GERRIT_PROJECT'
+ url: '{git-url}'
refspec: '{refspec}'
branches:
- 'refs/heads/{branch}'
recursive: '{submodule-recursive}'
choosing-strategy: '{choosing-strategy}'
+# WRAPPERS
- wrapper:
name: lf-infra-wrappers
wrappers:
--- /dev/null
+---
+#################
+# Job Templates #
+#################
+
+- job-template:
+ # Python projects typically use tox to run testing.
+ name: '{project-name}-tox-verify-{stream}'
+
+ # Required Variables:
+ # branch: git branch (default: master)
+ # tox-dir: directory containing the project's tox.ini relative to
+ # the workspace. Empty works if tox.ini is at project root.
+
+ project-type: freestyle
+ node: '{build-node}'
+ concurrent: true
+
+ ######################
+ # Default parameters #
+ ######################
+
+ branch: master
+ build-timeout: 10
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+ submodule-recursive: true
+ tox-dir: ''
+ tox-envs: ''
+
+ #####################
+ # Job Configuration #
+ #####################
+
+ properties:
+ - lf-infra-properties:
+ build-days-to-keep: 7
+
+ parameters:
+ - lf-infra-parameters:
+ project: '{project}'
+ branch: '{branch}'
+ - lf-infra-tox-parameters:
+ tox-dir: '{tox-dir}'
+ tox-envs: '{tox-envs}'
+
+ scm:
+ - lf-infra-gerrit-scm:
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+ git-url: '{git-url}'
+ refspec: '$GERRIT_REFSPEC'
+ branch: '$GERRIT_BRANCH'
+ submodule-recursive: '{submodule-recursive}'
+ choosing-strategy: gerrit
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ triggers:
+ - gerrit:
+ server-name: '{gerrit-server-name}'
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: false
+ exclude-trivial-rebase: false
+ exclude-no-code-change: false
+ - draft-published-event
+ - comment-added-contains-event:
+ comment-contains-value: recheck$
+ projects:
+ - project-compare-type: ANT
+ project-pattern: '{project}'
+ branches:
+ - branch-compare-type: ANT
+ branch-pattern: '**/{branch}'
+
+ builders:
+ - shell: !include-raw-escape:
+ # Workaround issue where the tox run later breaks the lftools virtualenv.
+ # Without running the install first the run in the publisher will fail
+ # due to missing lftools because it gets installed into a tox venv.
+ - ../shell/lftools-install.sh
+ - shell: !include-raw-escape:
+ - ../shell/tox-install.sh
+ - ../shell/tox-run.sh
+
+ publishers:
+ - lf-infra-publish
##############################################################################
echo "---> gpg-verify-git-signature.sh"
-if git log --show-signature -1 | egrep -q 'gpg: Signature made.*key ID'; then
+if git log --show-signature -1 | grep -E -q 'gpg: Signature made.*key ID'; then
echo "Git commit is GPG signed."
else
echo "WARNING: GPG signature missing for the commit."
# DO NOT set -u as virtualenv's activate script has unbound variables
set -e -o pipefail
-virtualenv "$WORKSPACE/.virtualenvs/lftools"
+virtualenv --quiet "$WORKSPACE/.virtualenvs/lftools"
# shellcheck source=./.virtualenvs/lftools/bin/activate disable=SC1091
source "$WORKSPACE/.virtualenvs/lftools/bin/activate"
-pip install --quiet --upgrade pip
-pip install --quiet --upgrade pipdeptree
-pip install --quiet --upgrade "lftools<1.0.0"
+PYTHON="$WORKSPACE/.virtualenvs/lftools/bin/python"
+$PYTHON -m pip install --quiet --upgrade pip
+$PYTHON -m pip install --quiet --upgrade pipdeptree
+$PYTHON -m pip install --quiet --upgrade "lftools<1.0.0"
echo "----> Pip Dependency Tree"
-pipdeptree
+$PYTHON -m pipdeptree
##############################################################################
# Clear log credential files
-rm $SETTINGS_FILE
+rm "$SETTINGS_FILE"
rm ~/.netrc
--- /dev/null
+#!/bin/bash
+echo "---> tox-install.sh"
+
+# Ensure we fail the job if any steps fail.
+# DO NOT set -u as virtualenv's activate script has unbound variables
+set -e -o pipefail
+
+virtualenv --quiet "$WORKSPACE/.virtualenvs/tox"
+# shellcheck source=./.virtualenvs/tox/bin/activate disable=SC1091
+source "$WORKSPACE/.virtualenvs/tox/bin/activate"
+PYTHON="$WORKSPACE/.virtualenvs/tox/bin/python"
+$PYTHON -m pip install --quiet --upgrade pip
+$PYTHON -m pip install --quiet --upgrade pipdeptree
+$PYTHON -m pip install --quiet --upgrade tox argparse
+
+echo "----> Pip Dependency Tree"
+$PYTHON -m pipdeptree
--- /dev/null
+#!/bin/bash
+echo "---> tox-install.sh"
+
+# Ensure we fail the job if any steps fail.
+# DO NOT set -u as virtualenv's activate script has unbound variables
+set -e -o pipefail
+
+cd "$WORKSPACE/$TOX_DIR"
+
+if [ -n "$TOX_ENVS" ];
+then
+ tox -e "$TOX_ENVS"
+else
+ tox
+fi