From: Anil Shashikumar Belur Date: Wed, 28 Jun 2017 07:22:22 +0000 (+0000) Subject: Merge "Add maven-deploy-file builder for packaging jobs" X-Git-Tag: v0.4.0^0 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=af3db4b5361d4ef71a37aed82fb75055201ffe31;hp=603e28b38f838bb8144949cb3281cdef53807854;p=releng%2Fglobal-jjb.git Merge "Add maven-deploy-file builder for packaging jobs" --- diff --git a/.coafile b/.coafile index 340ef8b6..4c0ea043 100644 --- a/.coafile +++ b/.coafile @@ -6,17 +6,35 @@ ignore_length_regex = Signed-off-by, http://, https:// -[YAML] -bears = YAMLLintBear -files = **/*.yaml -ignore = .*/** -document_start = True -yamllint_config = yamllint.conf +[Groovy] +bears = SpaceConsistencyBear +files = **/*.groovy +ignore = .git/**, + .tox/** +indent_size = 4 +use_spaces = true + +[MarkDown] +bears = MarkdownBear,SpaceConsistencyBear,WriteGoodLintBear +files = **/*.md, **/*.markdown +ignore = .git/**, + .tox/** +use_spaces = true [ShellCheck] bears = ShellCheckBear,SpaceConsistencyBear files = **/*.sh -ignore = .*/** +ignore = .git/**, + .tox/** shell = bash indent_size = 4 use_spaces = yeah + +[YAML] +bears = YAMLLintBear +files = **/*.yaml +ignore = .git/**, + .tox/** +document_start = True +yamllint_config = yamllint.conf + diff --git a/.gitignore b/.gitignore index 53704da6..2bb356d1 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,7 @@ target/ .tox/ __pycache__/ *.pyc + +# global-jjb +jjb/test.yaml +archives/ diff --git a/README.md b/README.md index cbaaa29d..40289bcc 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Global JJB -The purpose of this repository is store generically define reusable JJB -templates that can be deployed across LF projects. +The purpose of this repository is store generically defined, reusable JJB +templates, deployable across LF projects. -The following variables are necessary to be defined in the Jenkins server as +Define the following variables in the Jenkins server as global environment variables as scripts in this repo expect these variables to be available. @@ -17,7 +17,8 @@ LOGS_SERVER=https://logs.opendaylight.org NEXUS_URL=https://nexus.opendaylight.org SILO=releng ``` -Note: **GIT_CLONE_URL** is only used by Github projects as this + +Note: Use **GIT_CLONE_URL** for GitHub projects as this will be different from the URL used the poperties configuration. @@ -41,16 +42,16 @@ configuration. ## Installing global-jjb -global-jjb should be deployed in the ci-management repository's jjb directory as -a submodule. global-jjb is versioned and tagged in Gerrit so installing, -upgrading, and rolling back changes should be simple via the Gerrit tag system. +Deploy global-jjb in the ci-management repository's jjb directory as +a submodule. Installing, upgrading, and rolling back changes is simple via the +versioned git tags. ``` # Choose a global-jjb version to install GLOBAL_JJB_VERSION=v0.1.0 # Add the new submodule to ci-management's jjb directory. - # Note: Only needs to be performed once per ci-management repo. + # Note: Perform once per ci-management repo. cd jjb/ git submodule add https://gerrit.linuxfoundation.org/infra/releng/global-jjb @@ -69,8 +70,8 @@ upgrading, and rolling back changes should be simple via the Gerrit tag system. ## Parameters stored in defaults.yaml -There are a few project specific parameters that should be stored in the -ci-management repo's defaults.yaml file. +Configure the following parameters in the ci-management repo's defaults.yaml +file. **gerrit-server-name**: The name of the Gerrit Server as defined in Gerrit Trigger global configuration. @@ -78,16 +79,15 @@ Trigger global configuration. **jenkins-ssh-credential**: The name of the Jenkins Credential to use for ssh connections. -If you are using GitHub then there are two more parameters which -will need to be placed in the defaults.yaml +If you are using GitHub then configure the following parameters in defaults.yaml -**git-url**: This should be set to the base URL of your GitHub. In -general this should be https://github.com. If you are using GitHub +**git-url**: Set this to the base URL of your GitHub repo. In +general this should be . If you are using GitHub Enterprise, or some other GitHub-style system, then it should be whatever your installation base URL is. -**git-clone-url**: This is the clone prefix used by GitHub jobs. This -should be set to either the same thing as **git-url** or the +**git-clone-url**: This is the clone prefix used by GitHub jobs. Set this to +either the same thing as **git-url** or the 'git@github.com:' including the trailing ':' **github-org**: The name of the GitHub organization. @@ -114,7 +114,7 @@ credentials for the logs user in Nexus. ## Deploying ci-jobs -The CI job group contains multiple jobs that should be deployed in all LF +The CI job group contains jobs that should deploy in all LF Jenkins infra. The minimal configuration needed to deploy the ci-management jobs is as follows which deploys the **{project-name}-ci-jobs** job group as defined in **lf-ci-jobs.yaml**. @@ -150,7 +150,7 @@ We provide the following Python jobs templates: ### {project-name}-tox-verify-{stream} -This job can be used to call python-tox to run builds and tests. The most common +Use this job to call python-tox to run builds and tests. The most common usage of this job is to run the Coala linter against projects. ``` @@ -170,7 +170,7 @@ Required parameters: **project**: is the project repo as defined in source control. **project-name**: is a custom name to call the job in Jenkins. **build-node**: is the name of the builder to use when building (Jenkins label). -**stream**: typically `master` or matching whatever branch is being built. This +**stream**: typically `master` or matching the build branch. This is a useful keywords to map a release codename to a branch. For example OpenDaylight uses this to map stream=carbon to branch=stable/carbon. @@ -188,11 +188,20 @@ There are 2 ways supported for archiving log information: 1) Job creates $WORKSPACE/archives directory and places logs there -In this method the entire archives directory will be pushed to the log server +This method pushes the entire archives directory to the log server in the same structure as configured in the archives directory. 2) Via job variable ARCHIVE_ARTIFACTS using globstar patterns. -In this method a job can define a globstar for example ``**/*.log`` which then +In this method a job can define a globstar for example `**/*.log` which then causes the archive script to do a globstar search for that pattern and archives any files it finds matching. + +## Appendix + +### ShellCheck + +When using ShellCheck to lint global-jjb or any projects that include +global-jjb as part of their project (common with ci-management repos) then +we require version 0.4.x of ShellCheck installed on the build vms. This version +introduces annotations used by shell scripts in this repo. diff --git a/jenkins-admin/del_computer.groovy b/jenkins-admin/del_computer.groovy new file mode 100644 index 00000000..01438870 --- /dev/null +++ b/jenkins-admin/del_computer.groovy @@ -0,0 +1,39 @@ +/***************** +* Removes offline slave nodes +* +* NOTE: Some slaves can't be removed as the backing instance already is +* missing but the UI collection didn't get the update. See the +* slay_computer.groovy for a more drastic destruction +* +* NOTE 2: If you have any slaves you want to live through this and you +* have them currently marked offline you _must_ bring them back online. +*****************/ + +import hudson.model.* + +def numberOfflineNodes = 0 +def numberNodes = 0 + +slaveNodes = hudson.model.Hudson.instance + +for (slave in slaveNodes.nodes) { + def computer = slave.computer + numberNodes ++ + println "" + println "Checking computer ${computer.name}:" + if (computer.offline) { + numberOfflineNodes ++ + println '\tcomputer.isOffline: ' + slave.getComputer().isOffline() + println '\tcomputer.offline: ' + computer.offline + println '\tRemoving slave' + slaveNodes.removeNode(slave) + } else { + println '\tcomputer.isOffline: ' + slave.getComputer().isOffline() + println '\tcomputer.offline: ' + computer.offline + } +} + +println "Number of Offline Nodes: " + numberOfflineNodes +println "Number of Nodes: " + numberNodes + +// vim: sw=4 sts=4 ts=4 et ai : diff --git a/jenkins-admin/slay_computer.groovy b/jenkins-admin/slay_computer.groovy new file mode 100644 index 00000000..97ede6cd --- /dev/null +++ b/jenkins-admin/slay_computer.groovy @@ -0,0 +1,35 @@ +/***************** +* Completely slays misbehaving slave nodes +* +* NOTE: Use del_computer.groovy first! If nodes are still hanging around +* _then_ consider using this script. This one is mucking around in a +* location we should not normally be touching, however if a slave +* refuses to go away (tosses an exception) this _will_ get rid of it. +* +* NOTE 2: If you have any slaves you want to live through this and you +* have them currently marked offline you _must_ bring them back online. +*****************/ + +import jenkins.* +import jenkins.model.* +import hudson.* +import hudson.model.* + +for (aComputer in Jenkins.instance.computers) { + try { + println "displayName: " + aComputer.properties.displayName + println "offline: " + aComputer.properties.offline + println "temporarilyOffline: " + aComputer.properties.temporarilyOffline + if (aComputer.properties.offline) { + println "Bad node, removing" + Jenkins.instance.removeComputer(aComputer) + } + println "" + } + catch (NullPointerException nullPointer) { + println "NullPointerException caught" + println "" + } +} + +// vim: sw=4 sts=4 ts=4 et ai : diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 358e7eb5..da8230fb 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -57,6 +57,7 @@ builders: - lf-infra-jjbini - shell: !include-raw-escape: + - ../shell/git-validate-jira-urls.sh - ../shell/jjb-install.sh - ../shell/jjb-verify-job.sh - ../shell/jjb-check-unicode.sh diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 4081f775..cb374292 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -13,6 +13,34 @@ properties-content: 'SERVER_ID={server-id}' - shell: !include-raw-escape: ../shell/create-netrc.sh +- builder: + name: lf-infra-docker-login + # Login into a custom hosted docker registry and / or docker.io + # + # The Jenkins system should have the following global variables defined + # + # DOCKER_REGISTRY : Optional + # The DNS address of the registry (IP or FQDN) + # ex: nexus3.example.com + # + # REGISTRY_PORTS : Required if DOCKER_REGISTRY is set + # Space separated listing of the registry ports to login + # to + # ex: 10001 10002 10003 10004 + # + # DOCKERHUB_EMAIL : Optional + # If this variable is set then an attempt to login to + # DockerHub (docker.io) will be also made. It should be + # set to the email address for the credentials that will + # get looked up. Only _one_ credential will ever be found + # in the maven settings file for DockerHub + builders: + - lf-provide-maven-settings: + global-settings-file: '{global-settings-file}' + settings-file: '{settings-file}' + - shell: !include-raw-escape: ../shell/docker-login.sh + - lf-provide-maven-settings-cleanup + - builder: name: lf-infra-gpg-verify-git-signature # Verify gpg signature of the latest commit message in $WORKSPACE diff --git a/shell/docker-login.sh b/shell/docker-login.sh new file mode 100644 index 00000000..a2055a9b --- /dev/null +++ b/shell/docker-login.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Log into a custom hosted docker registry and / or docker.io + +# $DOCKER_REGISTRY : Optional +# Jenkins global variable should be defined +# If set, then this is the base IP or FQDN that will be used +# for logging into the custom docker registry +# ex: nexus3.example.com +# +# $REGISTRY_PORTS : Required if DOCKER_REGISTRY is set +# Jenkins global variable should be defined (space separated) +# Listing of all the registry ports to login to +# ex: 10001 10002 10003 10004 +# +# $SETTINGS_FILE : Job level variable with maven settings file location +# +# $DOCKERHUB_EMAIL : Optional +# Jenkins global variable that defines the email address that +# should be used for logging into DockerHub +# If defined than an attempt to login to docker hub will +# happen + +# Ensure we fail the job if any steps fail +set -eu -o pipefail + +# Execute the credential lookup and login to the registry +do_login() { + echo "$1" + CREDENTIAL=$(xmlstarlet sel -N "x=http://maven.apache.org/SETTINGS/1.0.0" \ + -t -m "/x:settings/x:servers/x:server[starts-with(x:id, '${1}')]" \ + -v x:username -o ":" -v x:password \ + "$SETTINGS_FILE") + + USER=$(echo "$CREDENTIAL" | cut -f1 -d:) + PASS=$(echo "$CREDENTIAL" | cut -f2 -d:) + + if [ -z "$USER" ] + then + echo "ERROR: No user provided" + return 1 + fi + + if [ -z "$PASS" ] + then + echo "ERROR: No password provided" + return 1 + fi + + docker login -u "$USER" -p "$PASS" -e "$2" "$1" +} + +if [ "${REGISTRY:-none}" != 'none' ] +then + for PORT in $REGISTRY_PORTS + do + REGISTRY="${DOCKER_REGISTRY}:${PORT}" + + # docker login requests an email address if nothing is passed to it + # Nexus, however, does not need this and ignores the value + do_login "$REGISTRY" none + done +fi + +# Attempt to login to docker.io only if $DOCKERHUB_EMAIL is configured +if [ "${DOCKERHUB_EMAIL:-none}" != 'none' ] +then + do_login docker.io "$DOCKERHUB_EMAIL" +fi diff --git a/shell/git-validate-jira-urls.sh b/shell/git-validate-jira-urls.sh new file mode 100644 index 00000000..d44fec93 --- /dev/null +++ b/shell/git-validate-jira-urls.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# @License EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# This script will make sure that there are no JIRA URLs in the commit +# message. JIRA URLs will break the its-jira plugin + +set -e -o pipefail +set +u + +JIRA_LINK=$(git rev-list --format=%B --max-count=1 HEAD | grep -io 'http[s]*://jira\..*') +if [[ ! -z "$JIRA_LINK" ]] +then + echo 'Remove JIRA URLs from commit message' + echo 'Add jira references as: Issue: -, instead of URLs' + exit 1 +fi diff --git a/shell/sysstat.sh b/shell/sysstat.sh index 03cab6dc..d90061c0 100644 --- a/shell/sysstat.sh +++ b/shell/sysstat.sh @@ -15,6 +15,11 @@ OS=$(facter operatingsystem) case "$OS" in Ubuntu) SYSSTAT_PATH="/var/log/sysstat" + + # Dont run the script when systat is not enabled by default + if ! grep --quiet 'ENABLED="true"' "/etc/default/sysstat"; then + exit 0 + fi ;; CentOS|RedHat) SYSSTAT_PATH="/var/log/sa" diff --git a/test.template b/test.template new file mode 100644 index 00000000..898f43e4 --- /dev/null +++ b/test.template @@ -0,0 +1,73 @@ +- defaults: + name: global + + # General + jenkins-ssh-credential: test-credential + + # Gerrit Infra + gerrit-server-name: test-server + + # GitHub Infra + git-url: https://github.com + git-clone-url: 'git@github.com:' + github-org: example-org + + # Common test config + project: releng/ciman + stream: latest + build-node: build-vm + +################ +# GERRIT TESTS # +################ + +- project: + name: gerrit-ci-jobs + jobs: + - "{project-name}-ci-jobs" + + project-name: gerrit-ciman + +- project: + name: gerrit-maven-jobs + jobs: + - gerrit-maven-release + + project-name: gerrit-maven + staging-profile-id: uuddlrlrba + settings-file: gerrit-maven-project-settings + +- project: + name: gerrit-python-jobs + jobs: + - "{project-name}-python-jobs" + + project-name: gerrit-python + +################ +# GITHUB TESTS # +################ + +- project: + name: github-ci-jobs + jobs: + - "{project-name}-github-ci-jobs" + + project-name: github-ciman + +- project: + name: github-maven-jobs + jobs: + - github-maven-release + + project-name: github-maven + staging-profile-id: uuddlrlrba + settings-file: aproject-settings + +- project: + name: github-python-jobs + jobs: + - "{project-name}-github-python-jobs" + + project-name: github-python + diff --git a/tox.ini b/tox.ini index a8804405..8f9cb513 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = coala +envlist = coala,jjb skipsdist = true [testenv:coala] @@ -8,6 +8,21 @@ basepython = python3 deps = coala coala-bears + # Markdown lint requires nodejs + nodeenv commands = + nodeenv -p + npm install --global remark-cli remark-lint write-good python3 -m nltk.downloader punkt maxent_treebank_pos_tagger averaged_perceptron_tagger coala --non-interactive + +[testenv:jjb] +deps = + jenkins-job-builder +commands = + cp test.template {toxinidir}/jjb/test.yaml + jenkins-jobs -l DEBUG test --recursive -o {toxinidir}/archives/job-configs {toxinidir}/jjb/ + rm {toxinidir}/jjb/test.yaml +whitelist_externals = + cp + rm