From: Jessica Wagantall Date: Tue, 17 Sep 2019 23:56:24 +0000 (+0000) Subject: Merge "Add sonar-prescan jobs" X-Git-Tag: v0.44.0~5 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=5f28702c4a74e49627390ffd108c1beea141a27b;hp=e50da79e348b4302edf4f2cc2ed4842830f9cc54;p=releng%2Fglobal-jjb.git Merge "Add sonar-prescan jobs" --- diff --git a/.coafile b/.coafile index c0e13b0b..86889a57 100644 --- a/.coafile +++ b/.coafile @@ -36,7 +36,8 @@ ignore = .git/**, docs/jjb/lf-ci-jobs.rst, docs/jjb/lf-docker-jobs.rst, docs/jjb/lf-macros.rst, - docs/jjb/lf-maven-jobs.rst + docs/jjb/lf-maven-jobs.rst, + docs/jjb/lf-python-jobs.rst [all.ShellCheck] bears = ShellCheckBear,SpaceConsistencyBear diff --git a/.jjb-test/lf-ci-jobs.yaml b/.jjb-test/lf-ci-jobs.yaml index ed0ef468..baeced5b 100644 --- a/.jjb-test/lf-ci-jobs.yaml +++ b/.jjb-test/lf-ci-jobs.yaml @@ -66,6 +66,8 @@ - java-builder - mininet + update-cloud-images: false + - project: name: throttle-ci-jobs jobs: diff --git a/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml new file mode 100644 index 00000000..fd6eee8a --- /dev/null +++ b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml @@ -0,0 +1,16 @@ +--- +- project: + name: openstack-update-cloud-images-full-test + jobs: + - "gerrit-openstack-update-cloud-image" + + project: ciman + project-name: ciman-full + build-timeout: 10 + branch: master + archive-artifacts: "**/*.log" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + gerrit-user: "jenkins-user" + gerrit-host: "git.example.org" + gerrit-topic: "update-cloud-image" + reviewers-email: "jenkins-user@example.org" diff --git a/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml new file mode 100644 index 00000000..9f536002 --- /dev/null +++ b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml @@ -0,0 +1,11 @@ +--- +- project: + name: openstack-update-cloud-images-minimal-test + jobs: + - "gerrit-openstack-update-cloud-image" + + project-name: ciman-minimal + gerrit-user: "jenkins-user" + gerrit-host: "git.example.org" + gerrit-topic: "update-cloud-image" + reviewers-email: "jenkins-user@example.org" diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst index 140b979f..20dde4ef 100644 --- a/docs/jjb/lf-ci-jobs.rst +++ b/docs/jjb/lf-ci-jobs.rst @@ -650,6 +650,71 @@ Full Example: .. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-cron-full.yaml +.. _gjjb-openstack-update-cloud-image: + +OpenStack Update Cloud Image +---------------------------- + +This job finds and updates OpenStack cloud images on the ci-management source +repository. + +The job is triggered in two ways: + +1. When packer merge job completes, the new image name created is passed + down to the job. +2. When the job is triggered manually to update all new images. + +When the job is triggered through an upstream packer merge job, this only +generates a change request for the new image built. + +When the job is triggered manually, this job finds the latest images on +OpenStack cloud and compares them with the images currently used in the source +ci-management source repository. If the compared images have newer +time stamps are **all** updated through a change request. + +This job requires a Jenkins configuration merge and verify job setup and +working on Jenkins. + +:Template Names: + - {project-name}-openstack-update-cloud-image + - gerrit-openstack-update-cloud-image + - github-openstack-update-cloud-image + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally should + be configured in defaults.yaml) + :new-image-name: Name of new image name passed from packer merge job or + set to 'all' to update all images. (default: all) + +:Optional parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 90) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :openstack-cloud: OS_CLOUD setting to pass to openstack client. + (default: vex) + :stream: Keyword that can be used to represent a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. (default: false) + +Minimal Example: + +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml + +Full Example: + +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml + .. _gjjb-packer-merge: @@ -700,6 +765,9 @@ Packer Merge job runs `packer build` to build system images in the cloud. (default: false) :gerrit_verify_triggers: Override Gerrit Triggers. + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. (default: false) + Test an in-progress patch ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index 0f8fd20b..ff8c561a 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -126,6 +126,11 @@ Run `packer build` to build system images. :platform: Build platform as found in the vars directory. :template: Packer template to build as found in the templates directory. +:Optional parameters: + + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. + lf-infra-packer-validate ------------------------ @@ -167,6 +172,13 @@ lf-infra-sysstat Retrieves system stats. +lf-infra-update-packer-images +----------------------------- + +Find and update the new built cloud image{s} in the ci-management source +repository. + + lf-jacoco-nojava-workaround --------------------------- @@ -240,7 +252,12 @@ Script to trigger a build on http://readthedocs.org lf-rtd-verify ------------- -ReadTheDocs verify script. +ReadTheDocs verify script. Installs and runs tox. + +:Required parameters: + + :doc-dir: Document directory. + :python-version: Python version. check-info-votes ---------------- diff --git a/docs/jjb/lf-python-jobs.rst b/docs/jjb/lf-python-jobs.rst index c36ba8b9..175f0e26 100644 --- a/docs/jjb/lf-python-jobs.rst +++ b/docs/jjb/lf-python-jobs.rst @@ -87,7 +87,7 @@ IQ Server. :java-version: Version of Java to use for the build. (default: openjdk8) :pre-build-script: Shell script to execute before the CLM builder. For example, install prerequisites or move files to the repo root. - (default: a string with only a comment) + (default: a string with a shell comment) :stream: Keyword used to represent a release code-name. Often the same as the branch. (default: master) :submodule-recursive: Whether to checkout submodules recursively. @@ -106,16 +106,21 @@ IQ Server. Python Sonar with Tox --------------------- -Sonar scans for Python based repos. This job will invoke tox to run tests -and gather coverage statistics from test results. Then the job invokes Maven -with a Sonar goal, which runs a plugin to publish results to a Sonar server. +Sonar scans for Python based repos. This job invokes tox to run tests and +gather coverage statistics from the test results, then invokes Maven to +publish the results to a Sonar server. -To get the Sonar coverage results, tox.ini needs to exist and be configured -with coverage commands to run. +To get the Sonar coverage results, file tox.ini must exist and contain coverage +commands to run. The coverage commands define the code that gets executed by the test suites. -It does not guarantee that the code tests executed properly, but it will help -pointing out the code that is not tested at all. +Checking coverage does not guarantee that the tests execute properly, but it +identifies code that is not executed by any test. + +This job reuses the Sonar builder used in Java/Maven projects which runs maven +twice. The first invocation does nothing for Python projects, so the job uses +the goal 'validate' by default. The second invocation publishes results using +the goal 'sonar:sonar' by default. For example: @@ -166,7 +171,7 @@ https://docs.sonarqube.org/display/PLUG/Python+Coverage+Results+Import :mvn-version: Version of maven to use. (default: mvn35) :pre-build-script: Shell script to execute before the Sonar builder. For example, install prerequisites or move files to the repo root. - (default: a string with only a comment) + (default: a string with a comment) :python-version: Python version (default: python2) :sonar-mvn-goal: The Maven goal to run the Sonar plugin. (default: sonar:sonar) :stream: Keyword used to represent a release code-name. @@ -187,10 +192,10 @@ https://docs.sonarqube.org/display/PLUG/Python+Coverage+Results+Import Tox Verify ---------- -Tox runner to verify a project. This job is pyenv aware so if the image -contains an installation of pyenv at /opt/pyenv it will pick it up and run -Python tests with the appropriate Python versions. This job will set the -following pyenv variables before running. +Tox runner to verify a project on creation of a patch set. +This job is pyenv aware so if the image contains an installation of pyenv +at /opt/pyenv it will pick it up and run Python tests with the appropriate +Python versions. This job will set the following pyenv variables before running. .. code:: bash @@ -218,6 +223,63 @@ following pyenv variables before running. :build-timeout: Timeout in minutes before aborting build. (default: 10) :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) :pre-build-script: Shell script to execute before the Tox builder. + For example, install prerequisites or move files to the repo root. + (default: a string with a shell comment) + :python-version: Version of Python to configure as a base in virtualenv. + (default: python3) + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :tox-dir: Directory containing the project's tox.ini relative to + the workspace. Empty works if tox.ini is at project root. + (default: '') + :tox-envs: Tox environments to run. If blank run everything described + in tox.ini. (default: '') + :gerrit_trigger_file_paths: Override file paths used to filter which + file modifications will trigger a build. Refer to JJB documentation for + "file-path" details. + https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit + + +Tox Merge +--------- + +Tox runner to verify a project after merge of a patch set. +This job is pyenv aware so if the image contains an installation of pyenv +at /opt/pyenv it will pick it up and run Python tests with the appropriate +Python versions. This job will set the following pyenv variables before running. + +.. code:: bash + + export PYENV_ROOT="/opt/pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + +:Template Names: + + - {project-name}-tox-merge-{stream} + - gerrit-tox-merge + - github-tox-merge + +:Comment Trigger: remerge + +:Required Parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + +:Optional Parameters: + + :branch: The branch to build against. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 10) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :pre-build-script: Shell script to execute before the CLM builder. For example, install prerequisites or move files to the repo root. (default: a string with only a comment) :python-version: Version of Python to configure as a base in virtualenv. @@ -235,7 +297,7 @@ following pyenv variables before running. (default: '') :tox-envs: Tox environments to run. If blank run everything described in tox.ini. (default: '') - :gerrit_trigger_file_paths: Override file paths which used to filter which + :gerrit_trigger_file_paths: Override file paths used to filter which file modifications will trigger a build. Refer to JJB documentation for "file-path" details. https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst index 91ce7a05..4ef52ad7 100644 --- a/docs/jjb/lf-release-jobs.rst +++ b/docs/jjb/lf-release-jobs.rst @@ -8,24 +8,23 @@ Self-serve release jobs allow a project team to direct Jenkins to promote a jar file or container image from a staging area to a release area. To trigger the action, create a releases/ or .releases/ directory, add a release yaml file to it, and submit a change set with -exactly one release yaml file to Gerrit. Upon merge of the change, -Jenkins will sign the reference extrapolated by log_dir and promote -the artifact. The expected format of the release yaml file is shown in -examples below. +one release yaml file to Gerrit. Upon merge of the change, Jenkins will +sign the reference extrapolated by log_dir and promote the artifact. The +expected format of the release yaml file appears in schemas and examples +below. The build node for maven and container release jobs must be CentOS, which supports the sigul client for accessing a signing server. The build node for container release jobs must have Docker installed. -A release job can also be triggered from Jenkins via the "Build with +A Jenkins user can also trigger a release job via the "Build with parameters" action, removing the need for a release yaml file. The -parameters must be filled out in the same way as a release yaml file, +user must enter parameters in the same way as a release yaml file, except for the special USE_RELEASE_FILE and DRY_RUN check boxes. The -USE_RELEASE_FILE check box must be unchecked if the job is expected to +user must uncheck the USE_RELEASE_FILE check box if the job should run with a release file, while passing the required information as -build parameters. Similarly, the DRY_RUN check box must be unchecked -if the job needs to be tested while skipping repository promotion to -Nexus. +build parameters. Similarly, the user must uncheck the DRY_RUN check +box to test the job while skipping repository promotion to Nexus. The special parameters are as follows:: @@ -42,6 +41,31 @@ The special parameters are as follows:: In words, the directory name can be ".releases" or "releases"; the file name can be anything with suffix ".yaml". +The JSON schema for a maven release job appears below. + +.. code-block:: none + + --- + $schema: "http://json-schema.org/schema#" + $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-schema.yaml" + + required: + - "distribution_type" + - "log_dir" + - "project" + - "version" + + properties: + distribution_type: + type: "string" + log_dir: + type: "string" + project: + type: "string" + version: + type: "string" + + Example of a maven release file: .. code-block:: bash @@ -54,9 +78,47 @@ Example of a maven release file: log_dir: 'example-project-maven-stage-master/17/' -An example of a container release file appears below. The container_release_tag -string is applied to all released containers. The per-container version -strings are used to pull images from the container registry. +The JSON schema for a container release job appears below. + +.. code-block:: none + + --- + $schema: "http://json-schema.org/schema#" + $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-container-schema.yaml" + + required: + - "containers" + - "distribution_type" + - "project" + - "container_release_tag" + - "ref" + + properties: + containers: + type: "array" + properties: + name: + type: "string" + version: + type: "string" + additionalProperties: false + distribution_type: + type: "string" + project: + type: "string" + container_release_tag: + type: "string" + container_pull_registry" + type: "string" + container_push_registry" + type: "string" + ref: + type: "string" + + +An example of a container release file appears below. The job applies the +container_release_tag string to all released containers. The job uses the +per-container version strings to pull images from the container registry. .. code-block:: bash @@ -64,6 +126,8 @@ strings are used to pull images from the container registry. --- distribution_type: 'container' container_release_tag: '1.0.0' + container_pull_registry: 'nexus.onap.org:10003" + container_push_registry: 'nexus.onap.org:10002" project: 'test' containers: - name: test-backend @@ -74,7 +138,7 @@ strings are used to pull images from the container registry. .. note:: - Job should be appended under gerrit-maven-stage + Job should appear under gerrit-maven-stage Example of a terse Jenkins job to call the global-jjb macro: @@ -160,6 +224,12 @@ Jenkins configure -> Global properties -> Environment variables ``RELEASE_USERNAME = YOUR_RELEASE_USERNAME`` ``RELEASE_EMAIL = YOUR_RELEASE_EMAIL`` + +.. note:: + + These also need to be added to your global-vars-$SILO.sh + or they will be overwritten. + Jenkins configure -> Managed Files -> Add a New Config -> Custom File id: signing-pubkey @@ -186,7 +256,7 @@ Ci-management Upgrade your project's global-jjb if needed, then add the following to your global defaults file (e.g., jjb/defaults.yaml). -.. code-block:: bash +.. code-block:: none jenkins-ssh-release-credential: 'jenkins-release' diff --git a/docs/jjb/lf-rtd-jobs.rst b/docs/jjb/lf-rtd-jobs.rst index c983b58c..393d0917 100644 --- a/docs/jjb/lf-rtd-jobs.rst +++ b/docs/jjb/lf-rtd-jobs.rst @@ -99,7 +99,7 @@ ReadTheDocs. To do that follow these steps: ReadTheDocs Verify ------------------ -Verify job which runs a tox build of the docs project +Verify job which runs tox to test the docs project :Template Names: - {project-name}-rtd-verify-{stream} @@ -125,6 +125,7 @@ Verify job which runs a tox build of the docs project :gerrit-skip-vote: Skip voting for this job. (default: false) :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) :project-pattern: Project to trigger build against. (default: \*\*) + :python-version: Python version (default: python2) :stream: Keyword representing a release code-name. Often the same as the branch. (default: master) :submodule-recursive: Whether to checkout submodules recursively. diff --git a/docs/jjb/lf-whitesource-jobs.rst b/docs/jjb/lf-whitesource-jobs.rst index 641cfc3f..9bee53f0 100644 --- a/docs/jjb/lf-whitesource-jobs.rst +++ b/docs/jjb/lf-whitesource-jobs.rst @@ -47,6 +47,7 @@ https://s3.amazonaws.com/unified-agent/wss-unified-agent.config :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) :build-timeout: Timeout in minutes before aborting build. (default: 60) :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :java-opts: Java options. Example: -Xmx1024m :java-version: Version of Java to use for the build. (default: openjdk8) :mvn-clean-install: Run maven clean install before the code scan. (default: false) :mvn-global-settings: The name of the Maven global settings to use for diff --git a/jjb/lf-ci-job-groups.yaml b/jjb/lf-ci-job-groups.yaml index 7b32f4da..81e6b0b9 100644 --- a/jjb/lf-ci-job-groups.yaml +++ b/jjb/lf-ci-job-groups.yaml @@ -46,3 +46,17 @@ jobs: - github-packer-merge - github-packer-verify + +- job-group: + name: "{project-name}-openstack-jobs" + + jobs: + - gerrit-openstack-update-cloud-image + - gerrit-openstack-cron + +- job-group: + name: "{project-name}-github-openstack-jobs" + + jobs: + - github-openstack-update-cloud-image + - github-openstack-cron diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 1633c61a..981e84dd 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -1418,6 +1418,7 @@ openstack: true openstack-cloud: vex + update-cloud-image: false ##################### # Job Configuration # @@ -1430,6 +1431,10 @@ branch: "{branch}" - lf-packer-parameters: packer-version: "{packer-version}" + - bool: + name: UPDATE_CLOUD_IMAGE + default: "{update-cloud-image}" + description: "Update new built image on the cloud." builders: - lf-infra-packer-build: @@ -1439,9 +1444,30 @@ packer-version: "{packer-version}" platform: "{platforms}" template: "{templates}" + update-cloud-image: "{update-cloud-image}" - description-setter: regexp: '(\s+.*)(ZZCI\s+.*\d+-\d+\.\d+)' description: 'Image: \2' + # - trigger-builds: + # - project: '{project-name}-openstack-update-cloud-image' + # block: false + # predefined-parameters: | + # GERRIT_BRANCH=$GERRIT_BRANCH + # GERRIT_PROJECT=$GERRIT_PROJECT + # GERRIT_REFSPEC=$GERRIT_REFSPEC + # NEW_IMAGE_NAME=$NEW_IMAGE_NAME + # property-file: variables.jenkins-trigger + # property-file-fail-on-missing: true + + publishers: + - lf-infra-publish + - trigger-parameterized-builds: + - project: "{project-name}-openstack-update-cloud-image" + condition: UNSTABLE_OR_BETTER + predefined-parameters: | + NEW_IMAGE_NAME=$NEW_IMAGE_NAME + property-file: variables.jenkins-trigger + fail-on-missing: true - job-template: name: "{project-name}-packer-merge-{platforms}-{templates}" @@ -1687,6 +1713,123 @@ white-list-target-branches: - "{branch}" +################################ +# Openstack Update Cloud Image # +################################ + +- lf_openstack_cron: &lf_openstack_update_cloud_image + name: lf-openstack-update-cloud-image + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-timeout: 10 + cron: "@monthly" + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + new-image-name: "all" + openstack-cloud: vex + stream: master + submodule-timeout: 10 + submodule-disable: false + update-cloud-image: false + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: "{build-node}" + concurrent: false + disabled: "{disable-job}" + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + parameters: + - lf-infra-parameters: + project: "{project}" + stream: "{stream}" + branch: "{branch}" + - string: + name: NEW_IMAGE_NAME + default: "{new-image-name}" + description: "Name of cloud image to update in Jenkins" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + # Listed after to override openstack-infra-wrappers clouds.yaml definition + - config-file-provider: + files: + - file-id: clouds-yaml + target: "$HOME/.config/openstack/clouds.yaml" + - file-id: npmrc + target: "$HOME/.npmrc" + - file-id: pipconf + target: "$HOME/.config/pip/pip.conf" + + builders: + - lf-infra-pre-build + - inject: + properties-content: | + OS_CLOUD={openstack-cloud} + - shell: | + #!/bin/bash + echo "Extract the image type for commit message" + # echo IMAGE_TYPE=${{NEW_IMAGE_NAME% -*}} >> image-type.txt + IMAGE_TYPE=$(echo ${{NEW_IMAGE_NAME}} | tr -d "\'\"\ " | awk -F- '{{ print $2 " " $3 " " $4 }}') + echo IMAGE_TYPE=${{IMAGE_TYPE}} >> image-type.txt + cat image-type.txt + - inject: + properties-file: "image-type.txt" + - lf-infra-update-packer-images + - lf-infra-push-gerrit-patch: + project: "{project}" + gerrit-user: "{gerrit-user}" + gerrit-host: "{gerrit-host}" + gerrit-topic: "{gerrit-topic}" + gerrit-commit-message: "Update cloud image $IMAGE_TYPE" + reviewers-email: "{reviewers-email}" + + publishers: + - lf-infra-publish + +- job-template: + name: "{project-name}-openstack-update-cloud-image" + id: gerrit-openstack-update-cloud-image + <<: *lf_openstack_update_cloud_image + + ###################### + # Default parameters # + ###################### + + git-url: "$GIT_URL/$GERRIT_PROJECT" + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: false + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - timed: "{obj:cron}" + ################# # Puppet Verify # ################# diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 01f95fe2..2dd65704 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -111,6 +111,7 @@ PACKER_PLATFORM={platform} PACKER_TEMPLATE={template} PACKER_VERSION={packer-version} + UPDATE_CLOUD_IMAGE={update-cloud-image} - shell: !include-raw-escape: - ../shell/packer-install.sh - ../shell/packer-build.sh @@ -143,6 +144,11 @@ - shell: !include-raw: - ../shell/packer-clear-credentials.sh +- builder: + name: lf-infra-update-packer-images + builders: + - shell: !include-raw: ../shell/update-cloud-images.sh + - builder: name: lf-infra-push-gerrit-patch builders: @@ -260,7 +266,9 @@ name: lf-rtd-verify builders: - inject: - properties-content: DOC_DIR={doc-dir} + properties-content: | + DOC_DIR={doc-dir} + PYTHON={python-version} - shell: !include-raw-escape: ../shell/tox-install.sh - shell: !include-raw-escape: ../shell/rtd-verify.sh diff --git a/jjb/lf-python-job-groups.yaml b/jjb/lf-python-job-groups.yaml index 388eaa3b..59066f93 100644 --- a/jjb/lf-python-job-groups.yaml +++ b/jjb/lf-python-job-groups.yaml @@ -8,6 +8,7 @@ jobs: - gerrit-python-xc-clm - gerrit-tox-verify + - gerrit-tox-merge - job-group: name: "{project-name}-github-python-jobs" @@ -18,3 +19,4 @@ jobs: - github-python-xc-clm - github-tox-verify + - github-tox-merge diff --git a/jjb/lf-python-jobs.yaml b/jjb/lf-python-jobs.yaml index 0fdb2a2d..451eb635 100644 --- a/jjb/lf-python-jobs.yaml +++ b/jjb/lf-python-jobs.yaml @@ -7,7 +7,7 @@ name: lf-infra-tox-install builders: - inject: - properties-content: "PYTHON_VERSION={python-version}" + properties-content: "PYTHON={python-version}" - shell: !include-raw-escape: ../shell/tox-install.sh - builder: @@ -377,11 +377,11 @@ included-regions: "{obj:github_included_regions}" ############## -# Tox Verify # +# Tox Common # ############## -- lf_tox_verify: &lf_tox_verify - name: lf-tox-verify +- lf_tox_common: &lf_tox_common + name: lf-tox-common ###################### # Default parameters # @@ -456,7 +456,7 @@ # Python projects typically use tox to run testing. name: "{project-name}-tox-verify-{stream}" id: gerrit-tox-verify - <<: *lf_tox_verify + <<: *lf_tox_common ###################### # Default parameters # @@ -508,7 +508,7 @@ # Python projects typically use tox to run testing. name: "{project-name}-tox-verify-{stream}" id: github-tox-verify - <<: *lf_tox_verify + <<: *lf_tox_common properties: - lf-infra-properties: @@ -537,3 +537,83 @@ white-list-target-branches: - "{branch}" included-regions: "{obj:github_included_regions}" + +- job-template: + # Run tox after merge of gerrit change set + name: "{project-name}-tox-merge-{stream}" + id: gerrit-tox-merge + <<: *lf_tox_common + + ###################### + # Default parameters # + ###################### + + gerrit_merge_triggers: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$' + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_merge_triggers}" + projects: + - project-compare-type: ANT + project-pattern: "{project}" + branches: + - branch-compare-type: ANT + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" + +- job-template: + # Run tox after merge of github pull request + name: "{project-name}-tox-merge-{stream}" + id: github-tox-merge + <<: *lf_tox_common + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "" + branch: "refs/heads/{branch}" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^remerge$" + only-trigger-phrase: true + status-context: "Tox Merge" + permit-all: true + github-hooks: true + org-list: + - "{github-org}" + white-list: "{obj:github_pr_whitelist}" + admin-list: "{obj:github_pr_admin_list}" + white-list-target-branches: + - "{branch}" + included-regions: "{obj:github_included_regions}" diff --git a/jjb/lf-rtd-jobs.yaml b/jjb/lf-rtd-jobs.yaml index bb78d2c5..8a8e3aa8 100644 --- a/jjb/lf-rtd-jobs.yaml +++ b/jjb/lf-rtd-jobs.yaml @@ -164,6 +164,7 @@ git-url: "$GIT_URL/$PROJECT" github-url: "https://github.com" project-pattern: "**" + python-version: python2 stream: master submodule-recursive: true submodule-timeout: 10 @@ -189,6 +190,7 @@ - lf-infra-pre-build - lf-rtd-verify: doc-dir: "{doc-dir}" + python-version: "{python-version}" - job-template: name: "{project-name}-rtd-verify-{stream}" @@ -207,7 +209,7 @@ submodule-recursive: "{submodule-recursive}" submodule-timeout: "{submodule-timeout}" submodule-disable: "{submodule-disable}" - choosing-strategy: default + choosing-strategy: gerrit triggers: - gerrit: diff --git a/jjb/lf-whitesource-jobs.yaml b/jjb/lf-whitesource-jobs.yaml index 1f267ff6..f0adbc4d 100644 --- a/jjb/lf-whitesource-jobs.yaml +++ b/jjb/lf-whitesource-jobs.yaml @@ -6,6 +6,8 @@ - builder: name: lf-infra-wss-mvn-clean-install builders: + - inject: + properties-content: JAVA_OPTS={java-opts} - conditional-step: condition-kind: boolean-expression condition-expression: "{mvn-clean-install}" @@ -86,10 +88,11 @@ branch: master build-days-to-keep: 30 build-timeout: 60 - cron: "@daily" + cron: "H H * * 7" disable-job: false git-url: "$GIT_URL/$PROJECT" github-url: "https://github.com" + java-opts: "" java-version: openjdk8 mvn-clean-install: false mvn-global-settings: global-settings @@ -100,7 +103,7 @@ submodule-timeout: 10 submodule-disable: false wss-unified-agent-opts: "" - wss-unified-agent-version: 19.7.1 + wss-unified-agent-version: 19.8.1 gerrit_trigger_file_paths: - compare-type: ANT @@ -143,6 +146,7 @@ - lf-update-java-alternatives: java-version: "{java-version}" - lf-infra-wss-mvn-clean-install: + java-opts: "{java-opts}" mvn-clean-install: "{mvn-clean-install}" mvn-version: "{mvn-version}" mvn-pom: "{mvn-pom}" diff --git a/releasenotes/notes/add-python-tox-merge-template-c965c8b9ab644b20.yaml b/releasenotes/notes/add-python-tox-merge-template-c965c8b9ab644b20.yaml new file mode 100644 index 00000000..94c30f1d --- /dev/null +++ b/releasenotes/notes/add-python-tox-merge-template-c965c8b9ab644b20.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Add python tox merge job templates for gerrit and github. + These templates are triggered by merge events to test + the Python code on the branch that received the merge. + Renamed tox-verify macro to tox-common. diff --git a/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml b/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml new file mode 100644 index 00000000..41327100 --- /dev/null +++ b/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml @@ -0,0 +1,33 @@ +--- +features: + - | + Add template to update OpenStack cloud images. + - | + This job finds and updates OpenStack cloud images on the ci-management + source repository. + - | + The job is triggered in two ways: + - | + 1. When a packer merge job completes, the new image name created is passed + down to the job. + 2. Manually trigger the job to update all images. + - | + When the job is triggered through an upstream packer merge job, this only + generates a change request for the new image built. + - | + When the job is triggered manually, this job finds the latest images on + OpenStack cloud and compares them with the images currently used in + the source ci-management source repository. If the compared images have + newer time stamps are **all** updated through a change request. + - | + This job requires a Jenkins configuration merge and verify job setup and + working on Jenkins. +upgrade: + - | + Packer merge jobs have a new build parameter when checked also updates the + cloud image. + - | + **lf-infra-packer-build** macro now requires 1 new variables to be passed. + - | + #. **update-cloud-image:** Set to true when images need to be updated on + Jenkins. diff --git a/releasenotes/notes/fix-import-sigul-key-release-904e6d1668a8db33.yaml b/releasenotes/notes/fix-import-sigul-key-release-904e6d1668a8db33.yaml new file mode 100644 index 00000000..c9ad3034 --- /dev/null +++ b/releasenotes/notes/fix-import-sigul-key-release-904e6d1668a8db33.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Import GPG signing key in release jobs before verifying Gerrit tag details. diff --git a/releasenotes/notes/fix-rtd-verify-12b5784f1c22b5d9.yaml b/releasenotes/notes/fix-rtd-verify-12b5784f1c22b5d9.yaml new file mode 100644 index 00000000..35a414d3 --- /dev/null +++ b/releasenotes/notes/fix-rtd-verify-12b5784f1c22b5d9.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Update choosing-strategy from default to gerrit to allow building with + specific refspec parameters and while not having to also set the branch to master. diff --git a/releasenotes/notes/pin-python-cinderclient-4.3.0-5c0a9cdf2cc5ca0d.yaml b/releasenotes/notes/pin-python-cinderclient-4.3.0-5c0a9cdf2cc5ca0d.yaml new file mode 100644 index 00000000..d5286523 --- /dev/null +++ b/releasenotes/notes/pin-python-cinderclient-4.3.0-5c0a9cdf2cc5ca0d.yaml @@ -0,0 +1,15 @@ +--- +fixes: + - | + Pin python-cinderclient to 4.3.0. + + A new version of python-cinderclient 5.0.0 is released which breaks the + openstack jobs. + - | + openstack --os-cloud vex limits show --absolute + Error: No module named v1.contrib + - | + Debug info shows: + File "/home/jenkins/.local/lib/python2.7/site-packages/openstackclient/volume/client.py", line 40, in make_client + from cinderclient.v1.contrib import list_extensions + ImportError: No module named v1.contrib diff --git a/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml b/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml new file mode 100644 index 00000000..7d6bb561 --- /dev/null +++ b/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml @@ -0,0 +1,7 @@ +--- +fix: + - | + Builders may have diffrent pyenv versions installed. + Programically pick the latest pyenv version. + Since we change pyenv version when building images, we do not know which + pyenv version are avaliable. diff --git a/releasenotes/notes/release-jobs-container-registries-ece8f67039621d58.yaml b/releasenotes/notes/release-jobs-container-registries-ece8f67039621d58.yaml new file mode 100644 index 00000000..30be210a --- /dev/null +++ b/releasenotes/notes/release-jobs-container-registries-ece8f67039621d58.yaml @@ -0,0 +1,13 @@ +--- +upgrade: + - | + CONTAINER_PULL_REGISTRY and CONTAINER_PUSH_REGISTRY need to be + defined in Jenkins global environment varaibles. +fixes: + - | + Update echo commands in release-job.sh for easy identification. + - | + Update release-container-schema to use CONTAINER_PULL_REGISTRY + and CONTAINER_PUSH_REGISTRY from Jenkins global variables. These + will be used to pull and push operations for self release containers. + Can be overwritten in the releases files. diff --git a/releasenotes/notes/run-whitesource-weekly-31486a47d44e4c21.yaml b/releasenotes/notes/run-whitesource-weekly-31486a47d44e4c21.yaml new file mode 100644 index 00000000..52de3db4 --- /dev/null +++ b/releasenotes/notes/run-whitesource-weekly-31486a47d44e4c21.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Run WhiteSource scan jobs weekly on Sunday. diff --git a/releasenotes/notes/tox-install-8864409ad7415d5d.yaml b/releasenotes/notes/tox-install-8864409ad7415d5d.yaml new file mode 100644 index 00000000..97239b54 --- /dev/null +++ b/releasenotes/notes/tox-install-8864409ad7415d5d.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Pip install pyenv from python2 should force more-itertools to 5.0.0 + In a fresh python2.7 venv "pip install pyenv" correctly pulls down + more-itertools [required: Any, installed: 5.0.0] + If for some reason a higher version is already installed this will downgrade + more-itertools to a py2 compatible version diff --git a/releasenotes/notes/tox-install-use-python-version-486621184e7aa090.yaml b/releasenotes/notes/tox-install-use-python-version-486621184e7aa090.yaml new file mode 100644 index 00000000..ecfbe33c --- /dev/null +++ b/releasenotes/notes/tox-install-use-python-version-486621184e7aa090.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Revise tox-install.sh script to invoke python using environment variable + PYTHON instead of hardcoding "python", and remove the pip --quiet flag. + Extend the RTD template to pass python-version, default python2. diff --git a/releasenotes/notes/tox-warnings.yaml b/releasenotes/notes/tox-warnings.yaml new file mode 100644 index 00000000..76cfdd1c --- /dev/null +++ b/releasenotes/notes/tox-warnings.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Fix warnings from tox (shellcheck). diff --git a/releasenotes/notes/update-ws-cli-agent-calls-540edfb9ec9afb9f.yaml b/releasenotes/notes/update-ws-cli-agent-calls-540edfb9ec9afb9f.yaml new file mode 100644 index 00000000..2e53f9a5 --- /dev/null +++ b/releasenotes/notes/update-ws-cli-agent-calls-540edfb9ec9afb9f.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Remove quotes from optional var WSS_UNIFIED_AGENT_OPTIONS. + When empty, the quotes will cause WS Unified Agent failures. + - | + Upgrade WS Unified Agent CLI to latest version 19.8.1 diff --git a/releasenotes/notes/whitesource-allow-java-opts-69e3da20d4ca7167.yaml b/releasenotes/notes/whitesource-allow-java-opts-69e3da20d4ca7167.yaml new file mode 100644 index 00000000..da989ced --- /dev/null +++ b/releasenotes/notes/whitesource-allow-java-opts-69e3da20d4ca7167.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Allow java-opts to be defined in WhiteSource scans. This + avoids java heap failures. diff --git a/schema/release-container-schema.yaml b/schema/release-container-schema.yaml index dccd7b39..e3b73583 100644 --- a/schema/release-container-schema.yaml +++ b/schema/release-container-schema.yaml @@ -33,5 +33,9 @@ properties: type: "string" container_release_tag: type: "string" + container_pull_registry: + type: "string" + container_push_registry: + type: "string" ref: type: "string" diff --git a/shell/check-info-votes.sh b/shell/check-info-votes.sh index 51ca172f..32384be4 100644 --- a/shell/check-info-votes.sh +++ b/shell/check-info-votes.sh @@ -25,7 +25,14 @@ if [ -d "/opt/pyenv" ]; then export PATH="$PYENV_ROOT/bin:$PATH" PYTHONPATH=$(pwd) export PYTHONPATH - pyenv local 3.6.4 + + latest_version=$(pyenv versions \ + | sed s,*,,g \ + | awk '/[0-9]+/{ print $1 }' \ + | sort --version-sort \ + | awk '/./{line=$0} END{print line}') + + pyenv local "$latest_version" export PYENV_VERSION="3.6.4" fi diff --git a/shell/gerrit-push-patch.sh b/shell/gerrit-push-patch.sh index b7065130..62f8e9be 100644 --- a/shell/gerrit-push-patch.sh +++ b/shell/gerrit-push-patch.sh @@ -42,7 +42,6 @@ pip install --quiet --upgrade "pip==9.0.3" setuptools pip install --quiet --upgrade git-review set -u # End git-review workaround - # Remove any leading or trailing quotes surrounding the strings # which can cause parse errors when passed as CLI options to commands PROJECT="$(echo "$PROJECT" | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")" diff --git a/shell/packer-build.sh b/shell/packer-build.sh index 1caca8e3..dfacdeb7 100644 --- a/shell/packer-build.sh +++ b/shell/packer-build.sh @@ -22,11 +22,11 @@ PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" mkdir -p "$PACKER_LOGS_DIR" export PATH="${WORKSPACE}/bin:$PATH" -cd packer || exit +cd packer # Prioritize the project's own version of vars if available platform_file="common-packer/vars/$PACKER_PLATFORM.json" -if [ -f "vars/$PACKER_PLATFORM.json" ]; then +if [[ -f "vars/$PACKER_PLATFORM.json" ]]; then platform_file="vars/$PACKER_PLATFORM.json" fi @@ -41,6 +41,21 @@ packer.io build -color=false \ -var-file="$platform_file" \ "templates/$PACKER_TEMPLATE.json" +# Extract image name from log and store value in the downstream job +if [[ ${UPDATE_CLOUD_IMAGE} ]]; then + + NEW_IMAGE_NAME=$(grep -P '(\s+.*image: )(ZZCI\s+.*\d+-\d+\.\d+)' "$PACKER_BUILD_LOG" \ + | awk -F': ' '{print $4}')\") + + echo NEW_IMAGE_NAME="$NEW_IMAGE_NAME" >> "$WORKSPACE/variables.prop" + echo "NEW_IMAGE_NAME: ${NEW_IMAGE_NAME}" + + # Copy variables.prop to variables.jenkins-trigger so that the end of build + # trigger can pick up the file as input for triggering downstream jobs. + # Dont tigger downstream job when UPDATE_CLOUD_IMAGE is set to 'false' + cp $WORKSPACE/variables.prop $WORKSPACE/variables.jenkins-trigger +fi + # Retrive the list of cloud providers mapfile -t clouds < <(jq -r '.builders[].name' "templates/$PACKER_TEMPLATE.json") diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh index 603fece4..be6d1810 100644 --- a/shell/python-tools-install.sh +++ b/shell/python-tools-install.sh @@ -45,6 +45,7 @@ else echo "Generating Requirements File" cat << 'EOF' > "$requirements_file" lftools[openstack]~=0.26.2 +python-cinderclient~=4.3.0 python-heatclient~=1.16.1 python-openstackclient~=3.16.0 dogpile.cache~=0.6.8 # Version 0.7.[01] seems to break openstackclient diff --git a/shell/release-job.sh b/shell/release-job.sh index 8384455e..c27a87f8 100644 --- a/shell/release-job.sh +++ b/shell/release-job.sh @@ -13,216 +13,236 @@ set -eu -o pipefail #Python bits. Remove when centos 7.7 builder is avaliable. if [ -d "/opt/pyenv" ]; then - echo "---> Setting up pyenv" + echo "INFO: Setting up pyenv" export PYENV_ROOT="/opt/pyenv" export PATH="$PYENV_ROOT/bin:$PATH" fi PYTHONPATH=$(pwd) export PYTHONPATH -pyenv local 3.6.4 -export PYENV_VERSION="3.6.4" + +latest_version=$(pyenv versions \ + | sed s,*,,g \ + | awk '/[0-9]+/{ print $1 }' \ + | sort --version-sort \ + | awk '/./{line=$0} END{print line}') + +pyenv local "$latest_version" + +export PYENV_VERSION="$latest_version" pip install --user lftools[nexus] jsonschema niet yq #Functions. set_variables_common(){ -echo "---> INFO: Setting all common variables" -LOGS_SERVER="${LOGS_SERVER:-None}" -if [ "${LOGS_SERVER}" == 'None' ]; then - echo "FAILED: log server not found" - exit 1 -fi -NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/" -# Verify if using release file or parameters -if $USE_RELEASE_FILE ; then - release_files=$(git diff-tree --no-commit-id -r "$GERRIT_PATCHSET_REVISION" --name-only -- "releases/" ".releases/") - if (( $(grep -c . <<<"$release_files") > 1 )); then - echo "---> INFO: RELEASE FILES ARE AS FOLLOWS: $release_files" - echo "---> ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported." - exit 1 - else - release_file="$release_files" - echo "---> INFO: RELEASE FILE: $release_files" - fi -else - echo "This job is built with parameters, no release file needed. Continuing..." - release_file="None" -fi + echo "INFO: Setting all common variables" + LOGS_SERVER="${LOGS_SERVER:-None}" + if [ "${LOGS_SERVER}" == 'None' ]; then + echo "ERROR: log server not found" + exit 1 + fi + NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/" + # Verify if using release file or parameters + if $USE_RELEASE_FILE ; then + release_files=$(git diff-tree --no-commit-id -r "$GERRIT_PATCHSET_REVISION" --name-only -- "releases/" ".releases/") + if (( $(grep -c . <<<"$release_files") > 1 )); then + echo "INFO: RELEASE FILES ARE AS FOLLOWS: $release_files" + echo "ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported." + exit 1 + else + release_file="$release_files" + echo "INFO: RELEASE FILE: $release_files" + fi + else + echo "INFO: This job is built with parameters, no release file needed. Continuing..." + release_file="None" + fi -DISTRIBUTION_TYPE="${DISTRIBUTION_TYPE:-None}" -if [[ $DISTRIBUTION_TYPE == "None" ]]; then - DISTRIBUTION_TYPE="$(niet ".distribution_type" "$release_file")" -fi + DISTRIBUTION_TYPE="${DISTRIBUTION_TYPE:-None}" + if [[ $DISTRIBUTION_TYPE == "None" ]]; then + DISTRIBUTION_TYPE="$(niet ".distribution_type" "$release_file")" + fi -PATCH_DIR="$(mktemp -d)" + PATCH_DIR="$(mktemp -d)" -# Displaying Release Information (Common variables) -echo "RELEASE ENVIRONMENT INFO:" -echo "RELEASE_FILE: $release_file" -echo "LOGS_SERVER: $LOGS_SERVER" -echo "NEXUS_PATH: $NEXUS_PATH" -echo "JENKINS_HOSTNAME: $JENKINS_HOSTNAME" -echo "SILO: $SILO" -echo "PROJECT: $PROJECT" -echo "PROJECT-DASHED: ${PROJECT//\//-}" -echo "DISTRIBUTION_TYPE: $DISTRIBUTION_TYPE" + # Displaying Release Information (Common variables) + printf "\t%-30s\n" RELEASE_ENVIRONMENT_INFO: + printf "\t%-30s %s\n" RELEASE_FILE: $release_file + printf "\t%-30s %s\n" LOGS_SERVER: $LOGS_SERVER + printf "\t%-30s %s\n" NEXUS_PATH: $NEXUS_PATH + printf "\t%-30s %s\n" JENKINS_HOSTNAME: $JENKINS_HOSTNAME + printf "\t%-30s %s\n" SILO: $SILO + printf "\t%-30s %s\n" PROJECT: $PROJECT + printf "\t%-30s %s\n" PROJECT-DASHED: ${PROJECT//\//-} + printf "\t%-30s %s\n" DISTRIBUTION_TYPE: $DISTRIBUTION_TYPE } set_variables_maven(){ -VERSION="${VERSION:-None}" -if [[ $VERSION == "None" ]]; then - VERSION="$(niet ".version" "$release_file")" -fi -LOG_DIR="${LOG_DIR:-None}" -if [[ $LOG_DIR == "None" ]]; then - LOG_DIR="$(niet ".log_dir" "$release_file")" -fi -LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}" -LOGS_URL=${LOGS_URL%/} # strip any trailing '/' + VERSION="${VERSION:-None}" + if [[ $VERSION == "None" ]]; then + VERSION="$(niet ".version" "$release_file")" + fi + LOG_DIR="${LOG_DIR:-None}" + if [[ $LOG_DIR == "None" ]]; then + LOG_DIR="$(niet ".log_dir" "$release_file")" + fi + LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}" + LOGS_URL=${LOGS_URL%/} # strip any trailing '/' -# Continuing displaying Release Information (Maven) -echo "RELEASE MAVEN INFO:" -echo "VERSION: $VERSION" -echo "LOG DIR: $LOG_DIR" -echo "LOGS URL: $LOGS_URL" + # Continuing displaying Release Information (Maven) + printf "\t%-30s\n" RELEASE_MAVEN_INFO: + printf "\t%-30s %s\n" VERSION: $VERSION + printf "\t%-30s %s\n" LOG DIR: $LOG_DIR + printf "\t%-30s %s\n" LOGS URL: $LOGS_URL } set_variables_container(){ -VERSION="${VERSION:-None}" -if [[ $VERSION == "None" ]]; then - VERSION="$(niet ".container_release_tag" "$release_file")" -fi - -ref="$(niet ".ref" "$release_file")" + VERSION="${VERSION:-None}" + if [[ $VERSION == "None" ]]; then + VERSION="$(niet ".container_release_tag" "$release_file")" + fi + if grep -q "container_pull_registry" "$release_file" ; then + CONTAINER_PULL_REGISTRY="$(niet ".container_pull_registry" "$release_file")" + fi + if grep -q "container_push_registry" "$release_file" ; then + CONTAINER_PUSH_REGISTRY="$(niet ".container_push_registry" "$release_file")" + fi + # Make sure both pull and push registries are defined + if [ -z ${CONTAINER_PULL_REGISTRY+x} ] || [ -z ${CONTAINER_PUSH_REGISTRY+x} ]; then + echo "ERROR: CONTAINER_PULL_REGISTRY and CONTAINER_PUSH_REGISTRY need to be defined" + exit 1 + fi + ref="$(niet ".ref" "$release_file")" -# Continuing displaying Release Information (Container) -echo "RELEASE CONTAINER INFO:" -echo "CONTAINER_RELEASE_TAG: $VERSION" -echo "GERRIT_REF_TO_TAG: $ref" + # Continuing displaying Release Information (Container) + printf "\t%-30s\n" RELEASE_CONTAINER_INFO: + printf "\t%-30s %s\n" CONTAINER_RELEASE_TAG: $VERSION + printf "\t%-30s %s\n" CONTAINER_PULL_REGISTRY: $CONTAINER_PULL_REGISTRY + printf "\t%-30s %s\n" CONTAINER_PUSH_REGISTRY: $CONTAINER_PUSH_REGISTRY + printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: $ref } verify_schema(){ - echo "---> INFO: Verifying $release_file schema." - lftools schema verify "$release_file" "$RELEASE_SCHEMA" + echo "INFO: Verifying $release_file schema." + lftools schema verify "$release_file" "$RELEASE_SCHEMA" } verify_version(){ - # Verify allowed versions - # Allowed versions are "v#.#.#" or "#.#.#" aka SemVer - allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$" - if [[ ! $VERSION =~ $allowed_version_regex ]]; then - echo "The version $VERSION is not a semantic valid version" - echo "Allowed versions are \"v#.#.#\" or \"#.#.#\" aka SemVer" - echo "See https://semver.org/ for more details on SemVer" - exit 1 - fi + # Verify allowed versions + # Allowed versions are "v#.#.#" or "#.#.#" aka SemVer + allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$" + if [[ ! $VERSION =~ $allowed_version_regex ]]; then + echo "INFO: The version $VERSION is not a semantic valid version" + echo "INFO: Allowed versions are \"v#.#.#\" or \"#.#.#\" aka SemVer" + echo "INFO: See https://semver.org/ for more details on SemVer" + exit 1 + fi } tag(){ - if git tag -v "$VERSION"; then - echo "---> OK: Repo already tagged $VERSION Continuting to release" - else - - echo "---> INFO: Repo has not yet been tagged $VERSION" - git tag -am "${PROJECT//\//-} $VERSION" "$VERSION" - sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" "$VERSION" < "$SIGUL_PASSWORD" - echo "Showing latest signature for $PROJECT:" + # Import public signing key gpg --import "$SIGNING_PUBKEY" - echo "git tag -v $VERSION" - git tag -v "$VERSION" - ########## Merge Part ############## - if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then - echo "--> INFO: Running merge" - gerrit_ssh=$(echo "$GERRIT_URL" | awk -F"/" '{print $3}') - git remote set-url origin ssh://"$RELEASE_USERNAME"@"$gerrit_ssh":29418/"$PROJECT" - git config user.name "$RELEASE_USERNAME" - git config user.email "$RELEASE_EMAIL" - echo -e "Host $gerrit_ssh\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config - chmod 600 ~/.ssh/config - git push origin "$VERSION" + if git tag -v "$VERSION"; then + echo "OK: Repo already tagged $VERSION Continuting to release" + else + echo "INFO: Repo has not yet been tagged $VERSION" + git tag -am "${PROJECT//\//-} $VERSION" "$VERSION" + sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" "$VERSION" < "$SIGUL_PASSWORD" + echo "INFO: Showing latest signature for $PROJECT:" + echo "INFO: git tag -v $VERSION" + git tag -v "$VERSION" + + ########## Merge Part ############## + if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then + echo "INFO: Running merge, pushing tag" + gerrit_ssh=$(echo "$GERRIT_URL" | awk -F"/" '{print $3}') + git remote set-url origin ssh://"$RELEASE_USERNAME"@"$gerrit_ssh":29418/"$PROJECT" + git config user.name "$RELEASE_USERNAME" + git config user.email "$RELEASE_EMAIL" + echo -e "Host $gerrit_ssh\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config + chmod 600 ~/.ssh/config + git push origin "$VERSION" + fi fi - fi } nexus_release(){ - for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do - # extract the domain name from URL - NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||') - echo "---> INFO: NEXUS_URL: $NEXUS_URL" - # extract the staging repo from URL - STAGING_REPO=${staging_url#*repositories/} - echo "Running Nexus Verify" - lftools nexus release -v --server https://"$NEXUS_URL" "$STAGING_REPO" - echo "Merge will run" - echo "lftools nexus release --server https://$NEXUS_URL $STAGING_REPO" - done - - #Run the loop twice, to catch errors on either nexus repo - if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do - NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||') - STAGING_REPO=${staging_url#*repositories/} - echo "Promoting $STAGING_REPO on $NEXUS_URL." - lftools nexus release --server https://"$NEXUS_URL" "$STAGING_REPO" + # extract the domain name from URL + NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||') + echo "INFO: NEXUS_URL: $NEXUS_URL" + # extract the staging repo from URL + STAGING_REPO=${staging_url#*repositories/} + echo "INFO: Running Nexus Verify" + lftools nexus release -v --server https://"$NEXUS_URL" "$STAGING_REPO" + echo "INFO: Merge will run:" + echo "lftools nexus release --server https://$NEXUS_URL $STAGING_REPO" done - fi + + #Run the loop twice, to catch errors on either nexus repo + if [[ "$JOB_NAME" =~ "merge" ]] && [[ "$DRY_RUN" = false ]]; then + for staging_url in $(zcat "$PATCH_DIR"/staging-repo.txt.gz | awk -e '{print $2}'); do + NEXUS_URL=$(echo "$staging_url" | sed -e 's|^[^/]*//||' -e 's|/.*$||') + STAGING_REPO=${staging_url#*repositories/} + echo "INFO: Promoting $STAGING_REPO on $NEXUS_URL." + lftools nexus release --server https://"$NEXUS_URL" "$STAGING_REPO" + done + fi } container_release_file(){ - echo "---> Processing container release" - local lfn_umbrella - lfn_umbrella="$(echo "$GERRIT_HOST" | awk -F"." '{print $2}')" - + echo "INFO: Processing container release" + local lfn_umbrella + lfn_umbrella="$(echo "$GERRIT_HOST" | awk -F"." '{print $2}')" - for namequoted in $(cat $release_file | yq '.containers[].name'); do - versionquoted=$(cat $release_file | yq ".containers[] |select(.name=="$namequoted") |.version") + for namequoted in $(cat $release_file | yq '.containers[].name'); do + versionquoted=$(cat $release_file | yq ".containers[] |select(.name==$namequoted) |.version") - #Remove extra yaml quotes - name="${namequoted#\"}" - name="${name%\"}" - version="${versionquoted#\"}" - version="${version%\"}" + #Remove extra yaml quotes + name="${namequoted#\"}" + name="${name%\"}" + version="${versionquoted#\"}" + version="${version%\"}" - echo "$name" - echo "$version" - echo "---> INFO: Merge will release $name $version as $VERSION" - #Pull from public, to see if we have already tagged this. - if docker pull "$DOCKER_REGISTRY":10002/"$lfn_umbrella"/"$name":"$VERSION"; then - echo "---> OK: $VERSION is already released for image $name, Continuing..." - else - echo "---> OK: $VERSION not found in releases, release will be prepared. Continuing..." - docker pull "$DOCKER_REGISTRY":10001/"$lfn_umbrella"/"$name":"$version" - container_image_id="$(docker images | grep $name | grep $version | awk '{print $3}')" - echo "---> INFO: Merge will run the following commands:" - echo "docker tag $container_image_id $DOCKER_REGISTRY:10002/$lfn_umbrella/$name:$VERSION" - echo "docker push $DOCKER_REGISTRY:10002/$lfn_umbrella/$name:$VERSION" - if [[ "$JOB_NAME" =~ "merge" ]]; then - docker tag "$container_image_id" "$DOCKER_REGISTRY":10002/"$lfn_umbrella"/"$name":"$VERSION" - docker push "$DOCKER_REGISTRY":10002/"$lfn_umbrella"/"$name":"$VERSION" - fi - echo "#########################" - fi - done + echo "$name" + echo "$version" + echo "INFO: Merge will release $name $version as $VERSION" + # Attempt to pull from releases registry to see if the image has been released. + if docker pull "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION"; then + echo "OK: $VERSION is already released for image $name, Continuing..." + else + echo "OK: $VERSION not found in releases, release will be prepared. Continuing..." + docker pull "$CONTAINER_PULL_REGISTRY"/"$lfn_umbrella"/"$name":"$version" + container_image_id="$(docker images | grep $name | grep $version | awk '{print $3}')" + echo "INFO: Merge will run the following commands:" + echo "docker tag $container_image_id $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION" + echo "docker push $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION" + if [[ "$JOB_NAME" =~ "merge" ]]; then + docker tag "$container_image_id" "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION" + docker push "$CONTAINER_PUSH_REGISTRY"/"$lfn_umbrella"/"$name":"$VERSION" + fi + echo "#########################" + fi + done - echo "---> INFO: Merge will tag ref: $ref" - git checkout "$ref" - tag + echo "INFO: Merge will tag ref: $ref" + git checkout "$ref" + tag } maven_release_file(){ - echo "---> INFO: wget -P $PATCH_DIR ${LOGS_URL}/staging-repo.txt.gz" - wget -P "$PATCH_DIR" "${LOGS_URL}/"staging-repo.txt.gz - pushd "$PATCH_DIR" - echo "---> INFO: wget ${LOGS_URL}/patches/{${PROJECT//\//-}.bundle,taglist.log.gz}" - wget "${LOGS_URL}"/patches/{"${PROJECT//\//-}".bundle,taglist.log.gz} - gunzip taglist.log.gz - cat "$PATCH_DIR"/taglist.log - popd - git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")" - git fetch "$PATCH_DIR/${PROJECT//\//-}.bundle" - git merge --ff-only FETCH_HEAD - nexus_release - tag + echo "INFO: wget -P $PATCH_DIR ${LOGS_URL}/staging-repo.txt.gz" + wget -P "$PATCH_DIR" "${LOGS_URL}/"staging-repo.txt.gz + pushd "$PATCH_DIR" + echo "INFO: wget ${LOGS_URL}/patches/{${PROJECT//\//-}.bundle,taglist.log.gz}" + wget "${LOGS_URL}"/patches/{"${PROJECT//\//-}".bundle,taglist.log.gz} + gunzip taglist.log.gz + cat "$PATCH_DIR"/taglist.log + popd + git checkout "$(awk '{print $NF}' "$PATCH_DIR/taglist.log")" + git fetch "$PATCH_DIR/${PROJECT//\//-}.bundle" + git merge --ff-only FETCH_HEAD + nexus_release + tag } echo "########### Start Script release-job.sh ###################################" @@ -235,25 +255,25 @@ echo "########### Start Script release-job.sh ################################## set_variables_common if [[ "$DISTRIBUTION_TYPE" == "maven" ]]; then - wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml - RELEASE_SCHEMA="release-schema.yaml" - if $USE_RELEASE_FILE ; then - verify_schema - fi - set_variables_maven - verify_version - maven_release_file + wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml + RELEASE_SCHEMA="release-schema.yaml" + if $USE_RELEASE_FILE ; then + verify_schema + fi + set_variables_maven + verify_version + maven_release_file elif [[ "$DISTRIBUTION_TYPE" == "container" ]]; then - wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-container-schema.yaml - RELEASE_SCHEMA="release-container-schema.yaml" - verify_schema - set_variables_container - verify_version - container_release_file + wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-container-schema.yaml + RELEASE_SCHEMA="release-container-schema.yaml" + verify_schema + set_variables_container + verify_version + container_release_file else - echo "---> ERROR: distribution_type: $DISTRIBUTION_TYPE not supported" - echo "Must be maven or container" - exit 1 + echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported" + echo "Must be maven or container" + exit 1 fi ########################################## diff --git a/shell/tox-install.sh b/shell/tox-install.sh index 4708e88c..3734a7f4 100644 --- a/shell/tox-install.sh +++ b/shell/tox-install.sh @@ -10,12 +10,18 @@ ############################################################################## echo "---> tox-install.sh" -# Ensure we fail the job if any steps fail. -# DO NOT set -u as virtualenv's activate script has unbound variables -set -e -o pipefail +# Ensure we fail the job if any steps fail or variables are missing. +# Use -x to show value of $PYTHON in output +set -eux -o pipefail # Tox version is pulled in through detox to mitigate version conflict -python -m pip install --user --quiet --upgrade tox-pyenv -echo "----> pip freeze" -pip freeze + +if [[ $PYTHON == "python2" ]]; then + $PYTHON -m pip install --user --quiet --upgrade tox-pyenv more-itertools~=5.0.0 +else + $PYTHON -m pip install --user --quiet --upgrade tox-pyenv +fi + + +$PYTHON -m pip freeze diff --git a/shell/update-cloud-images.sh b/shell/update-cloud-images.sh new file mode 100644 index 00000000..6b39c7cc --- /dev/null +++ b/shell/update-cloud-images.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Auto-update packer image{s} when the job is started manually or a single +# image passed by upstream packer merge job: +# 1. Get a list of image{s} from the releng/builder repository +# 2. Search openstack cloud for the latest image{s} available or use the image +# name passed down from the upstream job. +# 3. Compare the time stamps of the new image{s} with the image in use +# 4. Update the image{s} in the config files and yaml files +# 5. Push the change to Gerrit + +virtualenv "/tmp/v/openstack" +# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091 +source "/tmp/v/openstack/bin/activate" +pip install --upgrade --quiet "pip<10.0.0" setuptools +pip install --upgrade --quiet python-openstackclient +pip freeze + +set -e + +mkdir -p "$WORKSPACE/archives" +echo "List of images used on the source repository:" +grep -Er '(_system_image:|IMAGE_NAME)' \ +--exclude-dir="global-jjb" --exclude-dir="common-packer" \ +| grep ZZCI | awk -F: -e '{print $3}' \ +| grep '\S' | tr -d \'\" | sort -n | uniq \ +| tee "$WORKSPACE/archives/used_image_list.txt" + +while read -r line ; do + image_in_use="${line}" + + # get image type - ex: builder, docker, gbp etc + image_type="${line% -*}" + # Get the latest images available on the cloud, when $NEW_IMAGE_NAME env + # var is unset and update all images on Jenkins to the latest. + if [[ ${NEW_IMAGE_NAME} != all ]]; then + new_image=${NEW_IMAGE_NAME} + else + new_image=$(openstack image list --long -f value -c Name -c Protected \ + | grep "${image_type}.*False" | tail -n-1 | sed 's/ False//') + fi + [[ -z ${new_image} ]] && continue + + # strip the timestamp from the image name amd compare + new_image_isotime=${new_image##*- } + image_in_use_isotime=${image_in_use##*- } + # compare timestamps + if [[ ${new_image_isotime//[\-\.]/} -gt ${image_in_use_isotime//[\-\.]/} ]]; then + # generate a patch to be submited to Gerrit + echo "Update old image: ${image_in_use} with new image: ${new_image}" + grep -rlE '(_system_image:|IMAGE_NAME)' | xargs sed -i "s/${image_in_use}/${new_image}/" + # When the script is triggered by upstream packer-merge job + # update only the requested image and break the loop + [[ ${NEW_IMAGE_NAME} != all ]] && break + else + echo "No new image to update: ${new_image}" + fi +done < "$WORKSPACE/archives/used_image_list.txt" + +git remote -v +git status +git diff > "$WORKSPACE/archives/new-images-patchset.diff" +git add -u diff --git a/shell/whitesource-unified-agent-cli.sh b/shell/whitesource-unified-agent-cli.sh index ff6db6e0..94dd3670 100644 --- a/shell/whitesource-unified-agent-cli.sh +++ b/shell/whitesource-unified-agent-cli.sh @@ -21,7 +21,7 @@ jar_location="/tmp/wss-unified-agent-${WSS_UNIFIED_AGENT_VERSION}.jar" wss_unified_agent_url="https://s3.amazonaws.com/unified-agent/wss-unified-agent-${WSS_UNIFIED_AGENT_VERSION}.jar" wget -nv "${wss_unified_agent_url}" -O "${jar_location}" echo "---> Running WhiteSource Unified Agent CLI ..." -java -jar "${jar_location}" -c wss-unified-agent.config \ +java ${JAVA_OPTS:-} -jar "${jar_location}" -c wss-unified-agent.config \ -product "${WSS_PRODUCT_NAME}" -project "${WSS_PROJECT_NAME}" \ - -projectVersion "${GERRIT_BRANCH}" "${WSS_UNIFIED_AGENT_OPTIONS:-}" + -projectVersion "${GERRIT_BRANCH}" ${WSS_UNIFIED_AGENT_OPTIONS:-} rm "${jar_location}"