From: Aric Gardner Date: Wed, 25 Sep 2019 18:35:57 +0000 (+0000) Subject: Merge "Revert "Fix rtd verify job choosing strategy"" X-Git-Tag: v0.44.1~5 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=3dd1773c0ec86b6f24f5e31df7ea0d8650c3c092;hp=3521ffe2da5571752d1bdf5508c7f9de2d946b4b;p=releng%2Fglobal-jjb.git Merge "Revert "Fix rtd verify job choosing strategy"" --- diff --git a/.jjb-test/lf-ci-jobs.yaml b/.jjb-test/lf-ci-jobs.yaml index ed0ef468..baeced5b 100644 --- a/.jjb-test/lf-ci-jobs.yaml +++ b/.jjb-test/lf-ci-jobs.yaml @@ -66,6 +66,8 @@ - java-builder - mininet + update-cloud-images: false + - project: name: throttle-ci-jobs jobs: diff --git a/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml new file mode 100644 index 00000000..fd6eee8a --- /dev/null +++ b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml @@ -0,0 +1,16 @@ +--- +- project: + name: openstack-update-cloud-images-full-test + jobs: + - "gerrit-openstack-update-cloud-image" + + project: ciman + project-name: ciman-full + build-timeout: 10 + branch: master + archive-artifacts: "**/*.log" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + gerrit-user: "jenkins-user" + gerrit-host: "git.example.org" + gerrit-topic: "update-cloud-image" + reviewers-email: "jenkins-user@example.org" diff --git a/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml new file mode 100644 index 00000000..9f536002 --- /dev/null +++ b/.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml @@ -0,0 +1,11 @@ +--- +- project: + name: openstack-update-cloud-images-minimal-test + jobs: + - "gerrit-openstack-update-cloud-image" + + project-name: ciman-minimal + gerrit-user: "jenkins-user" + gerrit-host: "git.example.org" + gerrit-topic: "update-cloud-image" + reviewers-email: "jenkins-user@example.org" diff --git a/.jjb-test/lf-python-jobs.yaml b/.jjb-test/lf-python-jobs.yaml index 6a10b8f5..dd94c646 100644 --- a/.jjb-test/lf-python-jobs.yaml +++ b/.jjb-test/lf-python-jobs.yaml @@ -4,6 +4,9 @@ jobs: - "{project-name}-python-jobs" - gerrit-tox-sonar + - gerrit-pypi-merge + - gerrit-pypi-release-verify + - gerrit-pypi-release-merge project-name: gerrit-python @@ -12,6 +15,9 @@ jobs: - "{project-name}-github-python-jobs" - github-tox-sonar + - github-pypi-merge + - github-pypi-release-verify + - github-pypi-release-merge project-name: github-python diff --git a/docs/conf.py b/docs/conf.py index 4457e120..ca63745c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -74,7 +74,7 @@ master_doc = 'index' # General information about the project. project = 'lf-releng-global-jjb' -copyright = '2017, The Linux Foundation' +copyright = '2019, The Linux Foundation' author = 'Linux Foundation Releng' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst index 9883dd7a..20dde4ef 100644 --- a/docs/jjb/lf-ci-jobs.rst +++ b/docs/jjb/lf-ci-jobs.rst @@ -650,6 +650,71 @@ Full Example: .. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-cron-full.yaml +.. _gjjb-openstack-update-cloud-image: + +OpenStack Update Cloud Image +---------------------------- + +This job finds and updates OpenStack cloud images on the ci-management source +repository. + +The job is triggered in two ways: + +1. When packer merge job completes, the new image name created is passed + down to the job. +2. When the job is triggered manually to update all new images. + +When the job is triggered through an upstream packer merge job, this only +generates a change request for the new image built. + +When the job is triggered manually, this job finds the latest images on +OpenStack cloud and compares them with the images currently used in the source +ci-management source repository. If the compared images have newer +time stamps are **all** updated through a change request. + +This job requires a Jenkins configuration merge and verify job setup and +working on Jenkins. + +:Template Names: + - {project-name}-openstack-update-cloud-image + - gerrit-openstack-update-cloud-image + - github-openstack-update-cloud-image + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally should + be configured in defaults.yaml) + :new-image-name: Name of new image name passed from packer merge job or + set to 'all' to update all images. (default: all) + +:Optional parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 90) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :openstack-cloud: OS_CLOUD setting to pass to openstack client. + (default: vex) + :stream: Keyword that can be used to represent a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. (default: false) + +Minimal Example: + +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-update-cloud-image-minimal.yaml + +Full Example: + +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-update-cloud-image-full.yaml + .. _gjjb-packer-merge: @@ -700,6 +765,9 @@ Packer Merge job runs `packer build` to build system images in the cloud. (default: false) :gerrit_verify_triggers: Override Gerrit Triggers. + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. (default: false) + Test an in-progress patch ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -838,3 +906,33 @@ Plug-in configurations :sonar-properties: Sonar configuration properties. (default: "") :sonar-java-opts: JVM options. (default: "") :sonar-additional-args: Additional command line arguments. (default: "") + + +Sonar with Prescan +------------------ + +The same as the Sonar job above, except the caller also defines a builder +called ``lf-sonar-prescan``, in which they can put any builders that they want +to run prior to the Sonar scan. + +.. code-block:: yaml + + - builder: + name: lf-sonar-prescan + builders: + - shell: "# Pre-scan shell script" + +:Template Names: + + - {project-name}-sonar-prescan + - gerrit-sonar-prescan + - github-sonar-prescan + +:Required Parameters: + :lf-sonar-prescan: A builder that will run prior to the Sonar scan. + +:Optional Parameters: + :sonar-task: Sonar task to run. (default: "") + :sonar-properties: Sonar configuration properties. (default: "") + :sonar-java-opts: JVM options. (default: "") + :sonar-additional-args: Additional command line arguments. (default: "") diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index a653bc55..ff8c561a 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -126,6 +126,11 @@ Run `packer build` to build system images. :platform: Build platform as found in the vars directory. :template: Packer template to build as found in the templates directory. +:Optional parameters: + + :update-cloud-image: Submit a change request to update new built cloud + image to Jenkins. + lf-infra-packer-validate ------------------------ @@ -167,6 +172,13 @@ lf-infra-sysstat Retrieves system stats. +lf-infra-update-packer-images +----------------------------- + +Find and update the new built cloud image{s} in the ci-management source +repository. + + lf-jacoco-nojava-workaround --------------------------- @@ -289,6 +301,23 @@ Runs Jenkins SonarQube plug-in. Requires ``SonarQube Scanner for Jenkins`` +:Optional Parameters: + :sonar-task: Sonar task to run. (default: "") + :sonar-properties: Sonar configuration properties. (default: "") + :sonar-java-opts: JVM options. (default: "") + :sonar-additional-args: Additional command line arguments. (default: "") + +lf-infra-sonar-with-prescan +--------------------------- + +Runs Jenkins SonarQube plug-in after a pre-scan builder, which is defined by +the macro's caller. + +Requires ``SonarQube Scanner for Jenkins`` + +:Required Parameters: + :lf-sonar-prescan: A builder that will run prior to the Sonar scan. + :Optional Parameters: :sonar-task: Sonar task to run. (default: "") :sonar-properties: Sonar configuration properties. (default: "") diff --git a/docs/jjb/lf-python-jobs.rst b/docs/jjb/lf-python-jobs.rst index 175f0e26..bea44fc4 100644 --- a/docs/jjb/lf-python-jobs.rst +++ b/docs/jjb/lf-python-jobs.rst @@ -7,7 +7,7 @@ Job Groups .. include:: ../job-groups.rst -Below is a list of Maven job groups: +Below is a list of Python job groups: .. literalinclude:: ../../jjb/lf-python-job-groups.yaml :language: yaml @@ -19,31 +19,51 @@ Macros lf-infra-clm-python ------------------- -Run CLM scanning against a Python project. +Runs CLM scanning against a Python project. :Required Parameters: :clm-project-name: Project name in Nexus IQ to send results to. +lf-infra-pypi-tag-release +------------------------- + +Checks the format of the release version string and checks the git +repository for that tag. In a merge job, continues to tag the repository +and push the tag to the git server. Also installs supporting tools +including Sigul and lftools. Sigul requires a CentOS build node. + +lf-infra-pypi-upload +-------------------- + +Uploads distribution files from subdirectory "dist" to a PyPI repository +using a Python virtual enviroment to install required packages. The +Jenkins server must have a configuration file ".pypirc". + +:Required Parameters: + + :pypi-repo: PyPI repository key in .pypirc configuration file; + e.g., "staging" or "pypi". + lf-infra-tox-install -------------------- -Install Tox into a virtualenv. +Installs Tox into a virtualenv. :Required Parameters: - :python-version: Version of Python to install into the Tox virtualenv. - Eg. python2 / python3 + :python-version: Version of Python to invoke the pip install of the tox-pyenv + package that creates a virtual environment, either "python2" or "python3". -lf-tox-install --------------- +lf-infra-tox-run +---------------- -Runs a shell script that installs tox in a Python virtualenv. +Creates a Tox virtual environment and invokes tox. :Required Parameters: - :python-version: Base Python version to use in the virtualenv. For example - python2 or python3. + :parallel: Boolean. If true use detox (distributed tox); + else use regular tox. Job Templates @@ -55,9 +75,9 @@ Python XC CLM CLM scans for Python based repos. This job will call the Nexus IQ CLI directly to run the scans. -A new credential named "nexus-iq-xc-clm" needs to exist in the Jenkins credentials. -The credential should contain the username and password to access Nexus -IQ Server. +A new credential named "nexus-iq-xc-clm" needs to exist in the Jenkins +credentials. The credential should contain the username and password +to access Nexus IQ Server. :Template Names: @@ -97,30 +117,29 @@ IQ Server. :submodule-disable: Disable submodule checkout operation. (default: false) :gerrit_clm_triggers: Override Gerrit Triggers. - :gerrit_trigger_file_paths: Override file paths which used to filter which - file modifications will trigger a build. Refer to JJB documentation for - "file-path" details. + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit Python Sonar with Tox --------------------- -Sonar scans for Python based repos. This job invokes tox to run tests and -gather coverage statistics from the test results, then invokes Maven to -publish the results to a Sonar server. +Sonar scans for Python based repos. This job invokes tox to run tests +and gather coverage statistics from the test results, then invokes +Maven to publish the results to a Sonar server. -To get the Sonar coverage results, file tox.ini must exist and contain coverage -commands to run. +To get the Sonar coverage results, file tox.ini must exist and contain +coverage commands to run. -The coverage commands define the code that gets executed by the test suites. -Checking coverage does not guarantee that the tests execute properly, but it -identifies code that is not executed by any test. +The coverage commands define the code that gets executed by the test +suites. Checking coverage does not guarantee that the tests execute +properly, but it identifies code that is not executed by any test. -This job reuses the Sonar builder used in Java/Maven projects which runs maven -twice. The first invocation does nothing for Python projects, so the job uses -the goal 'validate' by default. The second invocation publishes results using -the goal 'sonar:sonar' by default. +This job reuses the Sonar builder used in Java/Maven projects which +runs maven twice. The first invocation does nothing for Python +projects, so the job uses the goal 'validate' by default. The second +invocation publishes results using the goal 'sonar:sonar' by default. For example: @@ -169,10 +188,13 @@ https://docs.sonarqube.org/display/PLUG/Python+Coverage+Results+Import :mvn-global-settings: The name of the Maven global settings to use for :mvn-goals: The Maven goal to run first. (default: validate) :mvn-version: Version of maven to use. (default: mvn35) + :parallel: Boolean indicator for tox to run tests in parallel or series. + (default: false, in series) :pre-build-script: Shell script to execute before the Sonar builder. For example, install prerequisites or move files to the repo root. - (default: a string with a comment) - :python-version: Python version (default: python2) + (default: a string with a shell comment) + :python-version: Python version to invoke pip install of tox-pyenv + (default: python2) :sonar-mvn-goal: The Maven goal to run the Sonar plugin. (default: sonar:sonar) :stream: Keyword used to represent a release code-name. Often the same as the branch. (default: master) @@ -183,19 +205,19 @@ https://docs.sonarqube.org/display/PLUG/Python+Coverage+Results+Import :submodule-disable: Disable submodule checkout operation. (default: false) :gerrit_sonar_triggers: Override Gerrit Triggers. - :gerrit_trigger_file_paths: Override file paths which used to filter which - file modifications will trigger a build. Refer to JJB documentation for - "file-path" details. + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit Tox Verify ---------- -Tox runner to verify a project on creation of a patch set. -This job is pyenv aware so if the image contains an installation of pyenv -at /opt/pyenv it will pick it up and run Python tests with the appropriate -Python versions. This job will set the following pyenv variables before running. +Tox runner to verify a project on creation of a patch set. This job +is pyenv aware so if the image contains an installation of pyenv at +/opt/pyenv it will pick it up and run Python tests with the +appropriate Python versions. This job will set the following pyenv +variables before running. .. code:: bash @@ -225,8 +247,10 @@ Python versions. This job will set the following pyenv variables before running. :pre-build-script: Shell script to execute before the Tox builder. For example, install prerequisites or move files to the repo root. (default: a string with a shell comment) - :python-version: Version of Python to configure as a base in virtualenv. - (default: python3) + :parallel: Boolean indicator for tox to run tests in parallel or series. + (default: false, in series) + :python-version: Python version to invoke pip install of tox-pyenv + (default: python2) :stream: Keyword representing a release code-name. Often the same as the branch. (default: master) :submodule-recursive: Whether to checkout submodules recursively. @@ -236,23 +260,23 @@ Python versions. This job will set the following pyenv variables before running. :submodule-disable: Disable submodule checkout operation. (default: false) :tox-dir: Directory containing the project's tox.ini relative to - the workspace. Empty works if tox.ini is at project root. - (default: '') + the workspace. The default uses tox.ini at the project root. + (default: '.') :tox-envs: Tox environments to run. If blank run everything described in tox.ini. (default: '') - :gerrit_trigger_file_paths: Override file paths used to filter which - file modifications will trigger a build. Refer to JJB documentation for - "file-path" details. + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit Tox Merge --------- -Tox runner to verify a project after merge of a patch set. -This job is pyenv aware so if the image contains an installation of pyenv -at /opt/pyenv it will pick it up and run Python tests with the appropriate -Python versions. This job will set the following pyenv variables before running. +Tox runner to verify a project after merge of a patch set. This job +is pyenv aware so if the image contains an installation of pyenv at +/opt/pyenv it will pick it up and run Python tests with the +appropriate Python versions. This job will set the following pyenv +variables before running. .. code:: bash @@ -282,7 +306,352 @@ Python versions. This job will set the following pyenv variables before running. :pre-build-script: Shell script to execute before the CLM builder. For example, install prerequisites or move files to the repo root. (default: a string with only a comment) - :python-version: Version of Python to configure as a base in virtualenv. + :python-version: Python version to invoke pip install of tox-pyenv + (default: python2) + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :tox-dir: Directory containing the project's tox.ini relative to + the workspace. The default uses tox.ini at the project root. + (default: '.') + :tox-envs: Tox environments to run. If blank run everything described + in tox.ini. (default: '') + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. + https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit + + +PyPI Verify +----------- + +Verifies a Python library project on creation of a patch set. Runs tox +then builds a source distribution and (optionally) a binary +distribution. The project repository must have a setup.py file with +configuration for packaging the component. + +The tox runner is pyenv aware so if the image contains an installation +of pyenv at /opt/pyenv it will pick it up and run Python tests with +the appropriate Python versions. The tox runner sets the following +pyenv variables before running. + +.. code:: bash + + export PYENV_ROOT="/opt/pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + +:Template Names: + + - {project-name}-pypi-verify-{stream} + - gerrit-pypi-verify + - github-pypi-verify + +:Comment Trigger: recheck + +:Required Parameters: + + :build-node: The node to run the build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + +:Optional Parameters: + + :branch: The branch to build against. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :dist-binary: Whether to build a binary wheel distribution. (default: true) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :parallel: Boolean indicator for tox to run tests in parallel or series. + (default: false, in series) + :pre-build-script: Shell script to execute before the tox builder. For + example, install system prerequisites. (default: a shell comment) + :python-version: Python version to invoke pip install of tox-pyenv + (default: python3) + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :tox-dir: Directory containing the project's tox.ini relative to + the workspace. The default uses tox.ini at the project root. + (default: '.') + :tox-envs: Tox environments to run. If blank run everything described + in tox.ini. (default: '') + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. + https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit + + +PyPI Merge +---------- + +Creates and uploads distribution files on merge of a patch set. Runs +tox, builds a source distribution and (optionally) a binary +distribution, and uploads the distribution(s) to a PyPI repository. +This job should be configured to use a staging PyPI repository like +testpypi.python.org, not a public release area like the global PyPI +repository. Like the verify job, this requires a setup.py file for +packaging the component. + +The tox runner is pyenv aware so if the image contains an installation +of pyenv at /opt/pyenv it will pick it up and run Python tests with +the appropriate Python versions. The tox runner sets the following +pyenv variables before running. + +.. code:: bash + + export PYENV_ROOT="/opt/pyenv" + export PATH="$PYENV_ROOT/bin:$PATH" + + +Requires a .pypirc configuration file in the Jenkins builder home +directory, an example appears next. + +.. code-block:: bash + + [distutils] # this tells distutils what package indexes you can push to + index-servers = + staging + pypi + + [staging] + repository: https://testpypi.python.org/pypi + username: your_username + password: your_password + + [pypi] + repository: https://pypi.python.org/pypi + username: your_username + password: your_password + + +:Template Names: + + - {project-name}-pypi-merge-{stream} + - gerrit-pypi-merge + - github-pypi-merge + +:Comment Trigger: pypi-remerge + +:Required Parameters: + + :build-node: The node to run the build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + +:Optional Parameters: + + :branch: The branch to build against. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :dist-binary: Whether to build a binary wheel distribution. (default: true) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :parallel: Boolean indicator for tox to run tests in parallel or series. + (default: false, in series) + :pre-build-script: Shell script to execute before the tox builder. For + example, install system prerequisites. (default: a shell comment) + :pypi-repo: Key for PyPI repository parameters in the .pypirc file. + Merge jobs should use a server like testpypi.python.org. (default: staging) + :python-version: Python version to invoke pip install of tox-pyenv + (default: python3) + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :tox-dir: Directory containing the project's tox.ini relative to + the workspace. The default uses tox.ini at the project root. + (default: '.') + :tox-envs: Tox environments to run. If blank run everything described + in tox.ini. (default: '') + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. + https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit + + +PyPI Release Verify +------------------- + +Verifies a Python library project on creation of a patch set with a +release yaml file. Runs tox, builds source and (optionally) binary +distributions, checks the format of the version string, checks that +the distribution file names contain the release version string, and +checks that no tag exists in the code repository for the release +version. + +To initiate the release process, create a releases/ or .releases/ +directory at the root of the project repository, add one release yaml +file to it, and submit a change set with that release yaml file. A +schema and and an example for the release yaml file appear below. The +version in the release yaml file must be a valid Semantic Versioning +(SemVer) string, matching either the pattern "v#.#.#" or "#.#.#" where +"#" is one or more digits. + +This job is similar to the PyPI verify job, but is only triggered by a +patch set with a release yaml file. + +The build node for PyPI release verify jobs must be CentOS, which +supports the sigul client for accessing a signing server. + +.. note:: + + The release file regex is: (releases\/.*\.yaml|\.releases\/.*\.yaml). + In words, the directory name can be ".releases" or "releases"; the file + name can be anything with suffix ".yaml". + +The JSON schema for a pypi release file appears below. + +.. code-block:: none + + --- + $schema: "http://json-schema.org/schema#" + $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-pypi-schema.yaml" + + required: + - "distribution_type" + - "project" + - "version" + + properties: + distribution_type: + type: "string" + project: + type: "string" + version: + type: "string" + + +An example of a pypi release file appears below. + +.. code-block:: none + + $ cat releases/1.0.0-pypi.yaml + --- + distribution_type: pypi + version: 1.0.0 + project: 'example-project' + + +:Template Names: + + - {project-name}-pypi-release-verify-{stream} + - gerrit-pypi-release-verify + - github-pypi-release-verify + +:Required Parameters: + + :build-node: The node to run build on, which must be Centos. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + +:Optional Parameters: + + :branch: The branch to build against. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :dist-binary: Whether to build a binary wheel distribution. (default: true) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :parallel: Boolean indicator for tox to run tests in parallel or series. + (default: false, in series) + :pre-build-script: Shell script to execute before the tox builder. + For example, install prerequisites or move files to the repo root. + (default: a string with a shell comment) + :pypi-repo: Key for PyPI repository parameters in the .pypirc file. + Release jobs should use a server like pypy.org. (default: pypi) + :python-version: Python version to invoke pip install of tox-pyenv + (default: python3) + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + :tox-dir: Directory containing the project's tox.ini relative to + the workspace. The default uses tox.ini at the project root. + (default: '.') + :tox-envs: Tox environments to run. If blank run everything described + in tox.ini. (default: '') + :use-release-file: Whether to use the release file. (default: true) + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. + https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit + + +PyPI Release Merge +------------------ + +Publishes a Python library on merge of a patch set with a release yaml +file. Runs tox, builds source and (optionally) binary distributions, +checks the format of the version string, checks that the distribution +file names contain the release version string, checks that no tag +exists in the code repository for the release version, tags the code +repository with the release version, pushes the tag to the git server, +and uploads distributions to a PyPI repository. + +This job is similar to the PyPI merge job, but is only triggered by +merge of a release yaml file and checks the version and tag before +uploading to a public repository such as PyPI. + +See the PyPI Release Verify job above for documentation of the release +yaml file format. + +The build node for PyPI release merge jobs must be CentOS, which +supports the sigul client for accessing a signing server. + +A Jenkins user can also trigger this release job via the "Build with +parameters" action, removing the need to merge a release yaml file. +The user must enter parameters in the same way as a release yaml file, +except for the special USE_RELEASE_FILE and DRY_RUN check boxes. The +user must uncheck the USE_RELEASE_FILE check box if the job should run +with a release file, while passing the required information as build +parameters. Similarly, the user must uncheck the DRY_RUN check box to +test the job while skipping upload of files to a repository. + +The special parameters are as follows:: + + VERSION = 1.0.0 + USE_RELEASE_FILE = false + DRY_RUN = false + +:Template Names: + + - {project-name}-pypi-release-merge-{stream} + - gerrit-pypi-release-merge + - github-pypi-release-merge + +:Required Parameters: + + :build-node: The node to run build on, which must be Centos. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + +:Optional Parameters: + + :branch: The branch to build against. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :dist-binary: Whether to build a binary wheel distribution. (default: true) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :parallel: Boolean indicator for tox to run tests in parallel or series. + (default: false, in series) + :pre-build-script: Shell script to execute before the tox builder. + For example, install prerequisites or move files to the repo root. + (default: a string with a shell comment) + :pypi-repo: Key for PyPI repository parameters in the .pypirc file. + Release jobs should use a server like pypy.org. (default: pypi) + :python-version: Python version to invoke pip install of tox-pyenv (default: python3) :stream: Keyword representing a release code-name. Often the same as the branch. (default: master) @@ -293,11 +662,11 @@ Python versions. This job will set the following pyenv variables before running. :submodule-disable: Disable submodule checkout operation. (default: false) :tox-dir: Directory containing the project's tox.ini relative to - the workspace. Empty works if tox.ini is at project root. - (default: '') + the workspace. The default uses tox.ini at the project root. + (default: '.') :tox-envs: Tox environments to run. If blank run everything described in tox.ini. (default: '') - :gerrit_trigger_file_paths: Override file paths used to filter which - file modifications will trigger a build. Refer to JJB documentation for - "file-path" details. + :use-release-file: Whether to use the release file. (default: true) + :gerrit_trigger_file_paths: Override file paths used to filter which file + modifications trigger a build. Refer to JJB documentation for "file-path" details. https://docs.openstack.org/infra/jenkins-job-builder/triggers.html#triggers.gerrit diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst index 4d9f4f1b..3df7f8d3 100644 --- a/docs/jjb/lf-release-jobs.rst +++ b/docs/jjb/lf-release-jobs.rst @@ -224,6 +224,12 @@ Jenkins configure -> Global properties -> Environment variables ``RELEASE_USERNAME = YOUR_RELEASE_USERNAME`` ``RELEASE_EMAIL = YOUR_RELEASE_EMAIL`` + +.. note:: + + Add these variables to your global-vars-$SILO.sh file or they will + be overwritten. + Jenkins configure -> Managed Files -> Add a New Config -> Custom File id: signing-pubkey diff --git a/docs/jjb/lf-whitesource-jobs.rst b/docs/jjb/lf-whitesource-jobs.rst index 641cfc3f..9bee53f0 100644 --- a/docs/jjb/lf-whitesource-jobs.rst +++ b/docs/jjb/lf-whitesource-jobs.rst @@ -47,6 +47,7 @@ https://s3.amazonaws.com/unified-agent/wss-unified-agent.config :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) :build-timeout: Timeout in minutes before aborting build. (default: 60) :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :java-opts: Java options. Example: -Xmx1024m :java-version: Version of Java to use for the build. (default: openjdk8) :mvn-clean-install: Run maven clean install before the code scan. (default: false) :mvn-global-settings: The name of the Maven global settings to use for diff --git a/jjb/lf-c-cpp-jobs.yaml b/jjb/lf-c-cpp-jobs.yaml index 72cbfd84..6840228b 100644 --- a/jjb/lf-c-cpp-jobs.yaml +++ b/jjb/lf-c-cpp-jobs.yaml @@ -99,7 +99,7 @@ server-name: "{gerrit-server-name}" trigger-on: - comment-added-contains-event: - comment-contains-value: '^Patch Set\s+\d+:\s+stage-release\s*$' + comment-contains-value: '^Patch Set\s+\d+:\s+run-sonar\s*$' projects: - project-compare-type: "ANT" project-pattern: "{project}" diff --git a/jjb/lf-ci-job-groups.yaml b/jjb/lf-ci-job-groups.yaml index 7b32f4da..81e6b0b9 100644 --- a/jjb/lf-ci-job-groups.yaml +++ b/jjb/lf-ci-job-groups.yaml @@ -46,3 +46,17 @@ jobs: - github-packer-merge - github-packer-verify + +- job-group: + name: "{project-name}-openstack-jobs" + + jobs: + - gerrit-openstack-update-cloud-image + - gerrit-openstack-cron + +- job-group: + name: "{project-name}-github-openstack-jobs" + + jobs: + - github-openstack-update-cloud-image + - github-openstack-cron diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index 06a10d2b..981e84dd 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -1418,6 +1418,7 @@ openstack: true openstack-cloud: vex + update-cloud-image: false ##################### # Job Configuration # @@ -1430,6 +1431,10 @@ branch: "{branch}" - lf-packer-parameters: packer-version: "{packer-version}" + - bool: + name: UPDATE_CLOUD_IMAGE + default: "{update-cloud-image}" + description: "Update new built image on the cloud." builders: - lf-infra-packer-build: @@ -1439,9 +1444,30 @@ packer-version: "{packer-version}" platform: "{platforms}" template: "{templates}" + update-cloud-image: "{update-cloud-image}" - description-setter: regexp: '(\s+.*)(ZZCI\s+.*\d+-\d+\.\d+)' description: 'Image: \2' + # - trigger-builds: + # - project: '{project-name}-openstack-update-cloud-image' + # block: false + # predefined-parameters: | + # GERRIT_BRANCH=$GERRIT_BRANCH + # GERRIT_PROJECT=$GERRIT_PROJECT + # GERRIT_REFSPEC=$GERRIT_REFSPEC + # NEW_IMAGE_NAME=$NEW_IMAGE_NAME + # property-file: variables.jenkins-trigger + # property-file-fail-on-missing: true + + publishers: + - lf-infra-publish + - trigger-parameterized-builds: + - project: "{project-name}-openstack-update-cloud-image" + condition: UNSTABLE_OR_BETTER + predefined-parameters: | + NEW_IMAGE_NAME=$NEW_IMAGE_NAME + property-file: variables.jenkins-trigger + fail-on-missing: true - job-template: name: "{project-name}-packer-merge-{platforms}-{templates}" @@ -1687,6 +1713,123 @@ white-list-target-branches: - "{branch}" +################################ +# Openstack Update Cloud Image # +################################ + +- lf_openstack_cron: &lf_openstack_update_cloud_image + name: lf-openstack-update-cloud-image + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-timeout: 10 + cron: "@monthly" + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + new-image-name: "all" + openstack-cloud: vex + stream: master + submodule-timeout: 10 + submodule-disable: false + update-cloud-image: false + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: "{build-node}" + concurrent: false + disabled: "{disable-job}" + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + + parameters: + - lf-infra-parameters: + project: "{project}" + stream: "{stream}" + branch: "{branch}" + - string: + name: NEW_IMAGE_NAME + default: "{new-image-name}" + description: "Name of cloud image to update in Jenkins" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + # Listed after to override openstack-infra-wrappers clouds.yaml definition + - config-file-provider: + files: + - file-id: clouds-yaml + target: "$HOME/.config/openstack/clouds.yaml" + - file-id: npmrc + target: "$HOME/.npmrc" + - file-id: pipconf + target: "$HOME/.config/pip/pip.conf" + + builders: + - lf-infra-pre-build + - inject: + properties-content: | + OS_CLOUD={openstack-cloud} + - shell: | + #!/bin/bash + echo "Extract the image type for commit message" + # echo IMAGE_TYPE=${{NEW_IMAGE_NAME% -*}} >> image-type.txt + IMAGE_TYPE=$(echo ${{NEW_IMAGE_NAME}} | tr -d "\'\"\ " | awk -F- '{{ print $2 " " $3 " " $4 }}') + echo IMAGE_TYPE=${{IMAGE_TYPE}} >> image-type.txt + cat image-type.txt + - inject: + properties-file: "image-type.txt" + - lf-infra-update-packer-images + - lf-infra-push-gerrit-patch: + project: "{project}" + gerrit-user: "{gerrit-user}" + gerrit-host: "{gerrit-host}" + gerrit-topic: "{gerrit-topic}" + gerrit-commit-message: "Update cloud image $IMAGE_TYPE" + reviewers-email: "{reviewers-email}" + + publishers: + - lf-infra-publish + +- job-template: + name: "{project-name}-openstack-update-cloud-image" + id: gerrit-openstack-update-cloud-image + <<: *lf_openstack_update_cloud_image + + ###################### + # Default parameters # + ###################### + + git-url: "$GIT_URL/$GERRIT_PROJECT" + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: false + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - timed: "{obj:cron}" + ################# # Puppet Verify # ################# @@ -1903,6 +2046,11 @@ default: "{archive-artifacts}" description: Artifacts to archive to the logs server. + publishers: + - lf-infra-publish + +- lf_sonar_builders: &lf_sonar_builders + name: lf-sonar-builders builders: - lf-infra-pre-build - lf-infra-sonar: @@ -1911,27 +2059,22 @@ sonar-java-opts: "{sonar-java-opts}" sonar-additional-args: "{sonar-additional-args}" - publishers: - - lf-infra-publish - -- job-template: - name: "{project-name}-sonar" - id: gerrit-sonar - <<: *lf_sonar_common - # yamllint disable-line rule:key-duplicates - - ###################### - # Default parameters # - ###################### +- lf_sonar_builders_prescan: &lf_sonar_builders_prescan + name: lf-sonar-builders-prescan + builders: + - lf-infra-pre-build + - lf-infra-sonar-with-prescan: + sonar-task: "{sonar-task}" + sonar-properties: "{sonar-properties}" + sonar-java-opts: "{sonar-java-opts}" + sonar-additional-args: "{sonar-additional-args}" +- lf_sonar_gerrit_common: &lf_sonar_gerrit_common + name: lf-sonar-gerrit-common gerrit_sonar_triggers: - comment-added-contains-event: comment-contains-value: '^Patch Set\s+\d+:\s+run-sonar\s*$' - ##################### - # Job Configuration # - ##################### - scm: - lf-infra-gerrit-scm: jenkins-ssh-credential: "{jenkins-ssh-credential}" @@ -1961,12 +2104,8 @@ unstable: true notbuilt: true -- job-template: - name: "{project-name}-sonar" - id: github-sonar - <<: *lf_sonar_common - # yamllint disable-line rule:key-duplicates - +- lf_sonar_github_common: &lf_sonar_github_common + name: lf-sonar-github-common properties: - lf-infra-properties: build-days-to-keep: "{build-days-to-keep}" @@ -1994,3 +2133,39 @@ white-list-target-branches: - "{branch}" included-regions: "{obj:github_included_regions}" + +- job-template: + name: "{project-name}-sonar" + id: gerrit-sonar + <<: *lf_sonar_common + # yamllint disable-line rule:key-duplicates + <<: *lf_sonar_builders + # yamllint disable-line rule:key-duplicates + <<: *lf_sonar_gerrit_common + +- job-template: + name: "{project-name}-sonar" + id: github-sonar + <<: *lf_sonar_common + # yamllint disable-line rule:key-duplicates + <<: *lf_sonar_builders + # yamllint disable-line rule:key-duplicates + <<: *lf_sonar_github_common + +- job-template: + name: "{project-name}-sonar-prescan" + id: gerrit-sonar-prescan + <<: *lf_sonar_common + # yamllint disable-line rule:key-duplicates + <<: *lf_sonar_builders_prescan + # yamllint disable-line rule:key-duplicates + <<: *lf_sonar_gerrit_common + +- job-template: + name: "{project-name}-sonar-prescan" + id: github-sonar-prescan + <<: *lf_sonar_common + # yamllint disable-line rule:key-duplicates + <<: *lf_sonar_builders_prescan + # yamllint disable-line rule:key-duplicates + <<: *lf_sonar_github_common diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 91fd16b5..c06f0ce2 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -76,6 +76,8 @@ # Ensure python-tools are installed in case job template does not # call the lf-infra-pre-build macro. - ../shell/python-tools-install.sh + - shell: !include-raw: + - ../shell/sudo-logs.sh - shell: !include-raw: - ../shell/logs-deploy.sh - shell: !include-raw: @@ -111,6 +113,7 @@ PACKER_PLATFORM={platform} PACKER_TEMPLATE={template} PACKER_VERSION={packer-version} + UPDATE_CLOUD_IMAGE={update-cloud-image} - shell: !include-raw-escape: - ../shell/packer-install.sh - ../shell/packer-build.sh @@ -143,6 +146,11 @@ - shell: !include-raw: - ../shell/packer-clear-credentials.sh +- builder: + name: lf-infra-update-packer-images + builders: + - shell: !include-raw: ../shell/update-cloud-images.sh + - builder: name: lf-infra-push-gerrit-patch builders: @@ -331,6 +339,20 @@ java-opts: "{sonar-java-opts}" additional-arguments: "{sonar-additional-args}" +- builder: + name: lf-infra-sonar-with-prescan + # Run a Sonar Jenkins Plugin + builders: + - lf-sonar-prescan # Must be defined by caller + - sonar: + sonar-name: Sonar + scanner-name: SonarQubeScanner + task: "{sonar-task}" + project: "sonar-project.properties" + properties: "{sonar-properties}" + java-opts: "{sonar-java-opts}" + additional-arguments: "{sonar-additional-args}" + ############## # PARAMETERS # ############## @@ -501,10 +523,13 @@ name: LOG_DIR default: "" description: "Log dir, example: project-maven-stage-master/17/" - - string: + - choice: name: DISTRIBUTION_TYPE - default: "" - description: "Set to maven for build with parametes" + choices: + - None + - container + - maven + description: "Set to maven for build with parameters" - bool: name: USE_RELEASE_FILE default: "{use-release-file}" diff --git a/jjb/lf-python-job-groups.yaml b/jjb/lf-python-job-groups.yaml index 59066f93..60ecc660 100644 --- a/jjb/lf-python-job-groups.yaml +++ b/jjb/lf-python-job-groups.yaml @@ -3,7 +3,7 @@ name: "{project-name}-python-jobs" # This job group contains all the recommended jobs that should be deployed - # for any project ci that is using Gerrit. + # for a Gerrit-based Python project to verify commits using tox. jobs: - gerrit-python-xc-clm @@ -14,9 +14,33 @@ name: "{project-name}-github-python-jobs" # This job group contains all the recommended jobs that should be deployed - # for any project ci that is using GitHub. + # for a Github-based Python project to verify commits using tox. jobs: - github-python-xc-clm - github-tox-verify - github-tox-merge + +- job-group: + name: "{project-name}-gerrit-pypi-jobs" + + # This job group contains all the recommended jobs that should be deployed for + # a Gerrit-based Python project to test, build and deploy a library to PyPI. + + jobs: + - gerrit-pypi-verify + - gerrit-pypi-merge + - gerrit-pypi-release-verify + - gerrit-pypi-release-merge + +- job-group: + name: "{project-name}-github-pypi-jobs" + + # This job group contains all the recommended jobs that should be deployed for + # a Github-based Python project to test, build and deploy a library to PyPI. + + jobs: + - github-pypi-verify + - github-pypi-merge + - github-pypi-release-verify + - github-pypi-release-merge diff --git a/jjb/lf-python-jobs.yaml b/jjb/lf-python-jobs.yaml index 451eb635..e28bc22b 100644 --- a/jjb/lf-python-jobs.yaml +++ b/jjb/lf-python-jobs.yaml @@ -3,6 +3,41 @@ # Macros # ########## +- builder: + name: lf-infra-clm-python + builders: + - inject: + properties-content: "CLM_PROJECT_NAME={clm-project-name}" + - shell: !include-raw-escape: ../shell/nexus-iq-cli.sh + +- builder: + name: lf-infra-pypi-tag-release + builders: + - config-file-provider: + files: + - file-id: sigul-config + variable: SIGUL_CONFIG + - file-id: sigul-password + variable: SIGUL_PASSWORD + - file-id: sigul-pki + variable: SIGUL_PKI + - file-id: signing-pubkey + variable: SIGNING_PUBKEY + - shell: !include-raw: ../shell/sigul-configuration.sh + - shell: !include-raw: ../shell/sigul-install.sh + - shell: !include-raw: ../shell/pypi-tag-release.sh + +- builder: + name: lf-infra-pypi-upload + builders: + - config-file-provider: + files: + - file-id: pypirc + target: "$HOME/.pypirc" + - inject: + properties-content: "REPOSITORY={pypi-repo}" + - shell: !include-raw-escape: ../shell/pypi-upload.sh + - builder: name: lf-infra-tox-install builders: @@ -11,12 +46,11 @@ - shell: !include-raw-escape: ../shell/tox-install.sh - builder: - name: lf-infra-clm-python + name: lf-infra-tox-run builders: - inject: - properties-content: "CLM_PROJECT_NAME={clm-project-name}" - - shell: !include-raw-escape: - - ../shell/nexus-iq-cli.sh + properties-content: "PARALLEL={parallel}" + - shell: !include-raw-escape: ../shell/tox-run.sh #################### # COMMON FUNCTIONS # @@ -229,8 +263,9 @@ mvn-goals: validate mvn-settings: "{mvn-settings}" mvn-version: mvn35 + parallel: true pre-build-script: "# pre-build script goes here" - python-version: python2 + python-version: python3 sonar-mvn-goal: "sonar:sonar" stream: master submodule-recursive: true @@ -280,7 +315,8 @@ - lf-infra-tox-install: python-version: "{python-version}" - shell: "{pre-build-script}" - - shell: !include-raw-escape: ../shell/tox-run.sh + - lf-infra-tox-run: + parallel: "{parallel}" - lf-provide-maven-settings: global-settings-file: "{mvn-global-settings}" settings-file: "{mvn-settings}" @@ -393,14 +429,14 @@ disable-job: false git-url: "$GIT_URL/$GERRIT_PROJECT" github-url: "https://github.com" - parallel: true + parallel: false pre-build-script: "# pre-build script goes here" - python-version: python2 + python-version: python3 stream: master submodule-recursive: true submodule-timeout: 10 submodule-disable: false - tox-dir: "" + tox-dir: "." tox-envs: "" gerrit_trigger_file_paths: @@ -432,10 +468,6 @@ - lf-infra-tox-parameters: tox-dir: "{tox-dir}" tox-envs: "{tox-envs}" - - bool: - name: PARALLEL - default: "{parallel}" - description: Tox test type used to configure serial or parallel testing. wrappers: - lf-infra-wrappers: @@ -447,7 +479,8 @@ - lf-infra-tox-install: python-version: "{python-version}" - shell: "{pre-build-script}" - - shell: !include-raw-escape: ../shell/tox-run.sh + - lf-infra-tox-run: + parallel: "{parallel}" publishers: - lf-infra-publish @@ -617,3 +650,446 @@ white-list-target-branches: - "{branch}" included-regions: "{obj:github_included_regions}" + +######## +# PyPI # +######## + +- lf_pypi_common: &lf_pypi_common + name: lf-pypi-common + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-timeout: 15 + disable-job: false + dist-binary: true + git-url: "$GIT_URL/$GERRIT_PROJECT" + github-url: "https://github.com" + parallel: false + pre-build-script: "# pre-build script goes here" + python-version: python3 + stream: master + submodule-recursive: true + submodule-timeout: 10 + submodule-disable: false + tox-dir: "." + tox-envs: "" + + gerrit_trigger_file_paths: + - compare-type: ANT + pattern: ".*" + + # github_included_regions MUST match gerrit_trigger_file_paths + github_included_regions: + - ".*" + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - lf-infra-tox-parameters: + tox-dir: "{tox-dir}" + tox-envs: "{tox-envs}" + - bool: + name: BUILD_BDIST_WHEEL + default: "{dist-binary}" + description: "Set to True to build a wheel" + +- lf_pypi_verify_builders: &lf_pypi_verify_builders + name: lf-pypi-verify-builders + + builders: + - lf-infra-pre-build + - lf-infra-tox-install: + python-version: "{python-version}" + - shell: "{pre-build-script}" + - lf-infra-tox-run: + parallel: "{parallel}" + - shell: !include-raw-escape: ../shell/pypi-dist-build.sh + +- lf_pypi_merge_builders: &lf_pypi_merge_builders + name: lf-pypi-merge-builders + + builders: + - lf-infra-pre-build + - lf-infra-tox-install: + python-version: "{python-version}" + - shell: "{pre-build-script}" + - lf-infra-tox-run: + parallel: "{parallel}" + - shell: !include-raw-escape: ../shell/pypi-dist-build.sh + - lf-infra-pypi-upload: + pypi-repo: "{pypi-repo}" + +- lf_pypi_release_verify_builders: &lf_pypi_release_verify_builders + name: lf-pypi-release-verify-builders + + builders: + - lf-infra-pre-build + - lf-infra-tox-install: + python-version: "{python-version}" + - shell: "{pre-build-script}" + - lf-infra-tox-run: + parallel: "{parallel}" + - shell: !include-raw-escape: ../shell/pypi-dist-build.sh + - lf-infra-pypi-tag-release + +- lf_pypi_release_merge_builders: &lf_pypi_release_merge_builders + name: lf-pypi-release-merge-builders + + builders: + - lf-infra-pre-build + - lf-infra-tox-install: + python-version: "{python-version}" + - shell: "{pre-build-script}" + - lf-infra-tox-run: + parallel: "{parallel}" + - shell: !include-raw-escape: ../shell/pypi-dist-build.sh + - lf-infra-pypi-tag-release + - lf-infra-pypi-upload: + pypi-repo: "{pypi-repo}" + +- job-template: + name: "{project-name}-pypi-verify-{stream}" + id: gerrit-pypi-verify + <<: *lf_python_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pypi_common + <<: *lf_pypi_verify_builders + + gerrit_verify_triggers: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$' + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_verify_triggers}" + projects: + - project-compare-type: ANT + project-pattern: "{project}" + branches: + - branch-compare-type: ANT + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" + +- job-template: + name: "{project-name}-pypi-verify-{stream}" + id: github-pypi-verify + <<: *lf_python_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pypi_common + <<: *lf_pypi_verify_builders + + properties: + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "+refs/pull/*:refs/remotes/origin/pr/*" + branch: "$sha1" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^(recheck|reverify)$" + only-trigger-phrase: false + status-context: "PyPI Verify" + permit-all: true + github-hooks: true + white-list-target-branches: + - "{branch}" + included-regions: "{obj:github_included_regions}" + +- job-template: + name: "{project-name}-pypi-merge-{stream}" + id: gerrit-pypi-merge + <<: *lf_python_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pypi_common + <<: *lf_pypi_merge_builders + + pypi-repo: staging + + gerrit_merge_triggers: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$' + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_merge_triggers}" + projects: + - project-compare-type: ANT + project-pattern: "{project}" + branches: + - branch-compare-type: ANT + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" + +- job-template: + name: "{project-name}-pypi-merge-{stream}" + id: github-pypi-merge + <<: *lf_python_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pypi_common + <<: *lf_pypi_merge_builders + + pypi-repo: staging + + properties: + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "" + branch: "refs/heads/{branch}" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^remerge$" + only-trigger-phrase: false + status-context: "Merge" + permit-all: true + github-hooks: true + org-list: + - "{github-org}" + white-list: "{obj:github_pr_whitelist}" + admin-list: "{obj:github_pr_admin_list}" + white-list-target-branches: + - "{branch}" + included-regions: "{obj:github_included_regions}" + +- lf_pypi_release_common: &lf_pypi_release_common + name: lf-pypi-release-common + + dist-binary: true + pypi-repo: pypi + use-release-file: true + + gerrit_trigger_file_paths: + - compare-type: REG_EXP + pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)' + + # github_included_regions MUST match gerrit_trigger_file_paths + github_included_regions: + - 'releases\/.*\.yaml' + - '.releases\/.*\.yaml' + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - lf-infra-tox-parameters: + tox-dir: "{tox-dir}" + tox-envs: "{tox-envs}" + - bool: + name: BUILD_BDIST_WHEEL + default: "{dist-binary}" + description: "Set to True to build a wheel" + - string: + name: VERSION + default: "" + description: "This is the version, example: 1.0.0" + - bool: + name: USE_RELEASE_FILE + default: "{use-release-file}" + description: "Set to False for job built with parameters" + - bool: + name: DRY_RUN + default: false + description: | + If DRY_RUN is enabled artifacts are not published. + +- job-template: + name: "{project-name}-pypi-release-verify-{stream}" + id: gerrit-pypi-release-verify + <<: *lf_python_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pypi_common + <<: *lf_pypi_release_common + <<: *lf_pypi_release_verify_builders + + gerrit_verify_triggers: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$' + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_verify_triggers}" + projects: + - project-compare-type: "ANT" + project-pattern: "{project}" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**" + file-paths: "{obj:gerrit_trigger_file_paths}" + +- job-template: + name: "{project-name}-pypi-release-verify-{stream}" + id: github-pypi-release-verify + <<: *lf_python_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pypi_common + <<: *lf_pypi_release_common + <<: *lf_pypi_release_verify_builders + + properties: + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "" + branch: "refs/heads/{branch}" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^(recheck|reverify)$" + only-trigger-phrase: false + status-context: "PyPI Release Verify" + permit-all: true + github-hooks: true + white-list-target-branches: + - "{branch}" + included-regions: "{obj:github_included_regions}" + +- job-template: + name: "{project-name}-pypi-release-merge-{stream}" + id: gerrit-pypi-release-merge + <<: *lf_python_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pypi_common + <<: *lf_pypi_release_common + <<: *lf_pypi_release_merge_builders + + gerrit_release_triggers: + - change-merged-event + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_release_triggers}" + projects: + - project-compare-type: "ANT" + project-pattern: "{project}" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**" + file-paths: "{obj:gerrit_trigger_file_paths}" + +- job-template: + name: "{project-name}-pypi-release-merge-{stream}" + id: github-pypi-release-merge + <<: *lf_python_common + # yamllint disable-line rule:key-duplicates + <<: *lf_pypi_common + <<: *lf_pypi_release_common + <<: *lf_pypi_release_merge_builders + + properties: + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "" + branch: "refs/heads/{branch}" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^(remerge)$" + only-trigger-phrase: false + status-context: "PyPI Release Merge" + permit-all: true + github-hooks: true + white-list-target-branches: + - "{branch}" + included-regions: "{obj:github_included_regions}" diff --git a/jjb/lf-whitesource-jobs.yaml b/jjb/lf-whitesource-jobs.yaml index 418565d1..f0adbc4d 100644 --- a/jjb/lf-whitesource-jobs.yaml +++ b/jjb/lf-whitesource-jobs.yaml @@ -6,6 +6,8 @@ - builder: name: lf-infra-wss-mvn-clean-install builders: + - inject: + properties-content: JAVA_OPTS={java-opts} - conditional-step: condition-kind: boolean-expression condition-expression: "{mvn-clean-install}" @@ -86,10 +88,11 @@ branch: master build-days-to-keep: 30 build-timeout: 60 - cron: "@daily" + cron: "H H * * 7" disable-job: false git-url: "$GIT_URL/$PROJECT" github-url: "https://github.com" + java-opts: "" java-version: openjdk8 mvn-clean-install: false mvn-global-settings: global-settings @@ -143,6 +146,7 @@ - lf-update-java-alternatives: java-version: "{java-version}" - lf-infra-wss-mvn-clean-install: + java-opts: "{java-opts}" mvn-clean-install: "{mvn-clean-install}" mvn-version: "{mvn-version}" mvn-pom: "{mvn-pom}" diff --git a/releasenotes/notes/add-sonar-prescan-jobs-ae56dc8177dec9ab.yaml b/releasenotes/notes/add-sonar-prescan-jobs-ae56dc8177dec9ab.yaml new file mode 100644 index 00000000..39e8d74d --- /dev/null +++ b/releasenotes/notes/add-sonar-prescan-jobs-ae56dc8177dec9ab.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add an additonal Sonar job that allows the caller to provide a builder that + runs prior to the Sonar scan. diff --git a/releasenotes/notes/archive-sudo-logs-a9af4aff811feec5.yaml b/releasenotes/notes/archive-sudo-logs-a9af4aff811feec5.yaml new file mode 100644 index 00000000..d0be097b --- /dev/null +++ b/releasenotes/notes/archive-sudo-logs-a9af4aff811feec5.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + + Archive 'sudo' logs. The log will be located in the 'sudo' sub-directory of + the archive. The actual name of the log-file depends on the OS of the + builder. diff --git a/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml b/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml new file mode 100644 index 00000000..41327100 --- /dev/null +++ b/releasenotes/notes/auto-update-cloud-images-19a2241aca85abe3.yaml @@ -0,0 +1,33 @@ +--- +features: + - | + Add template to update OpenStack cloud images. + - | + This job finds and updates OpenStack cloud images on the ci-management + source repository. + - | + The job is triggered in two ways: + - | + 1. When a packer merge job completes, the new image name created is passed + down to the job. + 2. Manually trigger the job to update all images. + - | + When the job is triggered through an upstream packer merge job, this only + generates a change request for the new image built. + - | + When the job is triggered manually, this job finds the latest images on + OpenStack cloud and compares them with the images currently used in + the source ci-management source repository. If the compared images have + newer time stamps are **all** updated through a change request. + - | + This job requires a Jenkins configuration merge and verify job setup and + working on Jenkins. +upgrade: + - | + Packer merge jobs have a new build parameter when checked also updates the + cloud image. + - | + **lf-infra-packer-build** macro now requires 1 new variables to be passed. + - | + #. **update-cloud-image:** Set to true when images need to be updated on + Jenkins. diff --git a/releasenotes/notes/lf-pypi-merge-release-jobs-b729bc2331155364.yaml b/releasenotes/notes/lf-pypi-merge-release-jobs-b729bc2331155364.yaml new file mode 100644 index 00000000..34947a25 --- /dev/null +++ b/releasenotes/notes/lf-pypi-merge-release-jobs-b729bc2331155364.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + New templates to build and push Python source and binary distributions + to a PyPI server. Includes: + ``{project-name}-pypi-verify-{stream}``, ``gerrit-pypi-verify``, + ``github-pypi-verify``, + ``{project-name}-pypi-merge-{stream}``, ``gerrit-pypi-merge``, + ``github-pypi-merge``, + ``{project-name}-pypi-release-verify-{stream}``, + ``gerrit-pypi-release-verify``, ``github-pypi-release-verify``, + ``{project-name}-pypi-release-merge-{stream}``, + ``gerrit-pypi-release-merge``, ``github-pypi-release-merge``, diff --git a/releasenotes/notes/new_sonar_trigger-94d333307da7eb4f.yaml b/releasenotes/notes/new_sonar_trigger-94d333307da7eb4f.yaml new file mode 100644 index 00000000..b35dd881 --- /dev/null +++ b/releasenotes/notes/new_sonar_trigger-94d333307da7eb4f.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - Changed the trigger to run sonar from stage-release to run-sonar. + This makes it more concistent with the other parts. diff --git a/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml b/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml new file mode 100644 index 00000000..7d6bb561 --- /dev/null +++ b/releasenotes/notes/pyenv-6d86242e2a8be6eb.yaml @@ -0,0 +1,7 @@ +--- +fix: + - | + Builders may have diffrent pyenv versions installed. + Programically pick the latest pyenv version. + Since we change pyenv version when building images, we do not know which + pyenv version are avaliable. diff --git a/releasenotes/notes/run-whitesource-weekly-31486a47d44e4c21.yaml b/releasenotes/notes/run-whitesource-weekly-31486a47d44e4c21.yaml new file mode 100644 index 00000000..52de3db4 --- /dev/null +++ b/releasenotes/notes/run-whitesource-weekly-31486a47d44e4c21.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Run WhiteSource scan jobs weekly on Sunday. diff --git a/releasenotes/notes/tox-install-8864409ad7415d5d.yaml b/releasenotes/notes/tox-install-8864409ad7415d5d.yaml new file mode 100644 index 00000000..97239b54 --- /dev/null +++ b/releasenotes/notes/tox-install-8864409ad7415d5d.yaml @@ -0,0 +1,8 @@ +--- +fixes: + - | + Pip install pyenv from python2 should force more-itertools to 5.0.0 + In a fresh python2.7 venv "pip install pyenv" correctly pulls down + more-itertools [required: Any, installed: 5.0.0] + If for some reason a higher version is already installed this will downgrade + more-itertools to a py2 compatible version diff --git a/releasenotes/notes/whitesource-allow-java-opts-69e3da20d4ca7167.yaml b/releasenotes/notes/whitesource-allow-java-opts-69e3da20d4ca7167.yaml new file mode 100644 index 00000000..da989ced --- /dev/null +++ b/releasenotes/notes/whitesource-allow-java-opts-69e3da20d4ca7167.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Allow java-opts to be defined in WhiteSource scans. This + avoids java heap failures. diff --git a/schema/release-pypi-schema.yaml b/schema/release-pypi-schema.yaml new file mode 100644 index 00000000..e56eed69 --- /dev/null +++ b/schema/release-pypi-schema.yaml @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +--- +$schema: "http://json-schema.org/schema#" +$id: "https://github.com/lfit/releng-global-jjb/blob/master/release-pypi-schema.yaml" + +required: + - "distribution_type" + - "project" + - "version" + +properties: + distribution_type: + type: "string" + project: + type: "string" + version: + type: "string" diff --git a/shell/check-info-votes.sh b/shell/check-info-votes.sh index 51ca172f..32384be4 100644 --- a/shell/check-info-votes.sh +++ b/shell/check-info-votes.sh @@ -25,7 +25,14 @@ if [ -d "/opt/pyenv" ]; then export PATH="$PYENV_ROOT/bin:$PATH" PYTHONPATH=$(pwd) export PYTHONPATH - pyenv local 3.6.4 + + latest_version=$(pyenv versions \ + | sed s,*,,g \ + | awk '/[0-9]+/{ print $1 }' \ + | sort --version-sort \ + | awk '/./{line=$0} END{print line}') + + pyenv local "$latest_version" export PYENV_VERSION="3.6.4" fi diff --git a/shell/gerrit-push-patch.sh b/shell/gerrit-push-patch.sh index b7065130..62f8e9be 100644 --- a/shell/gerrit-push-patch.sh +++ b/shell/gerrit-push-patch.sh @@ -42,7 +42,6 @@ pip install --quiet --upgrade "pip==9.0.3" setuptools pip install --quiet --upgrade git-review set -u # End git-review workaround - # Remove any leading or trailing quotes surrounding the strings # which can cause parse errors when passed as CLI options to commands PROJECT="$(echo "$PROJECT" | sed "s/^\([\"']\)\(.*\)\1\$/\2/g")" diff --git a/shell/packer-build.sh b/shell/packer-build.sh index 1caca8e3..b015271c 100644 --- a/shell/packer-build.sh +++ b/shell/packer-build.sh @@ -22,11 +22,11 @@ PACKER_BUILD_LOG="$PACKER_LOGS_DIR/packer-build.log" mkdir -p "$PACKER_LOGS_DIR" export PATH="${WORKSPACE}/bin:$PATH" -cd packer || exit +cd packer # Prioritize the project's own version of vars if available platform_file="common-packer/vars/$PACKER_PLATFORM.json" -if [ -f "vars/$PACKER_PLATFORM.json" ]; then +if [[ -f "vars/$PACKER_PLATFORM.json" ]]; then platform_file="vars/$PACKER_PLATFORM.json" fi @@ -41,6 +41,21 @@ packer.io build -color=false \ -var-file="$platform_file" \ "templates/$PACKER_TEMPLATE.json" +# Extract image name from log and store value in the downstream job +if [[ ${UPDATE_CLOUD_IMAGE} ]]; then + + NEW_IMAGE_NAME=$(grep -P '(\s+.*image: )(ZZCI\s+.*\d+-\d+\.\d+)' \ + "$PACKER_BUILD_LOG" | awk -F': ' '{print $4}') + + echo NEW_IMAGE_NAME="$NEW_IMAGE_NAME" >> "$WORKSPACE/variables.prop" + echo "NEW_IMAGE_NAME: ${NEW_IMAGE_NAME}" + + # Copy variables.prop to variables.jenkins-trigger so that the end of build + # trigger can pick up the file as input for triggering downstream jobs. + # Dont tigger downstream job when UPDATE_CLOUD_IMAGE is set to 'false' + cp $WORKSPACE/variables.prop $WORKSPACE/variables.jenkins-trigger +fi + # Retrive the list of cloud providers mapfile -t clouds < <(jq -r '.builders[].name' "templates/$PACKER_TEMPLATE.json") diff --git a/shell/pypi-dist-build.sh b/shell/pypi-dist-build.sh new file mode 100644 index 00000000..eb10371b --- /dev/null +++ b/shell/pypi-dist-build.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> pypi-dist-build.sh" + +# Script to create Python source and binary distributions +# Requires project file "setup.py" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +virtualenv -p python3 /tmp/pypi +PATH=/tmp/pypi/bin:$PATH + +bdist="" +if $BUILD_BDIST_WHEEL; then + echo "INFO: installing wheel to build binary distribution" + pip install wheel + bdist="bdist_wheel" +fi + +echo "INFO: cd to tox-dir $TOX_DIR" +cd "$WORKSPACE/$TOX_DIR" +echo "INFO: creating distributions" +python3 setup.py sdist $bdist +echo "---> pypi-dist-build.sh ends" diff --git a/shell/pypi-tag-release.sh b/shell/pypi-tag-release.sh new file mode 100644 index 00000000..d920c54d --- /dev/null +++ b/shell/pypi-tag-release.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> pypi-tag-release.sh" + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +# Functions. + +set_variables(){ + echo "INFO: Setting variables" + # Verify if using release file or parameters + if $USE_RELEASE_FILE; then + echo "INFO: Checking number of release yaml files" + release_files=$(git diff-tree --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/") + if (( $(echo "$release_files" | wc -w) != 1 )); then + echo "ERROR: RELEASE FILES: $release_files" + echo "ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported." + exit 1 + else + release_file="$release_files" + echo "INFO: RELEASE FILE: $release_file" + fi + else + echo "INFO: This job is built with parameters, no release file" + release_file="None" + fi + + if [[ -z ${DISTRIBUTION_TYPE:-} ]]; then + echo "INFO: reading DISTRIBUTION_TYPE from file $release_file" + DISTRIBUTION_TYPE="$(niet ".distribution_type" "$release_file")" + fi + if [[ -z ${VERSION:-} ]]; then + echo "INFO: reading VERSION from file $release_file" + VERSION="$(niet ".version" "$release_file")" + fi + + # Display Release Information + printf "\t%-30s\n" RELEASE_ENVIRONMENT_INFO: + printf "\t%-30s %s\n" RELEASE_FILE: $release_file + printf "\t%-30s %s\n" JENKINS_HOSTNAME: $JENKINS_HOSTNAME + printf "\t%-30s %s\n" SILO: $SILO + printf "\t%-30s %s\n" PROJECT: $PROJECT + printf "\t%-30s %s\n" PROJECT-DASHED: ${PROJECT//\//-} + printf "\t%-30s %s\n" DISTRIBUTION_TYPE: $DISTRIBUTION_TYPE + printf "\t%-30s %s\n" VERSION: $VERSION +} + +# needs to run in the repository root +verify_schema(){ + echo "INFO: Fetching schema" + pypi_schema="release-pypi-schema.yaml" + wget https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${pypi_schema} + echo "INFO: Verifying $release_file against schema $pypi_schema" + lftools schema verify "$release_file" "$pypi_schema" + echo "INFO: $release_file passed schema verification" +} + +verify_version(){ + # Verify allowed patterns "v#.#.#" or "#.#.#" aka SemVer + echo "INFO: Verifying version string $VERSION" + allowed_version_regex="^((v?)([0-9]+)\.([0-9]+)\.([0-9]+))$" + if [[ $VERSION =~ $allowed_version_regex ]]; then + echo "INFO: The version $VERSION is a valid semantic version" + else + echo "ERROR: The version $VERSION is not a valid semantic version" + echo "ERROR: Allowed versions are \"v#.#.#\" or \"#.#.#\" aka SemVer" + echo "ERROR: See https://semver.org/ for more details on SemVer" + exit 1 + fi +} + +verify_dist(){ + # Verify all file names in dist folder have the expected version string + dir="$WORKSPACE/$TOX_DIR/dist" + echo "INFO: Listing files in $dir" + ls $dir + echo "INFO: Checking files in $dir for $VERSION" + if unex_files=$(find $dir | grep -v $VERSION | egrep -v "^$dir$"); then + echo "ERROR: found unexpected files: $unex_files" + exit 1 + else + echo "INFO: All file names have expected string ${VERSION}" + fi +} + +# TODO: how to tag Github? +tag_gerrit(){ + echo "INFO: Verifying tag $VERSION in repo" + # Import public signing key + gpg --import "$SIGNING_PUBKEY" + # Fail if tag exists + if git tag -v "$VERSION"; then + echo "ERROR: Repo already tagged" + exit 1 + else + echo "INFO: Repo has not yet been tagged" + fi + echo "INFO: Tagging repo" + git tag -am "${PROJECT//\//-} $VERSION" "$VERSION" + echo "INFO: Signing tag" + sigul --batch -c "$SIGUL_CONFIG" sign-git-tag "$SIGUL_KEY" "$VERSION" < "$SIGUL_PASSWORD" + echo "INFO: Verifying tag" + # may fail due to missing public key + if ! git tag -v "$VERSION"; then + echo "WARN: failed to verify tag, continuing anyhow" + fi + # The verify job also calls this script + if [[ ! $JOB_NAME =~ "merge" ]] ; then + echo "INFO: job is not a merge, skipping push" + else + echo "INFO: configuring Gerrit remote" + gerrit_ssh=$(echo "$GERRIT_URL" | awk -F"/" '{print $3}') + git remote set-url origin "ssh://$RELEASE_USERNAME@$gerrit_ssh:29418/$PROJECT" + git config user.name "$RELEASE_USERNAME" + git config user.email "$RELEASE_EMAIL" + echo -e "Host $gerrit_ssh\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config + chmod 600 ~/.ssh/config + if $DRY_RUN; then + echo "INFO: dry run, skipping push" + else + echo "INFO: pushing tag" + git push origin "$VERSION" + fi + fi +} + +# Main +virtualenv -p python3 /tmp/pypi +PATH=/tmp/pypi/bin:$PATH +pip install lftools jsonschema niet +set_variables +if [[ $DISTRIBUTION_TYPE != "pypi" ]]; then + echo "ERROR: unexpected distribution type $DISTRIBUTION_TYPE" + exit 1 +fi +if $USE_RELEASE_FILE; then + verify_schema +fi +verify_version +verify_dist +tag_gerrit +echo "---> pypi-tag-release.sh ends" diff --git a/shell/pypi-upload.sh b/shell/pypi-upload.sh new file mode 100644 index 00000000..e1ecfdfb --- /dev/null +++ b/shell/pypi-upload.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> pypi-upload.sh" + +# Script to publish Python distributions from a folder +# to the PyPI repository in $REPOSITORY which must be a +# key in the .pypirc file + +# Ensure we fail the job if any steps fail. +set -eu -o pipefail + +virtualenv -p python3 /tmp/pypi +PATH=/tmp/pypi/bin:$PATH + +pip install twine +echo "INFO: cd to tox-dir $TOX_DIR" +cd "$WORKSPACE/$TOX_DIR" +cmd="twine upload -r $REPOSITORY dist/*" +if $DRY_RUN; then + echo "INFO: dry-run is set, echoing command only" + echo $cmd +else + echo "INFO: uploading distributions" + $cmd +fi +echo "---> pypi-upload.sh ends" diff --git a/shell/python-tools-install.sh b/shell/python-tools-install.sh index be6d1810..1f5beec0 100644 --- a/shell/python-tools-install.sh +++ b/shell/python-tools-install.sh @@ -19,7 +19,7 @@ pip_list_pre=/tmp/pip-list-pre.txt pip_list_post=/tmp/pip-list-post.txt pip_list_diffs=/tmp/pip-list-diffs.txt if [[ -f $pip_list_pre ]]; then - pip list > $pip_list_post + python3 -m pip list > $pip_list_post echo "Compare pip packages before/after..." if diff --suppress-common-lines $pip_list_pre $pip_list_post \ | tee $pip_list_diffs; then @@ -33,7 +33,7 @@ if [[ -f $pip_list_pre ]]; then # log-deploy.sh script is 'appended' to this file and it would not # be executed. else - pip list > "$pip_list_pre" + python3 -m pip list > "$pip_list_pre" # These 'pip installs' only need to be executed during pre-build requirements_file=$(mktemp /tmp/requirements-XXXX.txt) @@ -44,20 +44,18 @@ else echo "Generating Requirements File" cat << 'EOF' > "$requirements_file" -lftools[openstack]~=0.26.2 -python-cinderclient~=4.3.0 -python-heatclient~=1.16.1 -python-openstackclient~=3.16.0 -dogpile.cache~=0.6.8 # Version 0.7.[01] seems to break openstackclient -more-itertools~=5.0.0 -niet~=1.4.2 # Extract values from yaml -tox>=3.7.0. # Tox 3.7 or greater is necessary for parallel mode support -yq~=2.7.2 +lftools[openstack] +python-heatclient +python-openstackclient +niet~=1.4.2 +tox>=3.7.0 # Tox 3.7 or greater is necessary for parallel mode support +yq EOF # Use `python -m pip` to ensure we are using the latest version of pip - python -m pip install --user --quiet --upgrade pip - python -m pip install --user --quiet --upgrade setuptools - python -m pip install --user --quiet --upgrade -r "$requirements_file" + python3 -m venv ~/.local + python3 -m pip install --user --quiet --upgrade pip + python3 -m pip install --user --quiet --upgrade setuptools + python3 -m pip install --user --quiet --upgrade -r "$requirements_file" rm -rf "$requirements_file" fi diff --git a/shell/release-job.sh b/shell/release-job.sh index 2ac368f6..b764ac19 100644 --- a/shell/release-job.sh +++ b/shell/release-job.sh @@ -11,17 +11,12 @@ echo "---> release-job.sh" set -eu -o pipefail -#Python bits. Remove when centos 7.7 builder is avaliable. -if [ -d "/opt/pyenv" ]; then - echo "INFO: Setting up pyenv" - export PYENV_ROOT="/opt/pyenv" - export PATH="$PYENV_ROOT/bin:$PATH" -fi -PYTHONPATH=$(pwd) -export PYTHONPATH -pyenv local 3.6.4 -export PYENV_VERSION="3.6.4" -pip install --user lftools[nexus] jsonschema niet yq +set +u +python3 -m venv /tmp/v/venv/ +# shellcheck disable=SC1091 +source /tmp/v/venv/bin/activate +set -u +python -m pip install lftools[nexus] jsonschema niet yq #Functions. diff --git a/shell/sudo-logs.sh b/shell/sudo-logs.sh new file mode 100755 index 00000000..d925434f --- /dev/null +++ b/shell/sudo-logs.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> sudo-logs.sh" + +set -eu -o pipefail -o noglob + +# Copy/Generate 'sudo' log and copy to archive directory +function copy_log() +{ + case $os in + fedora|centos|redhat|ubuntu|debian) + if ! sudo cp $sudo_log /tmp; then + echo "Unable to archive 'sudo' logs ($sudo_log)" + return + fi + ;; + suse) + # Do I need 'sudo' to run 'journalctl'? + journalctl | grep sudo > $sudo_log + ;; + *) echo "Unexpected 'operatingsystem': $os" + exit 1 + ;; + esac + sudo_log=$(basename $sudo_log) + sudo chown jenkins:jenkins /tmp/$sudo_log + chmod 0644 /tmp/$sudo_log + mkdir -p $WORKSPACE/archives/sudo + mv /tmp/$sudo_log $WORKSPACE/archives/sudo/$sudo_log + +} # End copy_log() + +echo "Archiving 'sudo' log.." +os=$(facter operatingsystem | tr '[:upper:]' '[:lower:]') +case $os in + fedora|centos|redhat) sudo_log=/var/log/secure ;; + ubuntu|debian) sudo_log=/var/log/auth.log ;; + suse) sudo_log=/tmp/sudo.log ;; + *) echo "Unexpected 'operatingsystem': $os" + exit 1 + ;; +esac + +copy_log diff --git a/shell/tox-install.sh b/shell/tox-install.sh index 25ff3ca5..3734a7f4 100644 --- a/shell/tox-install.sh +++ b/shell/tox-install.sh @@ -15,6 +15,13 @@ echo "---> tox-install.sh" set -eux -o pipefail # Tox version is pulled in through detox to mitigate version conflict -$PYTHON -m pip install --user --quiet --upgrade tox-pyenv + + +if [[ $PYTHON == "python2" ]]; then + $PYTHON -m pip install --user --quiet --upgrade tox-pyenv more-itertools~=5.0.0 +else + $PYTHON -m pip install --user --quiet --upgrade tox-pyenv +fi + $PYTHON -m pip freeze diff --git a/shell/tox-run.sh b/shell/tox-run.sh index c1598917..8be8459f 100644 --- a/shell/tox-run.sh +++ b/shell/tox-run.sh @@ -10,27 +10,19 @@ ############################################################################## echo "---> tox-run.sh" -# Ensure we fail the job if any steps fail. -# DO NOT set -u as virtualenv's activate script has unbound variables -set -e -o pipefail - ARCHIVE_TOX_DIR="$WORKSPACE/archives/tox" mkdir -p "$ARCHIVE_TOX_DIR" -cd "$WORKSPACE/$TOX_DIR" +cd "$WORKSPACE/$TOX_DIR" || exit 1 if [ -d "/opt/pyenv" ]; then echo "---> Setting up pyenv" export PYENV_ROOT="/opt/pyenv" export PATH="$PYENV_ROOT/bin:$PATH" + PYTHONPATH="$(pwd)" + export PYTHONPATH + export TOX_TESTENV_PASSENV=PYTHONPATH fi -# Set and pass in PYTHONPATH to circumvent installation bug in tox>=3.2.0 -PYTHONPATH=$(pwd) -export PYTHONPATH -export TOX_TESTENV_PASSENV=PYTHONPATH - -set +e # Allow detox to fail so that we can collect the logs in the next step - PARALLEL="${PARALLEL:-true}" if [ "${PARALLEL}" = true ]; then if [ -n "$TOX_ENVS" ]; then @@ -56,7 +48,6 @@ for i in .tox/*/log; do tox_env=$(echo "$i" | awk -F'/' '{print $2}') cp -r "$i" "$ARCHIVE_TOX_DIR/$tox_env" done -set -e # Logs collected so re-enable echo "Completed tox runs." diff --git a/shell/update-cloud-images.sh b/shell/update-cloud-images.sh new file mode 100644 index 00000000..6b39c7cc --- /dev/null +++ b/shell/update-cloud-images.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2019 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Auto-update packer image{s} when the job is started manually or a single +# image passed by upstream packer merge job: +# 1. Get a list of image{s} from the releng/builder repository +# 2. Search openstack cloud for the latest image{s} available or use the image +# name passed down from the upstream job. +# 3. Compare the time stamps of the new image{s} with the image in use +# 4. Update the image{s} in the config files and yaml files +# 5. Push the change to Gerrit + +virtualenv "/tmp/v/openstack" +# shellcheck source=/tmp/v/openstack/bin/activate disable=SC1091 +source "/tmp/v/openstack/bin/activate" +pip install --upgrade --quiet "pip<10.0.0" setuptools +pip install --upgrade --quiet python-openstackclient +pip freeze + +set -e + +mkdir -p "$WORKSPACE/archives" +echo "List of images used on the source repository:" +grep -Er '(_system_image:|IMAGE_NAME)' \ +--exclude-dir="global-jjb" --exclude-dir="common-packer" \ +| grep ZZCI | awk -F: -e '{print $3}' \ +| grep '\S' | tr -d \'\" | sort -n | uniq \ +| tee "$WORKSPACE/archives/used_image_list.txt" + +while read -r line ; do + image_in_use="${line}" + + # get image type - ex: builder, docker, gbp etc + image_type="${line% -*}" + # Get the latest images available on the cloud, when $NEW_IMAGE_NAME env + # var is unset and update all images on Jenkins to the latest. + if [[ ${NEW_IMAGE_NAME} != all ]]; then + new_image=${NEW_IMAGE_NAME} + else + new_image=$(openstack image list --long -f value -c Name -c Protected \ + | grep "${image_type}.*False" | tail -n-1 | sed 's/ False//') + fi + [[ -z ${new_image} ]] && continue + + # strip the timestamp from the image name amd compare + new_image_isotime=${new_image##*- } + image_in_use_isotime=${image_in_use##*- } + # compare timestamps + if [[ ${new_image_isotime//[\-\.]/} -gt ${image_in_use_isotime//[\-\.]/} ]]; then + # generate a patch to be submited to Gerrit + echo "Update old image: ${image_in_use} with new image: ${new_image}" + grep -rlE '(_system_image:|IMAGE_NAME)' | xargs sed -i "s/${image_in_use}/${new_image}/" + # When the script is triggered by upstream packer-merge job + # update only the requested image and break the loop + [[ ${NEW_IMAGE_NAME} != all ]] && break + else + echo "No new image to update: ${new_image}" + fi +done < "$WORKSPACE/archives/used_image_list.txt" + +git remote -v +git status +git diff > "$WORKSPACE/archives/new-images-patchset.diff" +git add -u diff --git a/shell/whitesource-unified-agent-cli.sh b/shell/whitesource-unified-agent-cli.sh index c2c8857d..94dd3670 100644 --- a/shell/whitesource-unified-agent-cli.sh +++ b/shell/whitesource-unified-agent-cli.sh @@ -21,7 +21,7 @@ jar_location="/tmp/wss-unified-agent-${WSS_UNIFIED_AGENT_VERSION}.jar" wss_unified_agent_url="https://s3.amazonaws.com/unified-agent/wss-unified-agent-${WSS_UNIFIED_AGENT_VERSION}.jar" wget -nv "${wss_unified_agent_url}" -O "${jar_location}" echo "---> Running WhiteSource Unified Agent CLI ..." -java -jar "${jar_location}" -c wss-unified-agent.config \ +java ${JAVA_OPTS:-} -jar "${jar_location}" -c wss-unified-agent.config \ -product "${WSS_PRODUCT_NAME}" -project "${WSS_PROJECT_NAME}" \ -projectVersion "${GERRIT_BRANCH}" ${WSS_UNIFIED_AGENT_OPTIONS:-} rm "${jar_location}"