│ └── tox.ini
│
├── releases/
- │ └── release-helloworld.yaml
+ │ └── pypi-helloworld.yaml
│
├── .gitignore
├── LICENSE
PyPI Release Files
------------------
-An example of a PyPI release file appears below.
+An example of a PyPI release file appears below. Name of the release file must
+start with "pypi". For example releases/pypi-1.0.0-mypackage.yaml
.. code-block:: none
- $ cat releases/pypi-release.yaml
+ $ cat releases/pypi-1.0.0-mypackage.yaml
---
- distribution_type: pypi
pypi_project: mypackage
python_version: '3.4'
version: 1.0.0
+ log_dir: example-project-pypi-merge-master/17
The following parameters must appear in the PyPI release yaml file.
:Required Parameters:
- :distribution_type: Must be "pypi".
:log_dir: The suffix of the logs URL reported on completion by the
Jenkins merge job that created and pushed the distribution files
to the staging repository. For example, use value
$id: "https://github.com/lfit/releng-global-jjb/blob/master/release-pypi-schema.yaml"
required:
- - "distribution_type"
- "log_dir"
- "pypi_project"
- "python_version"
- "version"
properties:
- distribution_type:
- type: "string"
log_dir:
type: "string"
pypi_project:
PackageCloud Release Files
--------------------------
-An example of a PackageCloud release file appears below.
+An example of a PackageCloud release file appears below. Name of release file
+must start with "packagecloud". For example releases/packagecloud-1.6-tree.yaml
.. code-block:: none
- $ cat releases/packagecloud-release.yaml
+ $ cat releases/packagecloud-1.6-tree.yaml
---
- distribution_type: packagecloud
package_name:
- - name: 'tree-1.6.0-10.el7.x86_64.rpm'
- - name: 'test.rpm'
+ - name: tree-1.6.0-10.el7.x86_64.rpm
+ - name: test.rpm
The following parameters must appear in the PackageCloud release yaml file.
These are not part of the Jenkins job definition to allow independent
:Required Parameters:
- :distribution_type: Must be "packagecloud".
:package_name: A list of names that specify the packages to promote.
- (Found via jenkins log when using gem to initally push package up eg.
+ (Found in jenkins console log when using gem to push package eg.
"Pushing /path/of/package/name-of-package.rpm... success!"
- OR using rest api call with generated token from packagecloud.io
+ OR using rest api call to query packagecloud.io repo
"curl https://packagecloud.io/api/v1/repos/test_user/test_repo/search?q=
| yq -r .[].filename"
required:
- "package_name"
- - "distribution_type"
properties:
package_name:
properties:
name:
type: "string"
- distribution_type:
- type: "string"
Jenkins Jobs
------------
file. Checks the format of the version string, downloads the package
artifacts from the PyPI staging repository, uploads the package
artifacts to the PyPI release repository, tags the git repository,
-signs the tag and pushes the tag to the git server. The release verify
+signs the tag and pushes the tag to the git server. The release merge
template accepts neither a branch nor a stream parameter.
:Template Names:
should be the repository pypy.org. (default: pypi)
:use-release-file: Whether to use the release file. (default: true)
+ :gerrit_trigger_file_paths: Override file paths filter which checks which
+ file modifications will trigger a build.
+ **default**::
+
+ - compare-type: REG_EXP
+ pattern: '(releases\/pypi.*\.yaml|\.releases\/pypi.*\.yaml)'
PyPI Release Verify
~~~~~~~~~~~~~~~~~~~
should be the repository pypy.org (default: pypi)
:use-release-file: Whether to use the release file. (default: true)
+ :gerrit_trigger_file_paths: Override file paths filter which checks which
+ file modifications will trigger a build.
+ **default**::
+
+ - compare-type: REG_EXP
+ pattern: '(releases\/pypi.*\.yaml|\.releases\/pypi.*\.yaml)'
PackageCloud Release Verify
~~~~~~~~~~~~~~~~~~~~~~~~~~~
**default**::
- compare-type: REG_EXP
- pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+ pattern: '(releases\/packagecloud.*\.yaml|\.releases\/packagecloud.*\.yaml)'
PackageCloud Release Merge
**default**::
- compare-type: reg_exp
- pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+ pattern: '(releases\/packagecloud.*\.yaml|\.releases\/packagecloud.*\.yaml)'
Setup for LFID, Nexus, Jenkins and Gerrit
builders:
- lf-infra-jjbini
- shell: !include-raw-escape:
- - ../shell/jjb-install.sh
- ../shell/jenkins-sandbox-cleanup.sh
- - ../shell/jjb-cleanup.sh
- job-template:
name: "{project-name}-jenkins-sandbox-cleanup"
# define once and use twice; jobs MUST NOT override
gerrit_release_trigger_file_paths:
- compare-type: REG_EXP
- pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+ pattern: '(releases\/pypi.*\.yaml|\.releases\/pypi.*\.yaml)'
# yamllint disable-line rule:line-length
# github_release_included_regions MUST match gerrit_release_trigger_file_paths
github_release_included_regions:
- - 'releases\/.*\.yaml'
- - '.releases\/.*\.yaml'
+ - 'releases\/pypi.*\.yaml'
+ - '.releases\/pypi.*\.yaml'
parameters:
- lf-infra-parameters:
- "{branch}"
included-regions: "{obj:github_release_included_regions}"
-###############################
-# PackageCloud RELEASE VERIFY #
-###############################
+########################
+# PackageCloud RELEASE #
+########################
-- lf_packagecloud_verify: &lf_packagecloud_verify
- name: lf-packagecloud-verify
+- lf_packagecloud_release_common: &lf_packagecloud_release_common
+ name: lf-packagecloud-release-common
######################
# Default parameters #
git-url: "$GIT_URL/$PROJECT"
gerrit-skip-vote: false
use-release-file: true
-
- gerrit_verify_triggers:
- - patchset-created-event:
- exclude-drafts: true
- exclude-trivial-rebase: false
- exclude-no-code-change: false
- - draft-published-event
- - comment-added-contains-event:
- # yamllint disable-line rule:line-length
- comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$'
+ account-name: "{account-name}"
+ source-repo: "{source-repo}"
#####################
# Job Configuration #
project: "{project}"
branch: "$GERRIT_BRANCH"
stream: "$GERRIT_BRANCH"
- - lf-build-with-parameters-maven-release:
- use-release-file: "{use-release-file}"
+ - string:
+ name: DISTRIBUTION_TYPE
+ default: "packagecloud"
+ description: "The Jenkins release job distribution type."
+ - bool:
+ name: DRY_RUN
+ default: false
+ description: |
+ If DRY_RUN is enabled artifacts are not promoted.
+ - bool:
+ name: USE_RELEASE_FILE
+ default: "{use-release-file}"
+ description: "Set to False for job built with parameters"
builders:
- lf-infra-pre-build
files:
- file-id: "packagecloud_api"
target: "$HOME/packagecloud_api"
+ - file-id: packagecloud-account
+ variable: ACCOUNT_NAME_FILE
- lf-release
- job-template:
name: "{project-name}-packagecloud-release-verify"
id: gerrit-packagecloud-release-verify
<<: *lf_release_common
- # yamllint disable-line rule:key-duplicates
- <<: *lf_packagecloud_verify
+ <<: *lf_packagecloud_release_common
wrappers:
- lf-infra-wrappers:
triggers:
- gerrit:
server-name: "{gerrit-server-name}"
- trigger-on: "{obj:gerrit_verify_triggers}"
+ trigger-on:
+ - patchset-created-event:
+ exclude-drafts: true
+ exclude-trivial-rebase: false
+ exclude-no-code-change: false
+ - draft-published-event
+ - comment-added-contains-event:
+ # yamllint disable-line rule:line-length
+ comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$'
projects:
- project-compare-type: "ANT"
project-pattern: "{project}"
branch-pattern: "**"
file-paths:
- compare-type: REG_EXP
- pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+ pattern: '(releases\/packagecloud.*\.yaml|\.releases\/packagecloud.*\.yaml)'
skip-vote:
successful: "{gerrit-skip-vote}"
failed: "{gerrit-skip-vote}"
unstable: "{gerrit-skip-vote}"
notbuilt: "{gerrit-skip-vote}"
-##############################
-# PackageCloud RELEASE MERGE #
-##############################
-
-- lf_packagecloud_merge: &lf_packagecloud_merge
- name: lf-packagecloud-merge
-
- ######################
- # Default parameters #
- ######################
-
- build-days-to-keep: 7
- build-timeout: 15
- disable-job: false
- git-url: "$GIT_URL/$PROJECT"
- use-release-file: true
-
- gerrit_merge_triggers:
- - change-merged-event
- - comment-added-contains-event:
- comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$'
-
- #####################
- # Job Configuration #
- #####################
-
- disabled: "{disable-job}"
-
- parameters:
- - lf-infra-parameters:
- project: "{project}"
- branch: "$GERRIT_BRANCH"
- stream: "$GERRIT_BRANCH"
- - lf-build-with-parameters-maven-release:
- use-release-file: "{use-release-file}"
- - bool:
- name: DRY_RUN
- default: false
- description: |
- If DRY_RUN is enabled artifacts are not published.
-
- builders:
- - lf-infra-pre-build
- - config-file-provider:
- files:
- - file-id: "packagecloud_api"
- target: "$HOME/packagecloud_api"
- - lf-release
-
- job-template:
name: "{project-name}-packagecloud-release-merge"
id: gerrit-packagecloud-release-merge
<<: *lf_release_common
- # yamllint disable-line rule:key-duplicates
- <<: *lf_packagecloud_merge
+ <<: *lf_packagecloud_release_common
wrappers:
- lf-infra-wrappers:
triggers:
- gerrit:
server-name: "{gerrit-server-name}"
- trigger-on: "{obj:gerrit_merge_triggers}"
+ trigger-on:
+ - change-merged-event
+ - comment-added-contains-event:
+ comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$'
projects:
- project-compare-type: "ANT"
project-pattern: "{project}"
branch-pattern: "**"
file-paths:
- compare-type: REG_EXP
- pattern: '(releases\/.*\.yaml|\.releases\/.*\.yaml)'
+ pattern: '(releases\/packagecloud.*\.yaml|\.releases\/packagecloud.*\.yaml)'
--- /dev/null
+---
+fixes:
+ - |
+ Extend logs-deploy.sh with echo statements to show config/parameter values.
--- /dev/null
+---
+other:
+ - |
+ Disable generation of pip package lists during builds: packages_end.txt.gz,
+ packages_start.txt.gz & packages_diff.txt.gz. They are no longer valid.
--- /dev/null
+---
+fixes:
+ - |
+ Fix grep OS_CLOUD terminates jenkins-verify-images.sh if not found.
--- /dev/null
+---
+prelude: >
+ A project may need more than one type of release job
+ this patch changes the trigger to match on release file name
+ for pypi and packagecloud release jobs.
+features:
+ - |
+ Release jobs now support non-ff merges.
+issues:
+ - |
+ Release jobs still trigger when a release file is deleted.
--- /dev/null
+---
+fixes:
+ - |
+ Removed dependency on 'user' venv created by python-tools-install.sh from
+ the sandbox cleanup job. It will call lf-acitivate-venv() instead. Removed
+ references to jjb-install.sh & jjb-cleanup.sh.
required:
- "package_name"
- - "distribution_type"
properties:
package_name:
properties:
name:
type: "string"
- distribution_type:
- type: "string"
$id: "https://github.com/lfit/releng-global-jjb/blob/master/release-pypi-schema.yaml"
required:
- - "distribution_type"
- "log_dir"
- "pypi_project"
- "python_version"
- "version"
properties:
- distribution_type:
- type: "string"
log_dir:
type: "string"
pypi_project:
# Deletes all jobs on a Jenkins Sandbox system.
echo "---> jenkins-sandbox-cleanup.sh"
-set -eux -o pipefail
+set -euf -o pipefail
-bash -c "/usr/bin/yes 2>/dev/null || true" | jenkins-jobs -s sandbox delete-all
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+lf-activate-venv jenkins-job-builder
+
+yes | jenkins-jobs -s sandbox delete-all
# Recreate the All default view.
-cat << EOF > all-view.yaml
+cat << 'EOF' > all-view.yaml
- view:
name: All
view-type: all
done
}
+echo "Verifying that cloud has a master configuration file"
for cloud in jenkins-config/clouds/openstack/*; do
- echo "Verifying that cloud has a master configuration file"
- # Verify that we have a cloud config file
- if [ -f "$cloud/cloud.cfg" ]; then
- # Get the OS_CLOUD variable from cloud config and export it
- os_cloud=$(grep ^OS_CLOUD= "$cloud/cloud.cfg" | cut -d'=' -f2)
- OS_CLOUD="${os_cloud:-vex}"
- export OS_CLOUD
-
- verify_images "$cloud"
+ if [[ -f $cloud/cloud.cfg ]]; then
+ # Get the OS_CLOUD variable from cloud config
+ if ! os_cloud=$(egrep "^OS_CLOUD=" "$cloud/cloud.cfg" | cut -d'=' -f2); then
+ os_cloud="vex"
+ fi
+ OS_CLOUD=$os_cloud verify_images "$cloud"
else
echo "ERROR: No cloud.cfg for $cloud"
error=true
else
nexus_url="${NEXUSPROXY:-$NEXUS_URL}"
nexus_path="${SILO}/${JENKINS_HOSTNAME}/${JOB_NAME}/${BUILD_NUMBER}"
+ echo "INFO: Nexus URL $nexus_url path $nexus_path"
if [[ -n ${ARCHIVE_ARTIFACTS:-} ]] ; then
# Handle multiple search extensions as separate values to '-p|--pattern'
for arg in $ARCHIVE_ARTIFACTS; do
pattern_opts+=("-p" "$arg")
done
+ echo "INFO: archiving workspace using pattern(s): $ARCHIVE_ARTIFACTS"
lftools deploy archives "${pattern_opts[@]}" \
"$nexus_url" "$nexus_path" "$WORKSPACE"
else
+ echo "INFO: archiving workspace"
lftools deploy archives "$nexus_url" "$nexus_path" "$WORKSPACE"
fi
+ echo "INFO: archiving logs"
lftools deploy logs "$nexus_url" "$nexus_path" "${BUILD_URL:-}"
echo "Build logs: <a href=\"$LOGS_SERVER/$nexus_path\">$LOGS_SERVER/$nexus_path</a>"
-#!/bin/bash -l
+#!/bin/bash
# SPDX-License-Identifier: EPL-1.0
##############################################################################
# Copyright (c) 2018 The Linux Foundation and others.
##############################################################################
echo "---> python-tools-install.sh"
-set -eu -o pipefail
-
-# Generate a list of 'pip' packages pre-build/post-build
-# During post-build, perform a diff on the two lists and copy files to archive directory
-echo "Listing pip packages"
-pip_list_pre=/tmp/pip-list-pre.txt
-pip_list_post=/tmp/pip-list-post.txt
-pip_list_diffs=/tmp/pip-list-diffs.txt
-if [[ -f $pip_list_pre ]]; then
- python3 -m pip list > $pip_list_post
- echo "Compare pip packages before/after..."
- if diff --suppress-common-lines $pip_list_pre $pip_list_post \
- | tee $pip_list_diffs; then
- echo "No diffs" | tee $pip_list_diffs
- fi
- mkdir -p "$WORKSPACE/archives"
- cp "$pip_list_pre" "$pip_list_post" "$pip_list_diffs" "$WORKSPACE/archives"
- rm -rf "$pip_list_pre" "$pip_list_post" "$pip_list_diffs"
- ls "$WORKSPACE/archives"
- # Would just like to 'exit 0' here but we can't because the
- # log-deploy.sh script is 'appended' to this file and it would not
- # be executed.
-else
- python3 -m pip list > "$pip_list_pre"
- # These 'pip installs' only need to be executed during pre-build
+set -eufo pipefail
+# This script will typically be called during pre-build & post-build.
+# Create the user venv during pre-build.
+if [[ ! -f /tmp/pre-build-complete ]]; then
requirements_file=$(mktemp /tmp/requirements-XXXX.txt)
# Note: To test lftools master branch change the lftools configuration below in
yq
EOF
- # Use `python -m pip` to ensure we are using the latest version of pip
+ # Use `python -m pip` to ensure we are using pip from user venv
python3 -m venv ~/.local
python3 -m pip install --user --quiet --upgrade pip
python3 -m pip install --user --quiet --upgrade setuptools
python3 -m pip install --user --quiet --upgrade --upgrade-strategy eager -r "$requirements_file"
rm -rf "$requirements_file"
+ touch /tmp/pre-build-complete
fi
NEXUS_PATH="${SILO}/${JENKINS_HOSTNAME}/"
# Verify if using release file or parameters
if $USE_RELEASE_FILE ; then
- release_files=$(git diff-tree --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/")
+ release_files=$(git diff-tree -m --no-commit-id -r "$GIT_COMMIT" --name-only -- "releases/" ".releases/")
if (( $(grep -c . <<<"$release_files") > 1 )); then
echo "INFO: RELEASE FILES ARE AS FOLLOWS: $release_files"
echo "ERROR: Committing multiple release files in the same commit OR rename/amend of existing files is not supported."
verify_schema(){
echo "INFO: Verifying $release_file schema."
- lftools schema verify "$release_file" "$RELEASE_SCHEMA"
+ lftools schema verify "$release_file" "$release_schema"
}
verify_version(){
}
packagecloud_verify(){
- echo "INFO: ---> Verifying $1 exists in staging..."
+ echo "INFO: Verifying $1 exists in staging..."
if [[ $1 == $(curl --netrc-file ~/packagecloud_api --silent \
- https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+ https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
| yq -r .[].filename) ]]; then
echo "INFO: $1 exists in staging!"
echo "INFO: Existing package location: https://packagecloud.io$(curl \
--netrc-file ~/packagecloud_api --silent \
- https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+ https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
| yq -r .[].package_html_url)"
else
- echo "ERROR: $package_name does not exist in staging"
+ echo "ERROR: $1 does not exist in staging"
exit 1
fi
}
packagecloud_promote(){
- echo "INFO: ---> Preparing to promote $1..."
+ echo "INFO: Preparing to promote $1..."
promote_url="https://packagecloud.io$(curl --netrc-file ~/packagecloud_api \
- --silent https://packagecloud.io/api/v1/repos/"$2"/search?q="$1" \
+ --silent https://packagecloud.io/api/v1/repos/"$2"/staging/search?q="$1" \
| yq -r .[].promote_url)"
echo "INFO: Promoting $1 from staging to release"
curl --netrc-file ~/packagecloud_api -X POST -F \
- destination=o-ran-sc/release "$promote_url" \
+ destination="$2/release" "$promote_url" \
| echo "INFO: Promoted package location: \
https://packagecloud.io$(yq -r .package_html_url)"
}
# - maven, release-schema.yaml
# - pypi, release-pypi-schema.yaml
-if [[ "$DISTRIBUTION_TYPE" == "maven" ]]; then
- if $USE_RELEASE_FILE ; then
- RELEASE_SCHEMA="release-schema.yaml"
- echo "INFO: Fetching schema $RELEASE_SCHEMA"
- wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/release-schema.yaml
- verify_schema
- fi
- set_variables_maven
- verify_version
- verify_version_match_release
- maven_release_file
-elif [[ "$DISTRIBUTION_TYPE" == "container" ]]; then
- if $USE_RELEASE_FILE ; then
- RELEASE_SCHEMA="release-container-schema.yaml"
- echo "INFO: Fetching schema $RELEASE_SCHEMA"
- wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA}
- verify_schema
- fi
- set_variables_container
- verify_version
- container_release_file
-elif [[ "$DISTRIBUTION_TYPE" == "pypi" ]]; then
- if $USE_RELEASE_FILE ; then
- RELEASE_SCHEMA="release-pypi-schema.yaml"
- echo "INFO: Fetching schema $RELEASE_SCHEMA"
- wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA}
- verify_schema
- fi
- set_variables_pypi
- verify_version
- verify_pypi_match_release
- pypi_release_file
-elif [[ "$DISTRIBUTION_TYPE" == "packagecloud" ]]; then
- RELEASE_SCHEMA="release-packagecloud-schema.yaml"
- package_name=$(yq -r '.package_name' $release_file)
- username_repo="o-ran-sc/staging"
- echo "INFO: Fetching schema $RELEASE_SCHEMA"
- wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${RELEASE_SCHEMA}
- verify_schema
- for name in $(yq -r '.package_name[].name' $release_file); do
- package_name=$name
- packagecloud_verify "$package_name" "$username_repo"
- if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then
- packagecloud_promote "$package_name" "$username_repo"
+case $DISTRIBUTION_TYPE in
+
+ maven)
+ if $USE_RELEASE_FILE ; then
+ release_schema="release-schema.yaml"
+ echo "INFO: Fetching schema $release_schema"
+ wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/$release_schema
+ verify_schema
fi
- done
-else
- echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
- exit 1
-fi
+ set_variables_maven
+ verify_version
+ verify_version_match_release
+ maven_release_file
+ ;;
+
+ container)
+ if $USE_RELEASE_FILE ; then
+ release_schema="release-container-schema.yaml"
+ echo "INFO: Fetching schema $release_schema"
+ wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+ verify_schema
+ fi
+ set_variables_container
+ verify_version
+ container_release_file
+ ;;
+
+ pypi)
+ if $USE_RELEASE_FILE ; then
+ release_schema="release-pypi-schema.yaml"
+ echo "INFO: Fetching schema $release_schema"
+ wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+ verify_schema
+ fi
+ set_variables_pypi
+ verify_version
+ verify_pypi_match_release
+ pypi_release_file
+ ;;
+
+ packagecloud)
+ release_schema="release-packagecloud-schema.yaml"
+ package_name=$(yq -r '.package_name' $release_file)
+ packagecloud_account=$(cat "$ACCOUNT_NAME_FILE")
+ echo "INFO: Fetching schema $release_schema"
+ wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+ verify_schema
+ for name in $(yq -r '.package_name[].name' $release_file); do
+ package_name=$name
+ packagecloud_verify "$package_name" "$packagecloud_account"
+ if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then
+ packagecloud_promote "$package_name" "$packagecloud_account"
+ fi
+ done
+ ;;
+
+ *)
+ echo "ERROR: distribution_type: $DISTRIBUTION_TYPE not supported"
+ exit 1
+ ;;
+esac
echo "---> release-job.sh ends"
Code Review / releng / global-jjb.git / commitdiff