From 5f79451fbde6689fd5b43f3d085b2766022f669e Mon Sep 17 00:00:00 2001 From: "Lott, Christopher (cl778h)" Date: Fri, 3 Apr 2020 10:53:43 -0400 Subject: [PATCH] Add CMake templates to build/publish DEB/RPM pkgs New templates gerrit-cmake-packagecloud-stage and github-cmake-packagecloud-stage for building DEB/RPM package files and publishing them to a PackageCloud.io repository on posted comment. Adds builder macros lf-packagecloud-file-provider and lf-packagecloud-push. Adds script packagecloud-push.sh to call Ruby gem package_cloud. The new templates are lifted from the ORAN project. Change-Id: Ia39f5b43c2ef22485166a5f54fcf31d53eab84f8 Issue-ID: RELENG-2832 Signed-off-by: Lott, Christopher (cl778h) --- .jjb-test/lf-c-cpp-jobs.yaml | 10 ++ docs/jjb/lf-c-cpp-jobs.rst | 68 ++++++++- docs/jjb/lf-macros.rst | 21 +++ jjb/lf-c-cpp-jobs.yaml | 163 +++++++++++++++++++++ jjb/lf-macros.yaml | 24 +++ .../cmake-packagecloud-stage-c7f6f22243127193.yaml | 10 ++ shell/cmake-build.sh | 41 ++++-- shell/packagecloud-push.sh | 53 +++++++ 8 files changed, 378 insertions(+), 12 deletions(-) create mode 100644 releasenotes/notes/cmake-packagecloud-stage-c7f6f22243127193.yaml create mode 100755 shell/packagecloud-push.sh diff --git a/.jjb-test/lf-c-cpp-jobs.yaml b/.jjb-test/lf-c-cpp-jobs.yaml index a8236a71..6099d691 100644 --- a/.jjb-test/lf-c-cpp-jobs.yaml +++ b/.jjb-test/lf-c-cpp-jobs.yaml @@ -2,6 +2,7 @@ - project: name: gerrit-cmake-jobs jobs: + - gerrit-cmake-packagecloud-stage - gerrit-cmake-sonar - gerrit-cmake-sonarqube - gerrit-cmake-stage @@ -15,10 +16,15 @@ nexus-group-id: fakeproject-group-id nexus-snapshot-repo: fakeproject-snapshots lftools-version: bogus-lftools-version + debian-distribution-versions: ubuntu/bionic + packagecloud-account: example + packagecloud-repo: master + rpm-distribution-versions: el/5 - project: name: github-cmake-jobs jobs: + - github-cmake-packagecloud-stage - github-cmake-sonar - github-cmake-sonarqube - github-cmake-stage @@ -32,3 +38,7 @@ nexus-group-id: fakeproject-group-id nexus-snapshot-repo: fakeproject-snapshots lftools-version: bogus-lftools-version + debian-distribution-versions: ubuntu/bionic + packagecloud-account: example + packagecloud-repo: master + rpm-distribution-versions: el/5 diff --git a/docs/jjb/lf-c-cpp-jobs.rst b/docs/jjb/lf-c-cpp-jobs.rst index 98c14bfa..8b6fe243 100644 --- a/docs/jjb/lf-c-cpp-jobs.rst +++ b/docs/jjb/lf-c-cpp-jobs.rst @@ -216,6 +216,7 @@ project into a tar.xz tarball to produce a release candidate. :build-timeout: Timeout in minutes before aborting build. (default: 60) :cmake-opts: Parameters to pass to cmake. (default: '') :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :install: Install build products to /usr/local. (default: true) :install-prefix: CMAKE_INSTALL_PREFIX to use for install. (default: $BUILD_DIR/output) :make-opts: Parameters to pass to make. (default: '') @@ -250,10 +251,71 @@ project into a tar.xz tarball to produce a release candidate. PATCH_VERSION="$(grep 'set(OCIO_VERSION_PATCH' CMakeLists.txt | awk '{{print $NF}}' | awk -F')' '{{print $1}}')" echo "${{MAJOR_VERSION}}.${{MINOR_VERSION}}.${{PATCH_VERSION}}" > /tmp/artifact_version +CMake PackageCloud Stage +------------------------ + +Stage job which runs cmake && make, then uploads all DEB/RPM files in the +build directory to PackageCloud.io. Triggered by comment. + +The Jenkins system must have a configuration file provider that installs +files ".packagecloud" and "packagecloud_api" to the Jenkins home directory +with appropriate credentials. + +The Jenkins build minion must have the Ruby gem "package_cloud" installed. + +:Template Names: + + - {project-name}-cmake-packagecloud-stage-{stream} + - gerrit-cmake-packagecloud-stage + - github-cmake-packagecloud-stage + +:Comment Trigger: stage-release + +:Required parameters: + + :build-node: The node to run build on. + :debian-distribution-versions: list of DEB package distro/version strings + separated by space; example: "ubuntu/bionic debian/stretch" + :jenkins-ssh-credential: Credential to use for SSH. + (Configure in defaults.yaml) + :packagecloud-account: PackageCloud account ID; example: oran + :packagecloud-repo: PackageCloud repository; example: master, staging + :project: The git repository name. + :project-name: Prefix used to name jobs. + :rpm-distribution-versions: list of RPM package distro/version strings + separated by space; example: "el/4 el/5" + +:Optional parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-dir: Directory to build the project in. (default: $WORKSPACE/build) + :build-timeout: Timeout in minutes before aborting build. (default: 60) + :cmake-opts: Parameters to pass to cmake. (default: '') + :disable-job: Whether to disable the job (default: false) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :install: Install build products to /usr/local. (default: false) + :install-prefix: CMAKE_INSTALL_PREFIX to use for install. + (default: $BUILD_DIR/output) + :make-opts: Parameters to pass to make. (default: '') + :pre-build: Shell script to run before performing build. Useful for + setting up dependencies. (default: '') + :stream: Keyword that to represent a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + CMake Verify ------------ -Verify job which runs cmake && make && make install to test a project build. +Verify job which runs cmake && make to test a project build, then +runs make install, copies the build products to /usr/local and runs +ldconfig to make the shared lib(s) available. The install steps run +by default, see optional parameter "install". :Template Names: @@ -268,6 +330,8 @@ Verify job which runs cmake && make && make install to test a project build. :build-node: The node to run build on. :jenkins-ssh-credential: Credential to use for SSH. (Configure in defaults.yaml) + :project: The git repository name. + :project-name: Prefix used to name jobs. :Optional parameters: @@ -276,7 +340,9 @@ Verify job which runs cmake && make && make install to test a project build. :build-dir: Directory to build the project in. (default: $WORKSPACE/target) :build-timeout: Timeout in minutes before aborting build. (default: 60) :cmake-opts: Parameters to pass to cmake. (default: '') + :disable-job: Whether to disable the job (default: false) :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :install: Install build products to /usr/local. (default: true) :install-prefix: CMAKE_INSTALL_PREFIX to use for install. (default: $BUILD_DIR/output) :make-opts: Parameters to pass to make. (default: '') diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index f3eae1be..357ecb65 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -222,6 +222,27 @@ that want to use Maven. :mvn-version: Version of Maven to install. +lf-packagecloud-file-provider +----------------------------- + +Provisions files required by the Ruby gem package_cloud, namely +".packagecloud" and "packagecloud_api" in the Jenkins home directory. + +lf-packagecloud-push +-------------------- + +Pushes DEB/RPM package files to PackageCloud using the Ruby gem package_cloud. + +:Required parameters: + + :build-dir: Directory with deb/rpm files to push + :debian-distribution-versions: list of DEB package distro/version strings + separated by space; example: ubuntu/bionic debian/stretch + :packagecloud-account: PackageCloud account ID; example: oran + :packagecloud-repo: PackageCloud repository; example: master, staging + :rpm-distribution-versions: list of RPM package distro/version strings + separated by space; example: el/4 el/5 + lf-pip-install -------------- diff --git a/jjb/lf-c-cpp-jobs.yaml b/jjb/lf-c-cpp-jobs.yaml index d82a9bc1..115eac71 100644 --- a/jjb/lf-c-cpp-jobs.yaml +++ b/jjb/lf-c-cpp-jobs.yaml @@ -292,6 +292,7 @@ disable-job: false git-url: "$GIT_URL/$PROJECT" github-url: "https://github.com" + install: true install-prefix: "$BUILD_DIR/output" make-opts: "" mvn-global-settings: global-settings @@ -312,6 +313,10 @@ cmake-opts: "{cmake-opts}" install-prefix: "{install-prefix}" make-opts: "{make-opts}" + - bool: + name: INSTALL + default: "{install}" + description: "Install build products to /usr/local (true/false)." - string: name: NEXUS_GROUP_ID default: "{nexus-group-id}" @@ -424,6 +429,148 @@ white-list-target-branches: - "{branch}" +################## +# CMAKE+PC STAGE # +################## + +- lf_cmake_pc_stage: &lf_cmake_pc_stage + name: lf-cmake-packagecloud-stage + + ###################### + # Default parameters # + ###################### + + branch: master + build-days-to-keep: 7 + build-dir: "$WORKSPACE/build" + build-timeout: 15 + cmake-opts: "" + disable-job: false + git-url: "$GIT_URL/$PROJECT" + github-url: "https://github.com" + install: false + install-prefix: "$BUILD_DIR/output" + make-opts: "" + pre-build: "" + stream: master + submodule-recursive: true + submodule-timeout: 10 + submodule-disable: false + + gerrit_trigger_file_paths: + - compare-type: REG_EXP + pattern: ".*" + + # github_included_regions MUST match gerrit_trigger_file_paths + github_included_regions: + - ".*" + + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - lf-cmake-parameters: + build-dir: "{build-dir}" + cmake-opts: "{cmake-opts}" + install-prefix: "{install-prefix}" + make-opts: "{make-opts}" + - bool: + name: INSTALL + default: "{install}" + description: "Install build products to /usr/local (true/false)." + + builders: + - config-file-provider: + files: + - file-id: ".packagecloud" + target: "$HOME/.packagecloud" + - shell: "{pre-build}" + - shell: !include-raw-escape: ../shell/cmake-build.sh + - lf-packagecloud-push: + build-dir: "{build-dir}" + packagecloud-account: "{packagecloud-account}" + packagecloud-repo: "{packagecloud-repo}" + debian-distribution-versions: "{debian-distribution-versions}" + rpm-distribution-versions: "{rpm-distribution-versions}" + +- job-template: + # build with cmake and publish to PackageCloud on command + name: "{project-name}-cmake-packagecloud-stage-{stream}" + id: gerrit-cmake-packagecloud-stage + <<: *lf_cmake_common + <<: *lf_cmake_pc_stage + + ###################### + # Default parameters # + ###################### + + gerrit_stage_triggers: + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+stage-release\s*$' + + ##################### + # Job Configuration # + ##################### + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + # stage jobs always build from tip + choosing-strategy: default + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_stage_triggers}" + projects: + - project-compare-type: ANT + project-pattern: "{project}" + branches: + - branch-compare-type: ANT + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" + +- job-template: + name: "{project-name}-cmake-packagecloud-stage-{stream}" + id: github-cmake-packagecloud-stage + concurrent: true + <<: *lf_cmake_common + <<: *lf_cmake_pc_stage + + properties: + - lf-infra-properties: + build-days-to-keep: "{build-days-to-keep}" + - github: + url: "{github-url}/{github-org}/{project}" + + scm: + - lf-infra-github-scm: + url: "{git-clone-url}{github-org}/{project}" + refspec: "+refs/pull/*:refs/remotes/origin/pr/*" + branch: "$sha1" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: default + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + triggers: + - github-pull-request: + trigger-phrase: "^(stage-release)$" + only-trigger-phrase: true + status-context: "CMake PackageCloud Stage" + permit-all: true + github-hooks: true + white-list-target-branches: + - "{branch}" + ################ # CMAKE VERIFY # ################ @@ -443,6 +590,7 @@ disable-job: false git-url: "$GIT_URL/$PROJECT" github-url: "https://github.com" + install: true install-prefix: "$BUILD_DIR/output" make-opts: "" pre-build: "" @@ -468,6 +616,21 @@ github_included_regions: - ".*" + parameters: + - lf-infra-parameters: + project: "{project}" + branch: "{branch}" + stream: "{stream}" + - lf-cmake-parameters: + build-dir: "{build-dir}" + cmake-opts: "{cmake-opts}" + install-prefix: "{install-prefix}" + make-opts: "{make-opts}" + - bool: + name: INSTALL + default: "{install}" + description: "Install build products to /usr/local (true/false)." + builders: - shell: "{pre-build}" - shell: !include-raw-escape: ../shell/cmake-build.sh diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index 94847dcf..bf0b3fab 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -222,6 +222,30 @@ - shell: !include-raw-escape: ../shell/maven-central.sh - lf-provide-maven-settings-cleanup +- builder: + name: lf-packagecloud-file-provider + builders: + - config-file-provider: + files: + - file-id: ".packagecloud" + target: "$HOME/.packagecloud" + - config-file-provider: + files: + - file-id: "packagecloud_api" + target: "$HOME/packagecloud_api" + +- builder: + name: lf-packagecloud-push + builders: + - inject: + properties-content: | + BUILD_DIR={build-dir} + PACKAGECLOUD_ACCOUNT={packagecloud-account} + PACKAGECLOUD_REPO={packagecloud-repo} + DEBIAN_DISTRIBUTION_VERSIONS={debian-distribution-versions} + RPM_DISTRIBUTION_VERSIONS={rpm-distribution-versions} + - shell: !include-raw-escape: ../shell/packagecloud-push.sh + - builder: name: lf-maven-install builders: diff --git a/releasenotes/notes/cmake-packagecloud-stage-c7f6f22243127193.yaml b/releasenotes/notes/cmake-packagecloud-stage-c7f6f22243127193.yaml new file mode 100644 index 00000000..2fe79354 --- /dev/null +++ b/releasenotes/notes/cmake-packagecloud-stage-c7f6f22243127193.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + New templates gerrit-cmake-packagecloud-stage and + github-cmake-packagecloud-stage for building DEB/RPM package + files and publishing them to a PackageCloud.io repository on + posted comment. Adds builder macros lf-packagecloud-file-provider + and lf-packagecloud-push. Adds script packagecloud-push.sh to + call Ruby gem package_cloud. + The new templates are lifted from the ORAN project. diff --git a/shell/cmake-build.sh b/shell/cmake-build.sh index 2466c173..41c96b96 100644 --- a/shell/cmake-build.sh +++ b/shell/cmake-build.sh @@ -8,33 +8,52 @@ # which accompanies this distribution, and is available at # http://www.eclipse.org/legal/epl-v10.html ############################################################################## + +# Creates a build subdir then invokes cmake and make from that dir with the +# specified install prefix and options. Optionally runs make install and tars +# up all files from the install prefix, then uses sudo to extract those files +# to /usr/local and run ldconfig, leaving shared lib(s) ready for use. +# Prereqs: +# The build minion has cmake, make, gcc etc. +# Environment variables: +# WORKSPACE is a non-empty path (required) +# CMAKE_INSTALL_PREFIX is a non-empty path (required) +# PROJECT is a non-empty name (required) +# BUILD_DIR is a path (optional; has usable default) +# CMAKE_OPTS has options for cmake (optional, empty default) +# MAKE_OPTS has options for make (optional, empty default) +# INSTALL is "true" or "false" (optional, default true) + echo "---> cmake-build.sh" +# be careful and verbose +set -eux -o pipefail + build_dir="${BUILD_DIR:-$WORKSPACE/target}" cmake_opts="${CMAKE_OPTS:-}" make_opts="${MAKE_OPTS:-}" +install="${INSTALL:-true}" # Not a misspelling as shellcheck reports. # shellcheck disable=SC2153 project="${PROJECT//\//\-}" -################ -# Script start # -################ - -set -eux -o pipefail - mkdir -p "$build_dir" cd "$build_dir" || exit +cmake -version # $cmake_opts needs to wordsplit to pass options. # shellcheck disable=SC2086 eval cmake -DCMAKE_INSTALL_PREFIX="$INSTALL_PREFIX" $cmake_opts .. +make -version # $make_opts needs to wordsplit to pass options. # shellcheck disable=SC2086 make $make_opts -make install -mkdir -p "$WORKSPACE/dist" -tar -cJvf "$WORKSPACE/dist/$project.tar.xz" -C "$INSTALL_PREFIX" . +if [[ $install == true ]]; then + make install + mkdir -p "$WORKSPACE/dist" + tar -cJvf "$WORKSPACE/dist/$project.tar.xz" -C "$INSTALL_PREFIX" . + sudo tar -xvf "$WORKSPACE/dist/$project.tar.xz" -C "/usr/local" + sudo ldconfig +fi -sudo tar -xvf "$WORKSPACE/dist/$project.tar.xz" -C "/usr/local" -sudo ldconfig +echo "---> cmake-build.sh ends" diff --git a/shell/packagecloud-push.sh b/shell/packagecloud-push.sh new file mode 100755 index 00000000..beb94676 --- /dev/null +++ b/shell/packagecloud-push.sh @@ -0,0 +1,53 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2020 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## + +# Prereqs: +# The build minion has the ruby gem "package_cloud" +# The required credentials and API files have been provisioned +# The build directory has .deb/.rpm files +# Environment variables: +# BUILD_DIR is set and non-empty +# DEBIAN_DISTRIBUTION_VERSIONS has distro list like "debian/stretch" +# RPM_DISTRIBUTION_VERSIONS has distro list like "el/4 el/5" +# PACKAGECLOUD_ACCOUNT is set and non-empty +# PACKAGECLOUD_REPO is a value like "staging" + +echo "---> packagecloud-push.sh" +set -eu -o pipefail + +# Pushes packages to PackageCloud +# $1 is a shell-style glob pattern for package files +# $2 is a space-separated list of distribution versions +push_packages () { + echo "Expanding file pattern $1" + # shellcheck disable=SC2206 + pkgs=($1) + if [[ ! -f ${pkgs[0]} ]]; then + echo "WARN: no files matched pattern $1" + return + fi + echo "Found package file(s):" "${pkgs[@]}" + echo "Processing distribution version(s): $2" + for ver in $2; do + arg="${PACKAGECLOUD_ACCOUNT}/${PACKAGECLOUD_REPO}/${ver}" + for pkg in "${pkgs[@]}"; do + echo "Pushing $arg $pkg" + package_cloud push "$arg" "$pkg" + done + done +} + +echo "Working in directory $BUILD_DIR" +cd "$BUILD_DIR" +push_packages "*.deb" "$DEBIAN_DISTRIBUTION_VERSIONS" +push_packages "*.rpm" "$RPM_DISTRIBUTION_VERSIONS" + +echo "---> packagecloud-push.sh ends" -- 2.16.6