Update packagecloud jobs

Add sigul to lf-packagecloud-release-common. Add
set_variables_packagecloud. Allow tagging repo with ref or git_tag.
Add logs_dir to release-packagecloud-schema.yaml.

Issue-Id: RELENG-2579
Signed-off-by: Houa Yang <hyang@contractor.linuxfoundation.org>
Change-Id: I23a8ef1e0aed261b0fbca6375c694b4ed268c082

diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst
index 2413190..65cfb79 100644 (file)
--- a/docs/jjb/lf-release-jobs.rst
+++ b/docs/jjb/lf-release-jobs.rst
@@ -196,9 +196,13 @@ must start with "packagecloud". For example releases/packagecloud-1.6-tree.yaml
 
     $ cat releases/packagecloud-1.6-tree.yaml
     ---
-    package_name:
+    package_name: tree-1.6.0
+    packages:
         - name: tree-1.6.0-10.el7.x86_64.rpm
         - name: test.rpm
+    ref: 5555cd2dd345fbeec0d3e2162e00835852342cda
+    log_dir: example-project-packagecloud-merge/21
+    version: 1.6.0
 
 The following parameters must appear in the PackageCloud release yaml file.
 These are not part of the Jenkins job definition to allow independent
@@ -207,13 +211,25 @@ packages.
 
 :Required Parameters:
 
-    :package_name: A list of names that specify the packages to promote.
-        (Found in jenkins console log when using gem to push package eg.
+    :package_name: Name of the release package.
+    :packages: A list of names that specify the packages to promote.
+        Found in jenkins console log when using gem to push package eg.
         "Pushing /path/of/package/name-of-package.rpm... success!"
         OR using rest api call to query packagecloud.io repo
         "curl https://packagecloud.io/api/v1/repos/test_user/test_repo/search?q=
         | yq -r .[].filename"
+    :ref: The git commit reference (SHA-1 code) to tag with the version string.
+    :log_dir: The suffix of the logs URL reported on completion by the
+        Jenkins merge job that created and pushed the distribution files
+        to the staging repository. For example, use value
+        "example-project-packagecloud-merge-/21" for the logs URL
+        https://logs.lf-project.org/production/vex-sjc-lfp-jenkins-prod-1/example-project-packagecloud-merge/21
+    :version: The semantic version string used for the package.
 
+:Optional Parameters:
+
+    :git_tag: The tag string to sign and push to the Git repository.
+       (default: the semantic version string)
 
 The JSON schema for a PackageCloud release file appears below.
 

diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml
index 6be717d..f53a2d9 100644 (file)
--- a/jjb/lf-release-jobs.yaml
+++ b/jjb/lf-release-jobs.yaml
@@ -542,10 +542,20 @@
       - lf-infra-pre-build
       - config-file-provider:
           files:
+            - file-id: sigul-config
+              variable: SIGUL_CONFIG
+            - file-id: sigul-password
+              variable: SIGUL_PASSWORD
+            - file-id: sigul-pki
+              variable: SIGUL_PKI
+            - file-id: signing-pubkey
+              variable: SIGNING_PUBKEY
             - file-id: "packagecloud_api"
               target: "$HOME/packagecloud_api"
             - file-id: packagecloud-account
               variable: ACCOUNT_NAME_FILE
+      - shell: !include-raw-escape: ../shell/sigul-configuration.sh
+      - shell: !include-raw-escape: ../shell/sigul-install.sh
       - lf-release
 
 - job-template:

diff --git a/releasenotes/notes/update-packagecloud-jobs-fc7510b516ab2bf5.yaml b/releasenotes/notes/update-packagecloud-jobs-fc7510b516ab2bf5.yaml
new file mode 100644 (file)
index 0000000..2698318
--- /dev/null
+++ b/releasenotes/notes/update-packagecloud-jobs-fc7510b516ab2bf5.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Extend packagecloud release:
+      Release additionally requires log directory, ref and version.
+      Add set_variables_packagecloud & verify_packagecloud_match_release
+      to allow tagging the git repo.
+      Update packagecloud jobs to include sigul signing.

diff --git a/schema/release-packagecloud-schema.yaml b/schema/release-packagecloud-schema.yaml
index f17d174..8432f30 100644 (file)
--- a/schema/release-packagecloud-schema.yaml
+++ b/schema/release-packagecloud-schema.yaml
@@ -13,10 +13,24 @@ $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-packagecloud
 
 required:
   - "package_name"
+  - "packages"
+  - "ref"
+  - "log_dir"
+  - "version"
 
 properties:
   package_name:
+    type: "string"
+  packages:
     type: "array"
     properties:
       name:
         type: "string"
+  git_tag:
+    type: "string"
+  ref:
+    type: "string"
+  log_dir:
+    type: "string"
+  version:
+    type: "string"

diff --git a/shell/release-job.sh b/shell/release-job.sh
index aa347e2..220f016 100644 (file)
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -46,102 +46,102 @@ set_variables_common(){
     # Jenkins parameter drop-down defaults DISTRIBUTION_TYPE to None
     DISTRIBUTION_TYPE="${DISTRIBUTION_TYPE:-None}"
     if [[ $DISTRIBUTION_TYPE == "None" ]]; then
-        DISTRIBUTION_TYPE="$(niet ".distribution_type" "$release_file")"
+        DISTRIBUTION_TYPE=$(niet ".distribution_type" "$release_file")
     fi
 
-    PATCH_DIR="$(mktemp -d)"
+    PATCH_DIR=$(mktemp -d)
 
     # Displaying Release Information (Common variables)
     printf "\t%-30s\n" RELEASE_ENVIRONMENT_INFO:
-    printf "\t%-30s %s\n" RELEASE_FILE: $release_file
-    printf "\t%-30s %s\n" LOGS_SERVER: $LOGS_SERVER
-    printf "\t%-30s %s\n" NEXUS_PATH: $NEXUS_PATH
-    printf "\t%-30s %s\n" JENKINS_HOSTNAME: $JENKINS_HOSTNAME
-    printf "\t%-30s %s\n" SILO: $SILO
-    printf "\t%-30s %s\n" PROJECT: $PROJECT
-    printf "\t%-30s %s\n" PROJECT-DASHED: ${PROJECT//\//-}
-    printf "\t%-30s %s\n" DISTRIBUTION_TYPE: $DISTRIBUTION_TYPE
+    printf "\t%-30s %s\n" RELEASE_FILE: "$release_file"
+    printf "\t%-30s %s\n" LOGS_SERVER: "$LOGS_SERVER"
+    printf "\t%-30s %s\n" NEXUS_PATH: "$NEXUS_PATH"
+    printf "\t%-30s %s\n" JENKINS_HOSTNAME: "$JENKINS_HOSTNAME"
+    printf "\t%-30s %s\n" SILO: "$SILO"
+    printf "\t%-30s %s\n" PROJECT: "$PROJECT"
+    printf "\t%-30s %s\n" PROJECT-DASHED: "${PROJECT//\//-}"
+    printf "\t%-30s %s\n" DISTRIBUTION_TYPE: "$DISTRIBUTION_TYPE"
 }
 
 set_variables_maven(){
     echo "INFO: Setting maven variables"
     if [[ -z ${VERSION:-} ]]; then
-        VERSION="$(niet ".version" "$release_file")"
+        VERSION=$(niet ".version" "$release_file")
     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then
-            GIT_TAG="$(niet ".git_tag" "$release_file")"
+            GIT_TAG=$(niet ".git_tag" "$release_file")
         else
             GIT_TAG="$VERSION"
         fi
     fi
     if [[ -z ${LOG_DIR:-} ]]; then
-        LOG_DIR="$(niet ".log_dir" "$release_file")"
+        LOG_DIR=$(niet ".log_dir" "$release_file")
     fi
     LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
     LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
 
     # Continuing displaying Release Information (Maven)
     printf "\t%-30s\n" RELEASE_MAVEN_INFO:
-    printf "\t%-30s %s\n" VERSION: $VERSION
-    printf "\t%-30s %s\n" GIT_TAG: $GIT_TAG
-    printf "\t%-30s %s\n" LOG_DIR: $LOG_DIR
-    printf "\t%-30s %s\n" LOGS_URL: $LOGS_URL
+    printf "\t%-30s %s\n" VERSION: "$VERSION"
+    printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
+    printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
+    printf "\t%-30s %s\n" LOGS_URL: "$LOGS_URL"
 }
 
 set_variables_container(){
     echo "INFO: Setting container variables"
     if [[ -z ${VERSION:-} ]]; then
-        VERSION="$(niet ".container_release_tag" "$release_file")"
+        VERSION=$(niet ".container_release_tag" "$release_file")
     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then
-            GIT_TAG="$(niet ".git_tag" "$release_file")"
+            GIT_TAG=$(niet ".git_tag" "$release_file")
         else
             GIT_TAG="$VERSION"
         fi
    fi
     if grep -q "container_pull_registry" "$release_file" ; then
-        CONTAINER_PULL_REGISTRY="$(niet ".container_pull_registry" "$release_file")"
+        CONTAINER_PULL_REGISTRY=$(niet ".container_pull_registry" "$release_file")
     fi
     if grep -q "container_push_registry" "$release_file" ; then
-        CONTAINER_PUSH_REGISTRY="$(niet ".container_push_registry" "$release_file")"
+        CONTAINER_PUSH_REGISTRY=$(niet ".container_push_registry" "$release_file")
     fi
     # Make sure both pull and push registries are defined
     if [ -z ${CONTAINER_PULL_REGISTRY+x} ] || [ -z ${CONTAINER_PUSH_REGISTRY+x} ]; then
         echo "ERROR: CONTAINER_PULL_REGISTRY and CONTAINER_PUSH_REGISTRY need to be defined"
         exit 1
     fi
-    ref="$(niet ".ref" "$release_file")"
+    ref=$(niet ".ref" "$release_file")
 
     # Continuing displaying Release Information (Container)
     printf "\t%-30s\n" RELEASE_CONTAINER_INFO:
-    printf "\t%-30s %s\n" CONTAINER_RELEASE_TAG: $VERSION
-    printf "\t%-30s %s\n" CONTAINER_PULL_REGISTRY: $CONTAINER_PULL_REGISTRY
-    printf "\t%-30s %s\n" CONTAINER_PUSH_REGISTRY: $CONTAINER_PUSH_REGISTRY
-    printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: $ref
-    printf "\t%-30s %s\n" GIT_TAG: $GIT_TAG
+    printf "\t%-30s %s\n" CONTAINER_RELEASE_TAG: "$VERSION"
+    printf "\t%-30s %s\n" CONTAINER_PULL_REGISTRY: "$CONTAINER_PULL_REGISTRY"
+    printf "\t%-30s %s\n" CONTAINER_PUSH_REGISTRY: "$CONTAINER_PUSH_REGISTRY"
+    printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: "$ref"
+    printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
 }
 
 set_variables_pypi(){
     echo "INFO: Setting pypi variables"
     if [[ -z ${LOG_DIR:-} ]]; then
-        LOG_DIR="$(niet ".log_dir" "$release_file")"
+        LOG_DIR=$(niet ".log_dir" "$release_file")
     fi
     LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
     LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
     if [[ -z ${PYPI_PROJECT:-} ]]; then
-        PYPI_PROJECT="$(niet ".pypi_project" "$release_file")"
+        PYPI_PROJECT=$(niet ".pypi_project" "$release_file")
     fi
     if [[ -z ${PYTHON_VERSION:-} ]]; then
-        PYTHON_VERSION="$(niet ".python_version" "$release_file")"
+        PYTHON_VERSION=$(niet ".python_version" "$release_file")
     fi
     if [[ -z ${VERSION:-} ]]; then
-        VERSION="$(niet ".version" "$release_file")"
+        VERSION=$(niet ".version" "$release_file")
     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then
-            GIT_TAG="$(niet ".git_tag" "$release_file")"
+            GIT_TAG=$(niet ".git_tag" "$release_file")
         else
             GIT_TAG="$VERSION"
         fi
@@ -155,7 +155,39 @@ set_variables_pypi(){
     printf "\t%-30s %s\n" PYPI_PROJECT: "$PYPI_PROJECT"
     printf "\t%-30s %s\n" PYTHON_VERSION: "$PYTHON_VERSION"
     printf "\t%-30s %s\n" VERSION: "$VERSION"
-    printf "\t%-30s %s\n" GIT_TAG: $GIT_TAG
+    printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
+}
+
+set_variables_packagecloud(){
+     echo "INFO: Setting packagecloud variables"
+     if [[ -z ${VERSION:-} ]]; then
+         VERSION=$(niet ".version" "$release_file")
+     fi
+     if [[ -z ${GIT_TAG:-} ]]; then
+         if grep -q "git_tag" $release_file ; then
+             GIT_TAG=$(niet ".git_tag" "$release_file")
+         else
+             GIT_TAG="$VERSION"
+         fi
+     fi
+     if [[ -z ${LOG_DIR:-} ]]; then
+         LOG_DIR=$(niet ".log_dir" "$release_file")
+     fi
+     if [[ -z ${REF:-} ]]; then
+         REF=$(niet ".ref" "$release_file")
+     fi
+     if [[ -z ${PACKAGE_NAME:-} ]]; then
+         PACKAGE_NAME=$(niet ".package_name" "$release_file")
+     fi
+     logs_url="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
+     logs_url=${logs_url%/}  # strip any trailing '/'
+
+     printf "\t%-30s %s\n" PACKAGE_NAME: "$PACKAGE_NAME"
+     printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
+     printf "\t%-30s %s\n" LOGS_URL: "$logs_url"
+     printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: "$REF"
+     printf "\t%-30s %s\n" VERSION: "$VERSION"
+     printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
 }
 
 verify_schema(){
@@ -204,6 +236,18 @@ verify_pypi_match_release(){
     fi
 }
 
+verify_packagecloud_match_release(){
+    echo "INFO: Fetching console log from $logs_url"
+    wget -q -P /tmp "${logs_url}/"console.log.gz
+    echo "INFO: Searching for uploaded step, package name $PACKAGE_NAME and version $VERSION in job log"
+    if zgrep "Successfully uploaded" /tmp/console.log.gz | grep "$PACKAGE_NAME" | grep "$VERSION"; then
+        echo "INFO: found expected strings in job log"
+    else
+        echo "ERROR: failed to find expected strings in job log"
+        exit 1
+    fi
+}
+
 # sigul is only available on Centos
 # TODO: write tag-github-repo function
 tag-gerrit-repo(){
@@ -287,7 +331,7 @@ container_release_file(){
         else
             echo "INFO: $VERSION not found in releases, release will be prepared. Continuing..."
             docker pull "$CONTAINER_PULL_REGISTRY"/"$lfn_umbrella"/"$name":"$version"
-            container_image_id="$(docker images | grep $name | grep $version | awk '{print $3}')"
+            container_image_id=$(docker images | grep "$name" | grep "$version" | awk '{print $3}')
             echo "INFO: Merge will run the following commands:"
             echo "docker tag $container_image_id $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
             echo "docker push $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
@@ -391,6 +435,8 @@ packagecloud_promote(){
         destination="$2/release" "$promote_url" \
         | echo "INFO: Promoted package location: \
         https://packagecloud.io$(yq -r .package_html_url)"
+    git checkout "$REF"
+    tag-gerrit-repo
 }
 
 ##############################  End Function Declarations  ################################
@@ -444,17 +490,20 @@ case $DISTRIBUTION_TYPE in
         ;;
 
     packagecloud)
-        release_schema="release-packagecloud-schema.yaml"
-        package_name=$(yq -r '.package_name' $release_file)
-        packagecloud_account=$(cat "$ACCOUNT_NAME_FILE")
-        echo "INFO: Fetching schema $release_schema"
-        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
-        verify_schema
-        for name in $(yq -r '.package_name[].name' $release_file); do
-            package_name=$name
-            packagecloud_verify "$package_name" "$packagecloud_account"
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-packagecloud-schema.yaml"
+            packagecloud_account=$(cat "$ACCOUNT_NAME_FILE")
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+            verify_schema
+        fi
+        set_variables_packagecloud
+        verify_packagecloud_match_release
+        for name in $(yq -r '.packages[].name' $release_file); do
+            package=$name
+            packagecloud_verify "$package" "$packagecloud_account"
             if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then
-                packagecloud_promote "$package_name" "$packagecloud_account"
+                packagecloud_promote "$package" "$packagecloud_account"
             fi
         done
         ;;