Update packagecloud jobs

Add sigul to lf-packagecloud-release-common. Add
set_variables_packagecloud. Allow tagging repo with ref or git_tag.
Add logs_dir to release-packagecloud-schema.yaml.

Issue-Id: RELENG-2579
Signed-off-by: Houa Yang <hyang@contractor.linuxfoundation.org>
Change-Id: I23a8ef1e0aed261b0fbca6375c694b4ed268c082

diff --git a/docs/jjb/lf-release-jobs.rst b/docs/jjb/lf-release-jobs.rst
index 2413190..65cfb79 100644 (file)
--- a/docs/jjb/lf-release-jobs.rst
+++ b/docs/jjb/lf-release-jobs.rst
@@ -196,9 +196,13 @@ must start with "packagecloud". For example releases/packagecloud-1.6-tree.yaml
 
     $ cat releases/packagecloud-1.6-tree.yaml
     ---
 
     $ cat releases/packagecloud-1.6-tree.yaml
     ---

-    package_name:
+    package_name: tree-1.6.0
+    packages:

         - name: tree-1.6.0-10.el7.x86_64.rpm
         - name: test.rpm
         - name: tree-1.6.0-10.el7.x86_64.rpm
         - name: test.rpm

+    ref: 5555cd2dd345fbeec0d3e2162e00835852342cda
+    log_dir: example-project-packagecloud-merge/21
+    version: 1.6.0

 
 The following parameters must appear in the PackageCloud release yaml file.
 These are not part of the Jenkins job definition to allow independent
 
 The following parameters must appear in the PackageCloud release yaml file.
 These are not part of the Jenkins job definition to allow independent


@@ -207,13 +211,25 @@ packages.
 
 :Required Parameters:
 
 
 :Required Parameters:
 

-    :package_name: A list of names that specify the packages to promote.
-        (Found in jenkins console log when using gem to push package eg.
+    :package_name: Name of the release package.
+    :packages: A list of names that specify the packages to promote.
+        Found in jenkins console log when using gem to push package eg.

         "Pushing /path/of/package/name-of-package.rpm... success!"
         OR using rest api call to query packagecloud.io repo
         "curl https://packagecloud.io/api/v1/repos/test_user/test_repo/search?q=
         | yq -r .[].filename"
         "Pushing /path/of/package/name-of-package.rpm... success!"
         OR using rest api call to query packagecloud.io repo
         "curl https://packagecloud.io/api/v1/repos/test_user/test_repo/search?q=
         | yq -r .[].filename"

+    :ref: The git commit reference (SHA-1 code) to tag with the version string.
+    :log_dir: The suffix of the logs URL reported on completion by the
+        Jenkins merge job that created and pushed the distribution files
+        to the staging repository. For example, use value
+        "example-project-packagecloud-merge-/21" for the logs URL
+        https://logs.lf-project.org/production/vex-sjc-lfp-jenkins-prod-1/example-project-packagecloud-merge/21
+    :version: The semantic version string used for the package.

 
 

+:Optional Parameters:
+
+    :git_tag: The tag string to sign and push to the Git repository.
+       (default: the semantic version string)

 
 The JSON schema for a PackageCloud release file appears below.
 
 
 The JSON schema for a PackageCloud release file appears below.
 

diff --git a/jjb/lf-release-jobs.yaml b/jjb/lf-release-jobs.yaml
index 6be717d..f53a2d9 100644 (file)
--- a/jjb/lf-release-jobs.yaml
+++ b/jjb/lf-release-jobs.yaml
@@ -542,10 +542,20 @@
       - lf-infra-pre-build
       - config-file-provider:
           files:
       - lf-infra-pre-build
       - config-file-provider:
           files:

+            - file-id: sigul-config
+              variable: SIGUL_CONFIG
+            - file-id: sigul-password
+              variable: SIGUL_PASSWORD
+            - file-id: sigul-pki
+              variable: SIGUL_PKI
+            - file-id: signing-pubkey
+              variable: SIGNING_PUBKEY

             - file-id: "packagecloud_api"
               target: "$HOME/packagecloud_api"
             - file-id: packagecloud-account
               variable: ACCOUNT_NAME_FILE
             - file-id: "packagecloud_api"
               target: "$HOME/packagecloud_api"
             - file-id: packagecloud-account
               variable: ACCOUNT_NAME_FILE

+      - shell: !include-raw-escape: ../shell/sigul-configuration.sh
+      - shell: !include-raw-escape: ../shell/sigul-install.sh

       - lf-release
 
 - job-template:
       - lf-release
 
 - job-template:

diff --git a/releasenotes/notes/update-packagecloud-jobs-fc7510b516ab2bf5.yaml b/releasenotes/notes/update-packagecloud-jobs-fc7510b516ab2bf5.yaml
new file mode 100644 (file)
index 0000000..2698318
--- /dev/null
+++ b/releasenotes/notes/update-packagecloud-jobs-fc7510b516ab2bf5.yaml
@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Extend packagecloud release:
+      Release additionally requires log directory, ref and version.
+      Add set_variables_packagecloud & verify_packagecloud_match_release
+      to allow tagging the git repo.
+      Update packagecloud jobs to include sigul signing.

diff --git a/schema/release-packagecloud-schema.yaml b/schema/release-packagecloud-schema.yaml
index f17d174..8432f30 100644 (file)
--- a/schema/release-packagecloud-schema.yaml
+++ b/schema/release-packagecloud-schema.yaml
@@ -13,10 +13,24 @@ $id: "https://github.com/lfit/releng-global-jjb/blob/master/release-packagecloud
 
 required:
   - "package_name"
 
 required:
   - "package_name"

+  - "packages"
+  - "ref"
+  - "log_dir"
+  - "version"

 
 properties:
   package_name:
 
 properties:
   package_name:

+    type: "string"
+  packages:

     type: "array"
     properties:
       name:
         type: "string"
     type: "array"
     properties:
       name:
         type: "string"

+  git_tag:
+    type: "string"
+  ref:
+    type: "string"
+  log_dir:
+    type: "string"
+  version:
+    type: "string"

diff --git a/shell/release-job.sh b/shell/release-job.sh
index aa347e2..220f016 100644 (file)
--- a/shell/release-job.sh
+++ b/shell/release-job.sh
@@ -46,102 +46,102 @@ set_variables_common(){
     # Jenkins parameter drop-down defaults DISTRIBUTION_TYPE to None
     DISTRIBUTION_TYPE="${DISTRIBUTION_TYPE:-None}"
     if [[ $DISTRIBUTION_TYPE == "None" ]]; then
     # Jenkins parameter drop-down defaults DISTRIBUTION_TYPE to None
     DISTRIBUTION_TYPE="${DISTRIBUTION_TYPE:-None}"
     if [[ $DISTRIBUTION_TYPE == "None" ]]; then

-        DISTRIBUTION_TYPE="$(niet ".distribution_type" "$release_file")"
+        DISTRIBUTION_TYPE=$(niet ".distribution_type" "$release_file")

     fi
 
     fi
 

-    PATCH_DIR="$(mktemp -d)"
+    PATCH_DIR=$(mktemp -d)

 
     # Displaying Release Information (Common variables)
     printf "\t%-30s\n" RELEASE_ENVIRONMENT_INFO:
 
     # Displaying Release Information (Common variables)
     printf "\t%-30s\n" RELEASE_ENVIRONMENT_INFO:

-    printf "\t%-30s %s\n" RELEASE_FILE: $release_file
-    printf "\t%-30s %s\n" LOGS_SERVER: $LOGS_SERVER
-    printf "\t%-30s %s\n" NEXUS_PATH: $NEXUS_PATH
-    printf "\t%-30s %s\n" JENKINS_HOSTNAME: $JENKINS_HOSTNAME
-    printf "\t%-30s %s\n" SILO: $SILO
-    printf "\t%-30s %s\n" PROJECT: $PROJECT
-    printf "\t%-30s %s\n" PROJECT-DASHED: ${PROJECT//\//-}
-    printf "\t%-30s %s\n" DISTRIBUTION_TYPE: $DISTRIBUTION_TYPE
+    printf "\t%-30s %s\n" RELEASE_FILE: "$release_file"
+    printf "\t%-30s %s\n" LOGS_SERVER: "$LOGS_SERVER"
+    printf "\t%-30s %s\n" NEXUS_PATH: "$NEXUS_PATH"
+    printf "\t%-30s %s\n" JENKINS_HOSTNAME: "$JENKINS_HOSTNAME"
+    printf "\t%-30s %s\n" SILO: "$SILO"
+    printf "\t%-30s %s\n" PROJECT: "$PROJECT"
+    printf "\t%-30s %s\n" PROJECT-DASHED: "${PROJECT//\//-}"
+    printf "\t%-30s %s\n" DISTRIBUTION_TYPE: "$DISTRIBUTION_TYPE"

 }
 
 set_variables_maven(){
     echo "INFO: Setting maven variables"
     if [[ -z ${VERSION:-} ]]; then
 }
 
 set_variables_maven(){
     echo "INFO: Setting maven variables"
     if [[ -z ${VERSION:-} ]]; then

-        VERSION="$(niet ".version" "$release_file")"
+        VERSION=$(niet ".version" "$release_file")

     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then
     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then

-            GIT_TAG="$(niet ".git_tag" "$release_file")"
+            GIT_TAG=$(niet ".git_tag" "$release_file")

         else
             GIT_TAG="$VERSION"
         fi
     fi
     if [[ -z ${LOG_DIR:-} ]]; then
         else
             GIT_TAG="$VERSION"
         fi
     fi
     if [[ -z ${LOG_DIR:-} ]]; then

-        LOG_DIR="$(niet ".log_dir" "$release_file")"
+        LOG_DIR=$(niet ".log_dir" "$release_file")

     fi
     LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
     LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
 
     # Continuing displaying Release Information (Maven)
     printf "\t%-30s\n" RELEASE_MAVEN_INFO:
     fi
     LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
     LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
 
     # Continuing displaying Release Information (Maven)
     printf "\t%-30s\n" RELEASE_MAVEN_INFO:

-    printf "\t%-30s %s\n" VERSION: $VERSION
-    printf "\t%-30s %s\n" GIT_TAG: $GIT_TAG
-    printf "\t%-30s %s\n" LOG_DIR: $LOG_DIR
-    printf "\t%-30s %s\n" LOGS_URL: $LOGS_URL
+    printf "\t%-30s %s\n" VERSION: "$VERSION"
+    printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
+    printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
+    printf "\t%-30s %s\n" LOGS_URL: "$LOGS_URL"

 }
 
 set_variables_container(){
     echo "INFO: Setting container variables"
     if [[ -z ${VERSION:-} ]]; then
 }
 
 set_variables_container(){
     echo "INFO: Setting container variables"
     if [[ -z ${VERSION:-} ]]; then

-        VERSION="$(niet ".container_release_tag" "$release_file")"
+        VERSION=$(niet ".container_release_tag" "$release_file")

     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then
     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then

-            GIT_TAG="$(niet ".git_tag" "$release_file")"
+            GIT_TAG=$(niet ".git_tag" "$release_file")

         else
             GIT_TAG="$VERSION"
         fi
    fi
     if grep -q "container_pull_registry" "$release_file" ; then
         else
             GIT_TAG="$VERSION"
         fi
    fi
     if grep -q "container_pull_registry" "$release_file" ; then

-        CONTAINER_PULL_REGISTRY="$(niet ".container_pull_registry" "$release_file")"
+        CONTAINER_PULL_REGISTRY=$(niet ".container_pull_registry" "$release_file")

     fi
     if grep -q "container_push_registry" "$release_file" ; then
     fi
     if grep -q "container_push_registry" "$release_file" ; then

-        CONTAINER_PUSH_REGISTRY="$(niet ".container_push_registry" "$release_file")"
+        CONTAINER_PUSH_REGISTRY=$(niet ".container_push_registry" "$release_file")

     fi
     # Make sure both pull and push registries are defined
     if [ -z ${CONTAINER_PULL_REGISTRY+x} ] || [ -z ${CONTAINER_PUSH_REGISTRY+x} ]; then
         echo "ERROR: CONTAINER_PULL_REGISTRY and CONTAINER_PUSH_REGISTRY need to be defined"
         exit 1
     fi
     fi
     # Make sure both pull and push registries are defined
     if [ -z ${CONTAINER_PULL_REGISTRY+x} ] || [ -z ${CONTAINER_PUSH_REGISTRY+x} ]; then
         echo "ERROR: CONTAINER_PULL_REGISTRY and CONTAINER_PUSH_REGISTRY need to be defined"
         exit 1
     fi

-    ref="$(niet ".ref" "$release_file")"
+    ref=$(niet ".ref" "$release_file")

 
     # Continuing displaying Release Information (Container)
     printf "\t%-30s\n" RELEASE_CONTAINER_INFO:
 
     # Continuing displaying Release Information (Container)
     printf "\t%-30s\n" RELEASE_CONTAINER_INFO:

-    printf "\t%-30s %s\n" CONTAINER_RELEASE_TAG: $VERSION
-    printf "\t%-30s %s\n" CONTAINER_PULL_REGISTRY: $CONTAINER_PULL_REGISTRY
-    printf "\t%-30s %s\n" CONTAINER_PUSH_REGISTRY: $CONTAINER_PUSH_REGISTRY
-    printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: $ref
-    printf "\t%-30s %s\n" GIT_TAG: $GIT_TAG
+    printf "\t%-30s %s\n" CONTAINER_RELEASE_TAG: "$VERSION"
+    printf "\t%-30s %s\n" CONTAINER_PULL_REGISTRY: "$CONTAINER_PULL_REGISTRY"
+    printf "\t%-30s %s\n" CONTAINER_PUSH_REGISTRY: "$CONTAINER_PUSH_REGISTRY"
+    printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: "$ref"
+    printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"

 }
 
 set_variables_pypi(){
     echo "INFO: Setting pypi variables"
     if [[ -z ${LOG_DIR:-} ]]; then
 }
 
 set_variables_pypi(){
     echo "INFO: Setting pypi variables"
     if [[ -z ${LOG_DIR:-} ]]; then

-        LOG_DIR="$(niet ".log_dir" "$release_file")"
+        LOG_DIR=$(niet ".log_dir" "$release_file")

     fi
     LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
     LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
     if [[ -z ${PYPI_PROJECT:-} ]]; then
     fi
     LOGS_URL="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
     LOGS_URL=${LOGS_URL%/}  # strip any trailing '/'
     if [[ -z ${PYPI_PROJECT:-} ]]; then

-        PYPI_PROJECT="$(niet ".pypi_project" "$release_file")"
+        PYPI_PROJECT=$(niet ".pypi_project" "$release_file")

     fi
     if [[ -z ${PYTHON_VERSION:-} ]]; then
     fi
     if [[ -z ${PYTHON_VERSION:-} ]]; then

-        PYTHON_VERSION="$(niet ".python_version" "$release_file")"
+        PYTHON_VERSION=$(niet ".python_version" "$release_file")

     fi
     if [[ -z ${VERSION:-} ]]; then
     fi
     if [[ -z ${VERSION:-} ]]; then

-        VERSION="$(niet ".version" "$release_file")"
+        VERSION=$(niet ".version" "$release_file")

     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then
     fi
     if [[ -z ${GIT_TAG:-} ]]; then
         if grep -q "git_tag" "$release_file" ; then

-            GIT_TAG="$(niet ".git_tag" "$release_file")"
+            GIT_TAG=$(niet ".git_tag" "$release_file")

         else
             GIT_TAG="$VERSION"
         fi
         else
             GIT_TAG="$VERSION"
         fi


@@ -155,7 +155,39 @@ set_variables_pypi(){
     printf "\t%-30s %s\n" PYPI_PROJECT: "$PYPI_PROJECT"
     printf "\t%-30s %s\n" PYTHON_VERSION: "$PYTHON_VERSION"
     printf "\t%-30s %s\n" VERSION: "$VERSION"
     printf "\t%-30s %s\n" PYPI_PROJECT: "$PYPI_PROJECT"
     printf "\t%-30s %s\n" PYTHON_VERSION: "$PYTHON_VERSION"
     printf "\t%-30s %s\n" VERSION: "$VERSION"

-    printf "\t%-30s %s\n" GIT_TAG: $GIT_TAG
+    printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"
+}
+
+set_variables_packagecloud(){
+     echo "INFO: Setting packagecloud variables"
+     if [[ -z ${VERSION:-} ]]; then
+         VERSION=$(niet ".version" "$release_file")
+     fi
+     if [[ -z ${GIT_TAG:-} ]]; then
+         if grep -q "git_tag" $release_file ; then
+             GIT_TAG=$(niet ".git_tag" "$release_file")
+         else
+             GIT_TAG="$VERSION"
+         fi
+     fi
+     if [[ -z ${LOG_DIR:-} ]]; then
+         LOG_DIR=$(niet ".log_dir" "$release_file")
+     fi
+     if [[ -z ${REF:-} ]]; then
+         REF=$(niet ".ref" "$release_file")
+     fi
+     if [[ -z ${PACKAGE_NAME:-} ]]; then
+         PACKAGE_NAME=$(niet ".package_name" "$release_file")
+     fi
+     logs_url="${LOGS_SERVER}/${NEXUS_PATH}${LOG_DIR}"
+     logs_url=${logs_url%/}  # strip any trailing '/'
+
+     printf "\t%-30s %s\n" PACKAGE_NAME: "$PACKAGE_NAME"
+     printf "\t%-30s %s\n" LOG_DIR: "$LOG_DIR"
+     printf "\t%-30s %s\n" LOGS_URL: "$logs_url"
+     printf "\t%-30s %s\n" GERRIT_REF_TO_TAG: "$REF"
+     printf "\t%-30s %s\n" VERSION: "$VERSION"
+     printf "\t%-30s %s\n" GIT_TAG: "$GIT_TAG"

 }
 
 verify_schema(){
 }
 
 verify_schema(){


@@ -204,6 +236,18 @@ verify_pypi_match_release(){
     fi
 }
 
     fi
 }
 

+verify_packagecloud_match_release(){
+    echo "INFO: Fetching console log from $logs_url"
+    wget -q -P /tmp "${logs_url}/"console.log.gz
+    echo "INFO: Searching for uploaded step, package name $PACKAGE_NAME and version $VERSION in job log"
+    if zgrep "Successfully uploaded" /tmp/console.log.gz | grep "$PACKAGE_NAME" | grep "$VERSION"; then
+        echo "INFO: found expected strings in job log"
+    else
+        echo "ERROR: failed to find expected strings in job log"
+        exit 1
+    fi
+}
+

 # sigul is only available on Centos
 # TODO: write tag-github-repo function
 tag-gerrit-repo(){
 # sigul is only available on Centos
 # TODO: write tag-github-repo function
 tag-gerrit-repo(){


@@ -287,7 +331,7 @@ container_release_file(){
         else
             echo "INFO: $VERSION not found in releases, release will be prepared. Continuing..."
             docker pull "$CONTAINER_PULL_REGISTRY"/"$lfn_umbrella"/"$name":"$version"
         else
             echo "INFO: $VERSION not found in releases, release will be prepared. Continuing..."
             docker pull "$CONTAINER_PULL_REGISTRY"/"$lfn_umbrella"/"$name":"$version"

-            container_image_id="$(docker images | grep $name | grep $version | awk '{print $3}')"
+            container_image_id=$(docker images | grep "$name" | grep "$version" | awk '{print $3}')

             echo "INFO: Merge will run the following commands:"
             echo "docker tag $container_image_id $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
             echo "docker push $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
             echo "INFO: Merge will run the following commands:"
             echo "docker tag $container_image_id $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"
             echo "docker push $CONTAINER_PUSH_REGISTRY/$lfn_umbrella/$name:$VERSION"


@@ -391,6 +435,8 @@ packagecloud_promote(){
         destination="$2/release" "$promote_url" \
         | echo "INFO: Promoted package location: \
         https://packagecloud.io$(yq -r .package_html_url)"
         destination="$2/release" "$promote_url" \
         | echo "INFO: Promoted package location: \
         https://packagecloud.io$(yq -r .package_html_url)"

+    git checkout "$REF"
+    tag-gerrit-repo

 }
 
 ##############################  End Function Declarations  ################################
 }
 
 ##############################  End Function Declarations  ################################


@@ -444,17 +490,20 @@ case $DISTRIBUTION_TYPE in
         ;;
 
     packagecloud)
         ;;
 
     packagecloud)

-        release_schema="release-packagecloud-schema.yaml"
-        package_name=$(yq -r '.package_name' $release_file)
-        packagecloud_account=$(cat "$ACCOUNT_NAME_FILE")
-        echo "INFO: Fetching schema $release_schema"
-        wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
-        verify_schema
-        for name in $(yq -r '.package_name[].name' $release_file); do
-            package_name=$name
-            packagecloud_verify "$package_name" "$packagecloud_account"
+        if $USE_RELEASE_FILE ; then
+            release_schema="release-packagecloud-schema.yaml"
+            packagecloud_account=$(cat "$ACCOUNT_NAME_FILE")
+            echo "INFO: Fetching schema $release_schema"
+            wget -q https://raw.githubusercontent.com/lfit/releng-global-jjb/master/schema/${release_schema}
+            verify_schema
+        fi
+        set_variables_packagecloud
+        verify_packagecloud_match_release
+        for name in $(yq -r '.packages[].name' $release_file); do
+            package=$name
+            packagecloud_verify "$package" "$packagecloud_account"

             if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then
             if [[ "$JOB_NAME" =~ "merge" ]] && ! $DRY_RUN; then

-                packagecloud_promote "$package_name" "$packagecloud_account"
+                packagecloud_promote "$package" "$packagecloud_account"

             fi
         done
         ;;
             fi
         done
         ;;