From d60a07de920a48e44fa34242c774e3a9f04250dc Mon Sep 17 00:00:00 2001 From: Aric Gardner Date: Mon, 9 Sep 2019 14:25:50 -0400 Subject: [PATCH] Readthedocs v2 jobs This change will need a new release of lftools before It will work. Global job that triggers on any docs changes. Creates docs project in rtd if absent on merge Creates subproject association if project does not equal Jenkins global var MASTER_RTD_PROJECT Triggers docs build on merge fix tox job to push docs-dir to archives so that we can see generated docs ISSUE: RELENG-2362 Signed-off-by: Aric Gardner Change-Id: Id6d85068eb068301b91d4deb74490e76381a7e54 --- docs/jjb/lf-macros.rst | 11 ++ docs/jjb/lf-rtdv2-jobs.rst | 188 +++++++++++++++++++++ jjb/lf-macros.yaml | 9 + jjb/lf-rtdv2-job-groups.yaml | 7 + jjb/lf-rtdv2-jobs.yaml | 153 +++++++++++++++++ .../notes/lf-rtdv2-jobs-2885568080d2990a.yaml | 6 + shell/rtdv2.sh | 89 ++++++++++ shell/tox-run.sh | 8 + 8 files changed, 471 insertions(+) create mode 100644 docs/jjb/lf-rtdv2-jobs.rst create mode 100644 jjb/lf-rtdv2-job-groups.yaml create mode 100644 jjb/lf-rtdv2-jobs.yaml create mode 100644 releasenotes/notes/lf-rtdv2-jobs-2885568080d2990a.yaml create mode 100644 shell/rtdv2.sh diff --git a/docs/jjb/lf-macros.rst b/docs/jjb/lf-macros.rst index ff8c561a..6404cc9f 100644 --- a/docs/jjb/lf-macros.rst +++ b/docs/jjb/lf-macros.rst @@ -259,6 +259,17 @@ ReadTheDocs verify script. Installs and runs tox. :doc-dir: Document directory. :python-version: Python version. +lf-rtdv2-build +--------------- + +ReadTheDocs Version 2 verify and merge scripts. +This is an update to the read the docs scripts that leverages the new Read the Docs v3 api +`RTD v3 API `_ +Runs tox to verify that the docs are good and then runs the RTDv2 shell script. +This script handles creating projects as needed, assiging subprojects to the main +read the docs project and triggering builds to update the documentation. + + check-info-votes ---------------- diff --git a/docs/jjb/lf-rtdv2-jobs.rst b/docs/jjb/lf-rtdv2-jobs.rst new file mode 100644 index 00000000..c2fd2cd7 --- /dev/null +++ b/docs/jjb/lf-rtdv2-jobs.rst @@ -0,0 +1,188 @@ +.. _lf-global-jjb-rtdv2-jobs: + +########################## +ReadTheDocs Version:2 Jobs +########################## + +This is a global job that only needs to be added once to your project's ci-mangement git repository. It leverages the read the docs v3 api to create projects on the fly, as well as set up subproject associations with the master doc. + +The master doc must be defined in +jenkins-config/global-vars-{production|sandbox}.sh + +examples: +global-vars-sandbox.sh: +MASTER_RTD_PROJECT=doc-test +global-vars-production.sh: +MASTER_RTD_PROJECT=doc + +In this way sandbox jobs will create docs with a test suffix and will not stomp on production documentation. + +Example job config: + +example file: ci-management/jjb/rtd/rtd.yaml + +.. code-block:: bash + + --- + - project: + name: rtdv2-verify-global + build-node: centos7-builder-1c-1g + jobs: + - rtdv2-verify-global + stream: + - master: + branch: master + - foo: + branch: stable/{stream} + + - project: + name: rtdv2-merge-global + build-node: centos7-builder-1c-1g + jobs: + - rtdv2-merge-global + stream: + - master: + branch: master + - foo: + branch: stable/{stream} + +Or add both jobs via a job group: + + +.. code-block:: bash + + --- + - project: + name: rtdv2-global + build-node: centos7-builder-1c-1g + jobs: + - rtdv2-global + stream: + - master: + branch: master + + +Github jobs must be per project, and will be covered by a diffrent set of jobs once these are proven. + +Job requires an lftools config section, this is to provide api access to read the docs. + +.. code-block:: bash + + [rtd] + endpoint = https://readthedocs.org/api/v3/ + token = [hidden] + +Merge Job will create a project on read the docs if none exist. +Merge Job will assign a project as a subproject of the master project. +Merge job will trigger a build to update docs. + +Macros +====== + +lf-rtdv2-common +--------------- + +RTD verify and merge jobs are the same except for their scm, trigger, and +builders definition. This anchor is the common template. + + +Job Templates +============= + +ReadTheDocs v2 Merge +-------------------- + +Merge job which triggers a build of the docs to readthedocs. + +:Template Names: + - rtdv2-merge-global-{stream} + +:Comment Trigger: remerge + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + +:Optional parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :project-pattern: Project to trigger build against. (default: \*\*) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :disable-job: Whether to disable the job (default: false) + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + + :gerrit_merge_triggers: Override Gerrit Triggers. + :gerrit_trigger_file_paths: Override file paths filter which checks which + file modifications will trigger a build. + **default**:: + + - compare-type: REG_EXP + pattern: '.*\.css' + - compare-type: REG_EXP + pattern: '.*\.html' + - compare-type: REG_EXP + pattern: '.*\.rst' + - compare-type: REG_EXP + pattern: '.*\/conf.py' + + + +ReadTheDocs v2 Verify +--------------------- + +Verify job which runs a tox build of the docs project. +Also outputs some info on the build + +:Template Names: + - rtdv2-verify-global-{stream} + +:Comment Trigger: recheck|reverify + +:Required Parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally set + in defaults.yaml) + +:Optional Parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in minutes before aborting build. (default: 15) + :gerrit-skip-vote: Skip voting for this job. (default: false) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :disable-job: Whether to disable the job (default: false) + :project-pattern: Project to trigger build against. (default: \*\*) + :stream: Keyword representing a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :submodule-timeout: Timeout (in minutes) for checkout operation. + (default: 10) + :submodule-disable: Disable submodule checkout operation. + (default: false) + + :gerrit_verify_triggers: Override Gerrit Triggers. + :gerrit_trigger_file_paths: Override file paths filter which checks which + file modifications will trigger a build. + **default**:: + + - compare-type: REG_EXP + pattern: '.*\.css' + - compare-type: REG_EXP + pattern: '.*\.html' + - compare-type: REG_EXP + pattern: '.*\.rst' + - compare-type: REG_EXP + pattern: '.*\/conf.py' diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml index c06f0ce2..2fe9f228 100644 --- a/jjb/lf-macros.yaml +++ b/jjb/lf-macros.yaml @@ -274,6 +274,15 @@ - shell: !include-raw-escape: ../shell/tox-install.sh - shell: !include-raw-escape: ../shell/rtd-verify.sh +- builder: + name: lf-rtdv2-build + builders: + - lf-infra-pre-build + - lf-infra-tox-install: + python-version: "python3" + - shell: !include-raw: ../shell/tox-run.sh + - shell: !include-raw: ../shell/rtdv2.sh + - builder: name: check-info-votes builders: diff --git a/jjb/lf-rtdv2-job-groups.yaml b/jjb/lf-rtdv2-job-groups.yaml new file mode 100644 index 00000000..fdce6242 --- /dev/null +++ b/jjb/lf-rtdv2-job-groups.yaml @@ -0,0 +1,7 @@ +--- +- job-group: + name: "rtdv2-global" + + jobs: + - rtdv2-verify-global + - rtdv2-merge-global diff --git a/jjb/lf-rtdv2-jobs.yaml b/jjb/lf-rtdv2-jobs.yaml new file mode 100644 index 00000000..326c3638 --- /dev/null +++ b/jjb/lf-rtdv2-jobs.yaml @@ -0,0 +1,153 @@ +--- +- lf_rtdv2_common: &lf_rtdv2_common + name: lf-rtdv2-common + + ###################### + # Default parameters # + ###################### + + branch: master + stream: master + disabled: "{disable-job}" + build-days-to-keep: 7 + build-timeout: 15 + disable-job: false + git-url: "$GIT_URL/$PROJECT" + submodule-recursive: true + submodule-disable: false + submodule-timeout: 10 + gerrit-skip-vote: false + + gerrit_trigger_file_paths: + - compare-type: REG_EXP + pattern: '.*\.css' + - compare-type: REG_EXP + pattern: '.*\.html' + - compare-type: REG_EXP + pattern: '.*\.rst' + - compare-type: REG_EXP + pattern: '.*\/conf.py' + + ##################### + # Job Configuration # + ##################### + + project-type: freestyle + node: "{build-node}" + + properties: + - lf-infra-properties: + build-days-to-keep: 7 + + parameters: + - lf-infra-parameters: + project: $GERRIT_PROJECT + branch: "{branch}" + refspec: "refs/heads/{branch}" + stream: "{stream}" + + wrappers: + - lf-infra-wrappers: + build-timeout: "{build-timeout}" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + + publishers: + - lf-infra-publish + + builders: + - config-file-provider: + files: + - file-id: lftoolsini + target: "$HOME/.config/lftools/lftools.ini" + - lf-rtdv2-build + +- job-template: + name: "rtdv2-verify-global-{stream}" + id: rtdv2-verify-global + <<: *lf_rtdv2_common + # yamllint disable-line rule:key-duplicates + + stream: master + + gerrit_verify_triggers: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+(recheck|reverify)\s*$' + + scm: + - lf-infra-gerrit-scm: + branch: "$GERRIT_BRANCH" + jenkins-ssh-credential: "{jenkins-ssh-credential}" + git-url: "{git-url}" + refspec: "$GERRIT_REFSPEC" + submodule-recursive: "{submodule-recursive}" + submodule-timeout: "{submodule-timeout}" + submodule-disable: "{submodule-disable}" + choosing-strategy: gerrit + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_verify_triggers}" + projects: + - project-compare-type: "REG_EXP" + project-pattern: "**" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" + skip-vote: + successful: "{gerrit-skip-vote}" + failed: "{gerrit-skip-vote}" + unstable: "{gerrit-skip-vote}" + notbuilt: "{gerrit-skip-vote}" + +- job-template: + name: "rtdv2-merge-global-{stream}" + id: rtdv2-merge-global + <<: *lf_rtdv2_common + + ###################### + # Default parameters # + ###################### + + branch: "{branch}" + + gerrit_merge_triggers: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: '^Patch Set\s+\d+:\s+remerge\s*$' + - ref-updated-event + + ##################### + # Job Configuration # + ##################### + + disabled: "{disable-job}" + + scm: + - lf-infra-gerrit-scm: + jenkins-ssh-credential: "{jenkins-ssh-credential}" + refspec: "$GERRIT_REFSPEC" + branch: "$GERRIT_BRANCH" + git-url: "{git-url}" + submodule-disable: true + submodule-recursive: false + submodule-timeout: 10 + choosing-strategy: default + + triggers: + - gerrit: + server-name: "{gerrit-server-name}" + trigger-on: "{obj:gerrit_merge_triggers}" + projects: + - project-compare-type: "REG_EXP" + project-pattern: "**" + branches: + - branch-compare-type: "ANT" + branch-pattern: "**/{branch}" + file-paths: "{obj:gerrit_trigger_file_paths}" diff --git a/releasenotes/notes/lf-rtdv2-jobs-2885568080d2990a.yaml b/releasenotes/notes/lf-rtdv2-jobs-2885568080d2990a.yaml new file mode 100644 index 00000000..aef5ced0 --- /dev/null +++ b/releasenotes/notes/lf-rtdv2-jobs-2885568080d2990a.yaml @@ -0,0 +1,6 @@ +--- +feature: > + Global job that triggers on any docs changes. + Creates docs project if absent + Creates subproject association with master doc project + Triggers docs build diff --git a/shell/rtdv2.sh b/shell/rtdv2.sh new file mode 100644 index 00000000..714821b4 --- /dev/null +++ b/shell/rtdv2.sh @@ -0,0 +1,89 @@ +#!/bin/bash -l +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2017 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo "---> rtd-verify.sh" +set -euo pipefail + +project_dashed="${PROJECT////-}" +umbrella="$(echo "$GERRIT_URL" | awk -F"." '{print $2}')" +if [[ "$SILO" == "sandbox" ]]; then + rtdproject="$umbrella-$project_dashed-test" +else + rtdproject="$umbrella-$project_dashed" +fi + +#MASTER_RTD_PROJECT as a global jenkins cnt +masterproject="$umbrella-$MASTER_RTD_PROJECT" + +echo "INFO:" +echo "INFO: Project: $PROJECT" +echo "INFO: Read the Docs Project: https://$rtdproject.readthedocs.io" +echo "INFO: Read the Docs master Project: https://$masterproject.readthedocs.io" + + +if [[ "$JOB_NAME" =~ "verify" ]]; then + if [[ "$(lftools rtd project-details "$rtdproject" | yq -r '.detail')" == "Not found." ]]; then + echo "INFO: Project not found, merge will create project https://$rtdproject.readthedocs.io" + fi + +echo "INFO: Merge will run" +echo "INFO: lftools rtd project-build-trigger $rtdproject $STREAM" + +fi + +if [[ "$JOB_NAME" =~ "merge" ]]; then + + # This retuns null if project exists. + project_exists=false + project_created=false + + declare -i cnt=0 + while [[ $project_exists == "false" ]]; do + if [[ "$(lftools rtd project-details "$rtdproject" | yq -r '.detail')" == "Not found." ]]; then + echo "INFO: Project not found" + if [[ $project_created == "false" ]]; then + echo "INFO: Creating project https://$rtdproject.readthedocs.io" + lftools rtd project-create "$rtdproject" "$GERRIT_URL/$PROJECT" git "https://$rtdproject.readthedocs.io" py en + project_created="true" + fi + echo "INFO sleeping for 30 seconds $cnt times" + sleep 30 + ((cnt+=1)) + if (( cnt >= 20 )); then + echo "INFO: Job has timed out" + exit 1 + fi + else + echo "INFO: Project exists in read the docs as https://$rtdproject.readthedocs.io" + project_exists="true" + fi + done + + if [[ "$rtdproject" != "$masterproject" ]]; then + subproject_exists=false + while read -r subproject; do + if [[ "$subproject" == "$rtdproject" ]]; then + subproject_exists=true + break + fi + done < <(lftools rtd subproject-list "$masterproject") + + if $subproject_exists; then + echo "INFO: subproject relationship already created" + else + echo "INFO: Need to create subproject relationship" + lftools rtd subproject-create "$masterproject" "$rtdproject" + echo "INFO sleeping for 10 seconds" + sleep 10 + fi + fi + + lftools rtd project-build-trigger "$rtdproject" "$STREAM" +fi diff --git a/shell/tox-run.sh b/shell/tox-run.sh index 8be8459f..e87c98f2 100644 --- a/shell/tox-run.sh +++ b/shell/tox-run.sh @@ -51,4 +51,12 @@ done echo "Completed tox runs." + +# If docs are generated push them to archives. +DOC_DIR="${DOC_DIR:-docs/_build/html}" +if [[ -d "$DOC_DIR" ]]; then + echo "---> Archiving generated docs" + mv "$DOC_DIR" archives/ +fi + test "$tox_status" -eq 0 || exit "$tox_status" -- 2.16.6