Merge "Allow JJB Cache directory to be configured"
authorThanh Ha (zxiiro) <thanh.ha@linuxfoundation.org>
Thu, 15 Nov 2018 02:38:46 +0000 (02:38 +0000)
committerGerrit Code Review <gerrit@linuxfoundation.org>
Thu, 15 Nov 2018 02:38:46 +0000 (02:38 +0000)
1  2 
docs/jjb/lf-ci-jobs.rst
jjb/lf-ci-jobs.yaml

diff --combined docs/jjb/lf-ci-jobs.rst
@@@ -21,6 -21,8 +21,8 @@@ lf-infra-jjb-parameter
  
  :Required Parameters:
  
+     :jjb-cache: Location of Jenkins Job Builder (JJB) cache used for jjb
+         jobs.
      :jjb-version: Version of Jenkins Job Builder (JJB) to install and use in
          the jjb jobs.
  
@@@ -282,33 -284,6 +284,33 @@@ Troubleshootin
      used to push to Jenkins. In the event of a job failure this file can be
      inspected.
  
 + .. _lf-global-jjb-jenkins-cfg-verify:
 +
 +Jenkins Configuration Verify
 +----------------------------
 +
 +Jenkins job to verify the Global Jenkins configuration.
 +
 +Requires the ``clouds-yaml`` file to be setup on the Jenkins host.
 +
 +:Template names:
 +
 +    - {project-name}-jenkins-cfg-verify
 +    - gerrit-jenkins-cfg-verify
 +    - github-jenkins-cfg-verify
 +
 +:Optional parameters:
 +
 +    :branch: Git branch to build against. (default: master)
 +    :git-url: URL to clone project from. (default: $GIT_URL/$GERRIT_PROJECT)
 +
 +This job is not part of the "{project-name}-ci-jobs" group. It must be called
 +explicitly.
 +
 +Example:
 +
 +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/jenkins-cfg-verify.yaml
 +   :language: yaml
  
  Jenkins Sandbox Cleanup
  -----------------------
@@@ -392,6 -367,7 +394,7 @@@ Runs `jenkins-jobs update` to update pr
      :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
      :build-timeout: Timeout in minutes before aborting build. (default: 10)
      :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+     :jjb-cache: JJB cache location. (default: $HOME/.cache/jenkins_jobs)
      :jjb-version: JJB version to install. (default: see job-template)
      :stream: Keyword that can be used to represent a release code-name.
          Often the same as the branch. (default: master)
@@@ -430,6 -406,7 +433,7 @@@ Runs `jenkins-jobs test` to validate JJ
      :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
      :build-timeout: Timeout in minutes before aborting build. (default: 10)
      :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+     :jjb-cache: JJB cache location. (default: $HOME/.cache/jenkins_jobs)
      :jjb-version: JJB version to install. (default: see job-template)
      :stream: Keyword that can be used to represent a release code-name.
          Often the same as the branch. (default: master)
@@@ -470,6 -447,7 +474,7 @@@ project level jobs
      :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
      :build-timeout: Timeout in minutes before aborting build. (default: 10)
      :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
+     :jjb-cache: JJB cache location. (default: $HOME/.cache/jenkins_jobs)
      :jjb-version: JJB version to install. (default: see job-template)
      :stream: Keyword that can be used to represent a release code-name.
          Often the same as the branch. (default: master)
@@@ -560,29 -538,16 +565,29 @@@ containing the credentials for the clou
      :build-node: The node to run build on.
      :jenkins-ssh-credential: Credential to use for SSH. (Generally should
          be configured in defaults.yaml)
 +    :jenkins-urls: URLs to Jenkins systems to check for active builds.
  
  :Optional parameters:
  
      :branch: Git branch to fetch for the build. (default: master)
      :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7)
      :build-timeout: Timeout in minutes before aborting build. (default: 90)
 -    :cron: Time when the packer image should be rebuilt (default: @daily)
 +    :cron: Time when the packer image should be rebuilt (default: @hourly)
      :git-url: URL clone project from. (default: $GIT_URL/$PROJECT)
      :openstack-cloud: OS_CLOUD setting to pass to openstack client.
          (default: vex)
 +    :openstack-image-cleanup: Whether or not to run the image cleanup script.
 +        (default: true)
 +    :openstack-image-cleanup-age: Age in days of image before marking it for
 +        removal. (default: 30)
 +    :openstack-image-protect: Whether or not to run the image protect script.
 +        (default: true)
 +    :openstack-server-cleanup: Whether or not to run the server cleanup script.
 +        (default: true)
 +    :openstack-stack-cleanup: Whether or not to run the stack cleanup script.
 +        (default: true)
 +    :openstack-volume-cleanup: Whether or not to run the volume cleanup script.
 +        (default: true)
      :stream: Keyword that can be used to represent a release code-name.
          Often the same as the branch. (default: master)
      :submodule-recursive: Whether to checkout submodules recursively.
      :submodule-timeout: Timeout (in minutes) for checkout operation.
          (default: 10)
  
 +Minimal Example:
 +
 +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-cron-minimal.yaml
 +
 +Full Example:
 +
 +.. literalinclude:: ../../.jjb-test/lf-ci-jobs/openstack-cron-full.yaml
 +
 +
  .. _gjjb-packer-merge:
  
  Packer Merge
diff --combined jjb/lf-ci-jobs.yaml
            name: JJB_VERSION
            default: '{jjb-version}'
            description: Jenkins Job Builder version to download and install.
+       - string:
+           name: XDG_CACHE_HOME
+           default: '{jjb-cache}'
+           description: Jenkins Job Builder cache location.
  
  - parameter:
      name: lf-packer-parameters
      build-timeout: 10
      disable-job: false
      github-url: 'https://github.com'
+     jjb-cache: '$HOME/.cache/jenkins_jobs'
      jjb-version: 2.2.1
      stream: master
      submodule-recursive: true
            branch: '{branch}'
            lftools-version: '{lftools-version}'
        - lf-infra-jjb-parameters:
+           jjb-cache: '{jjb-cache}'
            jjb-version: '{jjb-version}'
  
      wrappers:
            white-list-target-branches:
              - '{branch}'
  
 +###########################################
 +# Jenkins Configuration Management Verify #
 +###########################################
 +
 +- lf_jenkins_configuration: &lf_jenkins_cfg_verify
 +    name: lf-jenkins-cfg-verify
 +
 +    ######################
 +    # Default parameters #
 +    ######################
 +
 +    branch: master
 +    build-days-to-keep: 7
 +    disable-job: false
 +    git-url: '$GIT_URL/$GERRIT_PROJECT'
 +    github-url: 'https://github.com'
 +    openstack-cloud: vex
 +    submodule-timeout: 10
 +
 +    gerrit_verify_triggers:
 +      - patchset-created-event:
 +          exclude-drafts: true
 +          exclude-trivial-rebase: false
 +          exclude-no-code-change: false
 +      - draft-published-event
 +      - comment-added-contains-event:
 +          comment-contains-value: '^Patch Set[ ]+[0-9]+:([ ]+|[\n]+)(recheck|reverify)$'
 +
 +    gerrit_trigger_file_paths:
 +      - compare-type: REG_EXP
 +        pattern: 'jenkins-config\/.*'
 +
 +    # github_included_regions MUST match gerrit_trigger_file_paths
 +    github_included_regions:
 +      - 'jenkins-config\/.*'
 +
 +    #####################
 +    # Job Configuration #
 +    #####################
 +
 +    project-type: freestyle
 +    node: '{build-node}'
 +    concurrent: true
 +    disabled: '{disable-job}'
 +
 +    properties:
 +      - lf-infra-properties:
 +          project: '{project}'
 +          build-days-to-keep: 1
 +
 +    parameters:
 +      - lf-infra-parameters:
 +          project: '{project}'
 +          stream: ''
 +          branch: master
 +          lftools-version: '{lftools-version}'
 +
 +    wrappers:
 +      - lf-infra-wrappers:
 +          build-timeout: 10
 +          jenkins-ssh-credential: '{jenkins-ssh-credential}'
 +      - config-file-provider:
 +          # Listed after to override openstack-infra-wrappers clouds.yaml
 +          # definition
 +          files:
 +            - file-id: clouds-yaml
 +              target: '$HOME/.config/openstack/clouds.yaml'
 +            - file-id: npmrc
 +              target: '$HOME/.npmrc'
 +            - file-id: pipconf
 +              target: '$HOME/.config/pip/pip.conf'
 +
 +    builders:
 +      - lf-infra-pre-build
 +      - inject:
 +          properties-content: OS_CLOUD={openstack-cloud}
 +      - shell: !include-raw-escape: ../shell/jenkins-verify-images.sh
 +
 +    publishers:
 +      - lf-infra-publish
 +
 +- job-template:
 +    name: '{project-name}-jenkins-cfg-verify'
 +    id: gerrit-jenkins-cfg-verify
 +    <<: *lf_jenkins_cfg_verify
 +
 +    scm:
 +      - lf-infra-gerrit-scm:
 +          git-url: '{git-url}'
 +          refspec: 'refs/heads/{branch}'
 +          branch: '{branch}'
 +          submodule-recursive: true
 +          submodule-timeout: '{submodule-timeout}'
 +          choosing-strategy: default
 +          jenkins-ssh-credential: '{jenkins-ssh-credential}'
 +
 +    triggers:
 +      - gerrit:
 +          server-name: '{gerrit-server-name}'
 +          trigger-on: '{obj:gerrit_verify_triggers}'
 +          projects:
 +            - project-compare-type: ANT
 +              project-pattern: '{project}'
 +              branches:
 +                - branch-compare-type: ANT
 +                  branch-pattern: '**/{branch}'
 +              file-paths: '{obj:gerrit_trigger_file_paths}'
 +
 +- job-template:
 +    name: '{project-name}-jenkins-cfg-verify'
 +    id: github-jenkins-cfg-verify
 +    <<: *lf_jenkins_cfg_verify
 +
 +    properties:
 +      - github:
 +          url: '{github-url}/{github-org}/{project}'
 +
 +    scm:
 +      - lf-infra-github-scm:
 +          url: '{git-clone-url}{github-org}/{project}'
 +          refspec: '+refs/pull/*:refs/remotes/origin/pr/*'
 +          branch: '$sha1'
 +          submodule-recursive: '{submodule-recursive}'
 +          submodule-timeout: '{submodule-timeout}'
 +          choosing-strategy: default
 +          jenkins-ssh-credential: '{jenkins-ssh-credential}'
 +
 +    triggers:
 +      - github-pull-request:
 +          trigger-phrase: '^(recheck|reverify)$'
 +          only-trigger-phrase: false
 +          status-context: 'Jenkins CFG Verify'
 +          permit-all: true
 +          github-hooks: true
 +          included-regions: '{obj:github_included_regions}'
 +          white-list-target-branches:
 +            - '{branch}'
  
  ###########################
  # JENKINS SANDBOX CLEANUP #
  
      parameters:
        - lf-infra-jjb-parameters:
+           jjb-cache: '{jjb-cache}'
            jjb-version: '{jjb-version}'
  
      triggers:
      branch: master
      build-days-to-keep: 7
      build-timeout: 10
 -    cron: '@daily'
 +    cron: '@hourly'
      disable-job: false
      git-url: '$GIT_URL/$PROJECT'
      github-url: 'https://github.com'
      openstack-cloud: vex
 +    openstack-image-cleanup: true
 +    openstack-image-cleanup-age: 30
 +    openstack-image-protect: true
 +    openstack-server-cleanup: true
 +    openstack-stack-cleanup: true
 +    openstack-volume-cleanup: true
      stream: master
      submodule-timeout: 10
  
            stream: '{stream}'
            branch: '{branch}'
            lftools-version: '{lftools-version}'
 +      - string:
 +          name: JENKINS_URLS
 +          default: '{jenkins-urls}'
 +          description: |
 +              Space separated list of Jenkins URLs to check for active builds.
  
      wrappers:
        - lf-infra-wrappers:
        - timed: '{obj:cron}'
  
      builders:
 +      - lf-infra-pre-build
        - inject:
            properties-content: OS_CLOUD={openstack-cloud}
 -      - shell: !include-raw-escape: ../shell/openstack-install.sh
 -      - shell: !include-raw-escape: ../shell/openstack-protect-in-use-images.sh
 +      # Stacks
 +      - conditional-step:
 +          condition-kind: boolean-expression
 +          condition-expression: '{openstack-stack-cleanup}'
 +          steps:
 +            - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-stacks.sh
 +      # Servers
 +      - conditional-step:
 +          condition-kind: boolean-expression
 +          condition-expression: '{openstack-server-cleanup}'
 +          steps:
 +            - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-servers.sh
 +      # Volumes
 +      - conditional-step:
 +          condition-kind: boolean-expression
 +          condition-expression: '{openstack-volume-cleanup}'
 +          steps:
 +            - shell: !include-raw-escape: ../shell/openstack-cleanup-orphaned-volumes.sh
 +      # Images
 +      - conditional-step:
 +          condition-kind: boolean-expression
 +          condition-expression: '{openstack-image-protect}'
 +          steps:
 +            - shell: !include-raw-escape: ../shell/openstack-protect-in-use-images.sh
 +      - conditional-step:
 +          condition-kind: boolean-expression
 +          condition-expression: '{openstack-image-cleanup}'
 +          steps:
 +            - inject:
 +                properties-content: OS_IMAGE_CLEANUP_AGE={openstack-image-cleanup-age}
 +            - shell: !include-raw-escape: ../shell/openstack-cleanup-old-images.sh
  
      publishers:
        - lf-infra-publish