From: Jessica Wagantall Date: Fri, 1 Jun 2018 03:02:38 +0000 (-0700) Subject: Add INFO.yaml verify jobs X-Git-Tag: v0.21.0~3 X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F24%2F10924%2F31;p=releng%2Fglobal-jjb.git Add INFO.yaml verify jobs Add '{project-name}-info-yaml-jobs' and '{project-name}-github-info-yaml-jobs' to validate that INFO.yaml file changes are kept isolated from other file changes. The addition of or changes to an INFO.yaml should be kept on its own gerrit change and should not be combined with other changes. Issue: RELENG-973 Change-Id: Id7ddb55c34a741d92b1e126f1237fb9753a4aa75 Signed-off-by: Jessica Wagantall Co-Authored-By: Trevor Bramwell --- diff --git a/docs/jjb/lf-ci-jobs.rst b/docs/jjb/lf-ci-jobs.rst index f4e11d1d..70a6fa21 100644 --- a/docs/jjb/lf-ci-jobs.rst +++ b/docs/jjb/lf-ci-jobs.rst @@ -29,6 +29,24 @@ Recommended jobs that should be deployed CI using GitHub. - github-jjb-merge - github-jjb-verify +{project-name}-info-yaml-jobs +----------------------------- + +Jobs to verify INFO.yaml file changes. + +:Includes: + + - gerrit-info-yaml-verify + +{project-name}-github-info-yaml-jobs +------------------------------------ + +Jobs to verify INFO.yaml file changes using Github. + +:Includes: + + - github-info-yaml-verify + {project-name}-packer-jobs -------------------------- @@ -376,6 +394,37 @@ Runs `jenkins-jobs test` to validate JJB syntax filter which file modifications will trigger a build. (default defined by lf_jjb_common) +.. _info-yaml-verify: + +Info YAML Verify +---------------- + +Info YAML Verify job validates that INFO.yaml file changes are kept isolated from +other file changes. Verifies INFO.yaml files follow the schema defined in +`global-jjb/info-schema`. + +:Template Names: + - {project-name}-info-yaml-verify + - gerrit-info-yaml-verify + - github-info-yaml-verify + +:Required parameters: + + :build-node: The node to run build on. + :jenkins-ssh-credential: Credential to use for SSH. (Generally should + be configured in defaults.yaml) + +:Optional parameters: + + :branch: Git branch to fetch for the build. (default: master) + :build-days-to-keep: Days to keep build logs in Jenkins. (default: 7) + :build-timeout: Timeout in seconds before aborting build. (default: 10) + :git-url: URL clone project from. (default: $GIT_URL/$PROJECT) + :stream: Keyword that can be used to represent a release code-name. + Often the same as the branch. (default: master) + :submodule-recursive: Whether to checkout submodules recursively. + (default: true) + :gerrit_verify_triggers: Override Gerrit Triggers. .. _gjjb-packer-merge: diff --git a/info-schema b/info-schema new file mode 100644 index 00000000..c98a4708 --- /dev/null +++ b/info-schema @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +--- +$schema: 'http://json-schema.org/schema#' +$id: 'https://github.com/lfit/releng-global-jjb/blob/master/info-schema' + +required: + - 'project' + - 'version' + - 'project_creation_date' + - 'project_category' + - 'lifecycle_state' + - 'project_lead' + - 'primary_contact' + - 'issue_tracking' + - 'mailing_list' + - 'realtime_discussion' + - 'repositories' + - 'committers' + - 'tsc' + +properties: + project: + type: 'string' + version: + type: 'string' + project_creation_date: + type: 'string' + project_category: + type: 'string' + lifecycle_state: + type: 'string' + enum: ['Incubation', 'Proposal', 'Mature', 'Core', 'Top Level', 'Archived', 'Null', 'Integration'] + project_lead: &user_object + type: 'object' + properties: + name: + type: 'string' + email: + type: 'string' + format: 'email' + id: + type: 'string' + company: + type: 'string' + timezone: + type: 'string' + pattern: '([A-Z][a-z]*\s*)+\/([A-Za-z_-])+' + additionalProperties: false + primary_contact: *user_object + issue_tracking: + type: 'object' + properties: + type: + type: 'string' + url: + type: 'string' + format: 'uri' + key: + type: 'string' + required: ['type', 'url'] + committers: + type: 'array' + items: *user_object diff --git a/jjb/lf-ci-jobs.yaml b/jjb/lf-ci-jobs.yaml index a01ce2bd..a5f77223 100644 --- a/jjb/lf-ci-jobs.yaml +++ b/jjb/lf-ci-jobs.yaml @@ -17,6 +17,18 @@ - github-jjb-merge - github-jjb-verify +- job-group: + name: '{project-name}-info-yaml-jobs' + + jobs: + - gerrit-info-yaml-verify + +- job-group: + name: '{project-name}-github-info-yaml-jobs' + + jobs: + - github-info-yaml-verify + - job-group: name: '{project-name}-packer-jobs' @@ -700,6 +712,99 @@ github-hooks: true included-regions: '{obj:github_included_regions}' +#################### +# Info YAML Verify # +#################### + +- lf_info_yaml_verify: &lf_info_yaml_verify + name: lf-info-yaml-verify + + ###################### + # Default parameters # + ###################### + + gerrit_verify_triggers: + - patchset-created-event: + exclude-drafts: true + exclude-trivial-rebase: false + exclude-no-code-change: false + - draft-published-event + - comment-added-contains-event: + comment-contains-value: recheck$ + + ##################### + # Job Configuration # + ##################### + + concurrent: true + + builders: + - shell: !include-raw-escape: + - ../shell/git-validate-info-yaml.sh + - ../shell/info-file-validate.sh + +- job-template: + name: '{project-name}-info-yaml-verify' + id: gerrit-info-yaml-verify + <<: *lf_jjb_common + # yamllint disable-line rule:key-duplicates + <<: *lf_info_yaml_verify + + git-url: '$GIT_URL/$GERRIT_PROJECT' + + scm: + - lf-infra-gerrit-scm: + git-url: '{git-url}' + refspec: '$GERRIT_REFSPEC' + branch: '$GERRIT_BRANCH' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: gerrit + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - gerrit: + server-name: '{gerrit-server-name}' + trigger-on: '{obj:gerrit_verify_triggers}' + projects: + - project-compare-type: ANT + project-pattern: '{project}' + branches: + - branch-compare-type: ANT + branch-pattern: '**/{branch}' + file-paths: + - compare-type: REG_EXP + pattern: 'INFO.yaml' + +- job-template: + name: '{project-name}-info-yaml-verify' + id: github-info-yaml-verify + <<: *lf_jjb_common + # yamllint disable-line rule:key-duplicates + <<: *lf_info_yaml_verify + + properties: + - github: + url: '{git-url}/{github-org}/{project}' + + scm: + - lf-infra-github-scm: + url: '{git-clone-url}{github-org}/{project}' + refspec: '+refs/pull/*:refs/remotes/origin/pr/*' + branch: '$sha1' + submodule-recursive: '{submodule-recursive}' + choosing-strategy: default + jenkins-ssh-credential: '{jenkins-ssh-credential}' + + triggers: + - github-pull-request: + trigger-phrase: '^recheck$' + only-trigger-phrase: false + status-context: 'INFO File Verify' + permit-all: true + github-hooks: true + # included-regions MUST match gerrit file-paths + included-regions: 'INFO.yaml' + ################ # Packer Merge # ################ diff --git a/shell/git-validate-info-yaml.sh b/shell/git-validate-info-yaml.sh new file mode 100644 index 00000000..80a4e24e --- /dev/null +++ b/shell/git-validate-info-yaml.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo '--> git-validate-info-yaml.sh' +# This script will make sure that the INFO.yaml file changes are kept +# isolated from other file changes. +# INFO.yaml file creation or moddifications should be kept in its own separate +# Gerrit. + +# This script will fail if other file changes are also included in the same +# patch. + +# Ensure we fail the job if any steps fail. +set -e -o pipefail +set +u + +MODIFIED_FILES=$(git diff --name-only HEAD~1) + +if [ "$MODIFIED_FILES" != "INFO.yaml" ]; then + echo 'ERROR: Do not combine INFO.yaml file changes with other files. Please isolate INFO.yaml changes.' + exit 1 +fi diff --git a/shell/info-file-validate.sh b/shell/info-file-validate.sh new file mode 100755 index 00000000..c3731f4d --- /dev/null +++ b/shell/info-file-validate.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +echo '--> info-file-validate.sh' +set -e -o pipefail +set -x # Enable trace + +virtualenv --quiet "/tmp/v/info" +# shellcheck source=/tmp/v/info/bin/activate disable=SC1091 +source "/tmp/v/info/bin/activate" +pip install PyYAML jsonschema rfc3987 + +# Cloning global-jjb to get access to needed scripts +git clone https://github.com/lfit/releng-global-jjb.git + +python releng-global-jjb/yaml-verify-schema.py \ + -s releng-global-jjb/info-schema \ + -y INFO.yaml + +rm -rf releng-global-jjb diff --git a/yaml-verify-schema.py b/yaml-verify-schema.py new file mode 100644 index 00000000..3a0d946e --- /dev/null +++ b/yaml-verify-schema.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: EPL-1.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +############################################################################## +""" +Verify YAML Schema +""" +import argparse +import logging +import jsonschema +import yaml + +LOADER = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader + + +def main(): + """Parse arguments and verify YAML""" + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument('--yaml', '-y', type=str, required=True) + parser.add_argument('--schema', '-s', type=str, required=True) + + args = parser.parse_args() + + with open(args.yaml) as _: + yaml_file = yaml.load(_, Loader=LOADER) + + with open(args.schema) as _: + schema_file = yaml.load(_, Loader=LOADER) + + validation = jsonschema.Draft4Validator( + schema_file, + format_checker=jsonschema.FormatChecker() + ) + + errors = 0 + for error in validation.iter_errors(yaml_file): + errors += 1 + logging.error(error) + if errors > 0: + raise RuntimeError("%d issues invalidate the schema" % errors) + + +if __name__ == "__main__": + main()