From b25baec0dafad8345d97fbbdf2022a4100e721bc Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Wed, 30 May 2018 17:58:20 -0400 Subject: [PATCH] Complete ci-management bootstrap steps Issue: RELENG-981 Change-Id: Ia372d258bb5db9d6944346bab73ca0509ce2a463 Signed-off-by: Thanh Ha --- docs/_static/ciman/coafile.example | 34 +++ docs/_static/ciman/defaults.yaml | 7 + docs/_static/ciman/jenkins-init-script.sh.example | 4 + docs/_static/ciman/tox.ini.example | 16 ++ docs/_static/ciman/yamllint.conf.example | 7 + docs/infra/bootstrap.rst | 295 ++++++++++++++++++++-- 6 files changed, 345 insertions(+), 18 deletions(-) create mode 100644 docs/_static/ciman/coafile.example create mode 100644 docs/_static/ciman/defaults.yaml create mode 100644 docs/_static/ciman/jenkins-init-script.sh.example create mode 100644 docs/_static/ciman/tox.ini.example create mode 100644 docs/_static/ciman/yamllint.conf.example diff --git a/docs/_static/ciman/coafile.example b/docs/_static/ciman/coafile.example new file mode 100644 index 0000000..f0abad4 --- /dev/null +++ b/docs/_static/ciman/coafile.example @@ -0,0 +1,34 @@ +[Documentation] +bears = WriteGoodLintBear +files = *.md +allow_so_beginning = False +allow_there_is = False +allow_cliche_phrases = False + +[GitCommit] +bears = GitCommitBear +ignore_length_regex = Signed-off-by, + Also-by, + Co-authored-by, + http://, + https:// + +[JSON] +bears = JSONFormatBear +files = packer/**.json +indent_size = 2 + +[ShellCheck] +bears = ShellCheckBear, + SpaceConsistencyBear +files = jjb/**.sh, + packer/**.sh +shell = bash +indent_size = 4 +use_spaces = yeah + +[YAML] +bears = YAMLLintBear +files = jjb/**/*.yaml +document_start = True +yamllint_config = .yamllint.conf diff --git a/docs/_static/ciman/defaults.yaml b/docs/_static/ciman/defaults.yaml new file mode 100644 index 0000000..519a71d --- /dev/null +++ b/docs/_static/ciman/defaults.yaml @@ -0,0 +1,7 @@ +- defaults: + name: global + + gerrit-server-name: Primary + git-url: 'ssh://jenkins-$SILO@gerrit.example.org:29418' + jenkins-ssh-credential: jenkins-ssh + lftools-version: '<1.0.0' diff --git a/docs/_static/ciman/jenkins-init-script.sh.example b/docs/_static/ciman/jenkins-init-script.sh.example new file mode 100644 index 0000000..ade7f75 --- /dev/null +++ b/docs/_static/ciman/jenkins-init-script.sh.example @@ -0,0 +1,4 @@ +#!/bin/bash +git clone https://gerrit.example.org/r/ci-management.git /ci-management +/ci-management/jenkins-scripts/jenkins-init-script.sh +rm -rf /ci-management diff --git a/docs/_static/ciman/tox.ini.example b/docs/_static/ciman/tox.ini.example new file mode 100644 index 0000000..e33d587 --- /dev/null +++ b/docs/_static/ciman/tox.ini.example @@ -0,0 +1,16 @@ +[tox] +minversion = 1.6 +envlist = coala +skipsdist = true + +[testenv:coala] +basepython = python3 +deps = + coala==0.11 + coala-bears==0.11 + nodeenv~=1.3.0 +commands = + nodeenv -p + npm install --global write-good + python3 -m nltk.downloader punkt maxent_treebank_pos_tagger averaged_perceptron_tagger + coala --non-interactive diff --git a/docs/_static/ciman/yamllint.conf.example b/docs/_static/ciman/yamllint.conf.example new file mode 100644 index 0000000..b821cb8 --- /dev/null +++ b/docs/_static/ciman/yamllint.conf.example @@ -0,0 +1,7 @@ +extends: default + +rules: + empty-lines: + max-end: 1 + line-length: + max: 120 diff --git a/docs/infra/bootstrap.rst b/docs/infra/bootstrap.rst index a59b038..b8210df 100644 --- a/docs/infra/bootstrap.rst +++ b/docs/infra/bootstrap.rst @@ -92,15 +92,252 @@ ci-management repo Once Jenkins is available we can initialize a new ci-management repo. -Steps +.. _infra-bootstrap-admin-files: -.. todo:: First bootstrap a builder so that we can bootstrap ci-management +Setup administrative files +-------------------------- -#. Create ci-management repo in Gerrit +#. Create ci-management repo in the project SCM system #. Create a README.md file explaining the purpose of the repo -#. Setup tox/coala linting for jjb/ and packer directories -#. Install global-jjb to GIT_ROOT/jjb/global-jjb -#. Create the CI Jobs in jjb/ci-management/ci-jobs.yaml + + :: + + # ci-management + + This repo contains configuration files for Jenkins jobs for the EXAMPLE + project. + +#. Setup tox/coala linting for ``jjb/`` and ``packer/`` directories + + **.yamllint.conf** + + .. literalinclude:: ../_static/ciman/yamllint.conf.example + :language: ini + + **.coafile** + + .. literalinclude:: ../_static/ciman/coafile.example + :language: ini + + **tox.ini** + + .. literalinclude:: ../_static/ciman/tox.ini.example + :language: ini + +#. Setup .gitignore + + .. code-block:: bash + + .tox/ + archives/ + jenkins.ini + + # Packer + .galaxy/ + *.retry + cloud-env.json + +#. ``git commit -asm "Setup repo administrative files"`` +#. ``git push`` files to the repository +#. Run ``tox`` + + .. note:: + + The ``jjb`` tox env will fail as the required ``jjb/`` directory does not + yet exist. This is fine and proves that tox is working before + we continue in the next step. + +.. _infra-bootstrap-cp: + +Bootstrap common-packer and initial builder +------------------------------------------- + +.. note:: + + This section assumes the usage of an OpenStack cloud provider for Jenkins + build nodes. Adjust as necessary if not using an OpenStack cloud. + +#. Navigate to the ``GIT_ROOT`` of the **ci-management** repo +#. Install **common-packer** to ``GIT_ROOT/packer/common-packer`` + + .. code-block:: bash + + git submodule add https://github.com/lfit/releng-common-packer.git packer/common-packer + +#. Follow common-packer doc to :ref:`setup a template ` +#. ``git commit -asm "Setup common-packer and initial builder"`` +#. ``git push`` files to repository +#. Upload a CentOS 7 cloudimg to use as a base for packer builds + + When uploading the cloudimg ensure it's name matches the ``base_image`` + name in ``common-packer/vars/centos-7.json``. + +#. Run ``packer build -var-file=cloud-env.json -var-file=common-packer/vars/centos-7.json templates/builder.json`` +#. Note down the image name from the packer build as we will need it later + +#. Navigate to ``https://jenkins.example.org/credentials/store/system/domain/_/newCredentials`` +#. Configure the openstack cloud credential as follows: + + .. code-block:: none + + Kind: OpenStack auth v3 + Project Domain: Default + Project Name: OPENSTACK_TENANT_ID + User Domain: Default + User Name: OPENSTACK_USERNAME + Password: OPENSTACK_PASSWORD + ID: os-cloud + Description: openstack-cloud-credential + + .. note:: + + Replace ALL_CAPS instances with your Cattle account credential. + +#. Configure an ssh keypair for the Jenkins <-> OpenStack connection + + #. Generate a new SSH Keypair + + .. code-block:: bash + + ssh-keygen -t rsa -C jenkins-ssh -f /tmp/jenkins + + #. Navigate to ``https://jenkins.example.org/credentials/store/system/domain/_/newCredentials`` + #. Configure the Jenkins SSH Key as follows: + + .. code-block:: none + + Kind: SSH Username and private key + Scope: Global + Username: jenkins + Private Key: Enter directly + Passphrase: + ID: jenkins + Description: jenkins-ssh + + Copy the contents of ``/tmp/jenkins`` into the Key field. + + #. Navigate to ``https://openstack-cloud.example.org/project/key_pairs`` + #. Import the contents of ``/tmp/jenkins.pub`` into the OpenStack cloud + provider account with the keypair name ``jenkins`` + +#. Navigate to ``https://jenkins.example.org/configfiles/selectProvider`` +#. Create a ``jenkins-init-system`` file with the following specs: + + .. code-block:: none + + Type: OpenStack User Data + ID: jenkins-init-script + Name: jenkins-init-script + Comment: jenkins-init-script + + With the contents (change the git clone URL as necessary for the project): + + .. literalinclude:: ../_static/ciman/jenkins-init-script.sh.example + +#. Configure ``cattle`` cloud + + #. Create cloud config directory ``mkdir -p jenkins-config/clouds/openstack/cattle`` + #. Configure the OpenStack cloud connection details in + ``jenkins-config/clouds/openstack/cattle/cloud.cfg`` + + Replace ```` and ```` in the below file + with the details for your cloud. + + .. code-block:: bash + :caption: jenkins-config/clouds/openstack/cattle/cloud.cfg + + # Cloud Configuration + CLOUD_CREDENTIAL_ID=os-cloud + CLOUD_URL=https://auth.vexxhost.net/v3/ + CLOUD_IGNORE_SSL=false + CLOUD_ZONE=ca-ymq-1 + + # Default Template Configuration + IMAGE_NAME= + HARDWARE_ID=v1-standard-1 + NETWORK_ID= + USER_DATA_ID=jenkins-init-script + INSTANCE_CAP=10 + SANDBOX_CAP=4 + FLOATING_IP_POOL= + SECURITY_GROUPS=default + AVAILABILITY_ZONE=ca-ymq-2 + STARTUP_TIMEOUT=600000 + KEY_PAIR_NAME=jenkins + NUM_EXECUTORS=1 + JVM_OPTIONS= + FS_ROOT=/w + RETENTION_TIME=0 + + #. Create ``jenkins-config/clouds/openstack/odlvex/centos7-builder-2c-1g.cfg`` + + .. code-block:: bash + + IMAGE_NAME=ZZCI - CentOS 7 - builder - 20180604-1653 + HARDWARE_ID=v1-standard-1 + + #. Run global-jjb jenkins-cfg script to update Jenkins cloud config + + Set ``jenkins_silos`` to match the config section name in the + previous step. + + Run the following commands (Ignore the ``Section not found: production`` + error): + + .. code-block:: bash + + export WORKSPACE=$(pwd) + export jenkins_silos=production + bash ./jjb/global-jjb/shell/jenkins-configure-clouds.sh + cat archives/groovy-inserts/production-cloud-cfg.groovy + + Then navigate to + ``https://jenkins.example.org/script`` and copy + the contents of ``archives/groovy-inserts/production-cloud-cfg.groovy`` + into the script console. This will initialize the OpenStack cloud + configuration. + + #. Commit the ``jenkins-config`` directory + + .. code-block:: bash + + git add jenkins-config/ + git commit -sm "Add OpenStack cloud configuration" + git push + +#. Navigate to ``https://jenkins.example.org/configure`` +#. Click ``Add a new cloud`` > ``Cloud (OpenStack)`` +#. Configure the cloud + + .. code-block:: none + :caption: example + + Cloud Name: cattle + End Point URL: https://auth.vexxhost.net/v3/ + Ignore unverified SSL certificates: false + Credential: openstack-cloud-credential + Region: ca-ymq-1 + + .. note:: + + The configuration here is temporary to bootstrap + +.. _infra-bootstrap-global-jjb: + +Setup global-jjb and ci-jobs +---------------------------- + +#. Install global-jjb to ``GIT_ROOT/jjb/global-jjb`` + + .. code-block:: bash + + git submodule add https://github.com/lfit/releng-global-jjb.git jjb/global-jjb + +#. Setup ``jjb/defaults.yaml`` + + .. literalinclude:: ../_static/ciman/defaults.yaml + +#. Create the CI Jobs in ``jjb/ci-management/ci-jobs.yaml`` .. code-block:: yaml @@ -115,20 +352,39 @@ Steps build-node: centos7-builder-2c-1g #. Manually push the initial ci-management jobs to Jenkins + + .. code-block:: bash + + jenkins-jobs update jjb/ + #. Git commit the current files and push to Gerrit + + .. code-block:: bash + + git commit -sm "Setup global-jjb and ci-jobs" + git push + #. Confirm verify jobs work #. Merge the patch and confirm merge job works -#. Install common-packer to GIT_ROOT/packer/common-packer - .. code-block:: bash +.. _setup-packer-jobs: - git submodule add https://github.com/lfit/releng-common-packer.git packer/common-packer +Setup packer jobs +----------------- -#. Git commit and merge patch in Gerrit #. Create Initial CI Packer job in jjb/ci-management/ci-packer.yaml .. code-block:: yaml + - project: + name: packer-verify + jobs: + - gerrit-packer-verify + + project: ci-management + project-name: ci-management + build-node: centos7-builder-2c-1g + - project: name: packer-builder-jobs jobs: @@ -138,16 +394,19 @@ Steps project-name: ci-management build-node: centos7-builder-2c-1g - platforms: centos templates: builder + platforms: + - centos-7 + - ubuntu-16.04 + +#. Git commit and push the patch to ci-management for review + + .. code-block:: bash + + git commit -sm "Add packer builder job" + git push ... -#. Git commit and merge patch in Gerrit -#. Symlink common-packer/templates/builder.json.example to templates/builder.json -#. Git commit and push patch to Gerrit #. Confirm packer verify job passes #. Merge patch and confirm merge job works -#. Update and Create appropriate builders in Jenkins using the newly created image -.. todo:: provide example README text -.. todo:: provide example tox.ini and .coafile -.. todo:: we need to make sure the ci-jobs macro includes the tox job for linting +.. todo:: Create a post-bootstrap doc for recommendations on where to go next -- 2.16.6