From 0f180ed1236c637af8c254f95ad106931cc77145 Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Tue, 19 Apr 2022 14:50:28 -0700 Subject: [PATCH] Feat: Add initial docker pipelines Issue: RELENG-4145 Signed-off-by: Eric Ball Change-Id: Ife17ac4b7a2a05169c35d81b944dfa47ab370ff9 --- Jenkinsfile.example.groovy | 14 +++ docs/vars/lfCommon.rst | 17 ++++ docs/vars/lfDocker.rst | 107 +++++++++++++++++++++ src/test/groovy/LFDockerSpec.groovy | 119 ++++++++++++++++++++++++ vars/lfCommon.groovy | 45 +++++++++ vars/lfDefaults.groovy | 3 + vars/lfDocker.groovy | 180 ++++++++++++++++++++++++++++++++++++ 7 files changed, 485 insertions(+) create mode 100644 docs/vars/lfDocker.rst create mode 100644 src/test/groovy/LFDockerSpec.groovy create mode 100644 vars/lfDocker.groovy diff --git a/Jenkinsfile.example.groovy b/Jenkinsfile.example.groovy index 6906075..bd8d17b 100644 --- a/Jenkinsfile.example.groovy +++ b/Jenkinsfile.example.groovy @@ -33,6 +33,15 @@ pipeline { environment { // The settings file needs to exist on the target Jenkins system mvnSettings = "sandbox-settings" + // Optional env vars + DOCKER_FILE_PATH = "docker/Dockerfile.groovy" + DOCKER_BUILD_CONTEXT = "docker" + DOCKER_CUSTOM_TAGS = "stable latest" + } + + triggers { + issueCommentTrigger('.*^recheck$.*') + issueCommentTrigger('.*^stage$.*') } stages { @@ -46,6 +55,11 @@ pipeline { lfNode() } } + stage("Docker Build") { + steps { + lfDocker(mvnSettings=env.mvnSettings, project="example") + } + } stage("Parallel Testing") { parallel { stage("amd") { diff --git a/docs/vars/lfCommon.rst b/docs/vars/lfCommon.rst index e7be391..dbfc47c 100644 --- a/docs/vars/lfCommon.rst +++ b/docs/vars/lfCommon.rst @@ -37,3 +37,20 @@ Signs the specified directory as a single artifact. :signDir: Path to directory to be signed (absolute path, or relative to the current working directory). :signMode: Serial or parallel. If left blank, the default (serial) is used. + +containerRegistryLogin +---------------------- + +Function to login to all needed Docker registries. + +:Required parameters: + + :settingsFile: Maven settings config file ID. + +:Optional parameters: + + :containerRegistry: Local container registry. + :containerRegistryPorts: Ports to use for local container registry. + :externalContainerRegistry: Path to external registry (generally Docker Hub). + :dockerHubEmail: Email for logging into external registry (Docker <17.06.0 + only). diff --git a/docs/vars/lfDocker.rst b/docs/vars/lfDocker.rst new file mode 100644 index 0000000..24e9d3f --- /dev/null +++ b/docs/vars/lfDocker.rst @@ -0,0 +1,107 @@ +######## +lfDocker +######## + +Parameters +========== + +:Required Parameters: + + :mvnSettings: Jenkins ID of maven settings file to be used by this job. + :project: Name to be given to the built container. + +:Optional Parameters: + + :containerPushRegistry: Override default registry to push built image. + +:Environment Variables: + + :DOCKER_BUILD_ARGS: Build-time arguments to pass to Docker. + :VERSION: If version is not being set through another means, it can be + passed in via this env var. + :DOCKER_CUSTOM_TAGS: Space-separated string of tags to push. + +Usage +===== + +Calling lfDocker will log into Docker registries (via the global-jjb shell +script docker-login.sh), and then build the container. If the branch is "master" +or "main", the container will then be pushed to the containerPushRegistry (this +should be defined in lfDefaults, but can also optionally be passed in). + +There are also environment variables that can be used to further tune the build +or push process, but these are always optional. + +Functions +========= + +dockerBuild +----------- + +Function to build a docker image. + +:Required Parameters: + + :dockerImageName: Image name to build. + +:Environment Variables: + + :DOCKER_BUILD_ARGS: Build-time arguments to pass to Docker. + :DOCKER_BUILD_CONTEXT: Override default build context of "." (present + working directory). + :DOCKER_FILE_PATH: Override default path to Dockerfile. + +dockerPush +---------- + +Function to push a docker image to a registry. + +:Required Parameters: + + :dockerImage: Image name to push. + :registry: Registry to push to. + +:Optional Parameters: + + :latest: Boolean indicating if this push should be tagged "latest". + :tags: List of tags. Used in lieu of env.DOCKER_CUSTOM_TAGS. + +:Environment Variables: + + :CONTAINER_PUSH_REGISTRY: The registry that the image is being pushed to. + :DOCKER_CUSTOM_TAGS: Space-separated string of additional tags. + :GIT_COMMIT: Git commit SHA. Should always be part of the build env. + :VERSION: If version is not being set through another means, it can be + passed in. + +getDockerTags +------------- + +This function polls multiple possible sources for Docker tags, compiling them +and returning all tags in a single list. + +:Optional Parameters: + + :latest: Boolean indicating if this should be tagged "latest". + :customTags: Space-separated string of additional tags. + +:Environment Variables: + + :DOCKER_CUSTOM_TAGS: Space-separated string of additional tags. + :GIT_COMMIT: Git commit SHA. Should always be part of the build env. + :VERSION: If version is not being set through another means, it can be + passed in. + +finalImageName +-------------- + +Function to prepend registry to image name (generally needed when using multiple +registries). + +:Required Parameters: + + :imageName: Base image name. + +:Environment Variables: + + :CONTAINER_PUSH_REGISTRY: The registry that the image is being pushed to. diff --git a/src/test/groovy/LFDockerSpec.groovy b/src/test/groovy/LFDockerSpec.groovy new file mode 100644 index 0000000..d482b5f --- /dev/null +++ b/src/test/groovy/LFDockerSpec.groovy @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (c) 2022 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import com.homeaway.devtools.jenkins.testing.JenkinsPipelineSpecification + +public class LFDockerSpec extends JenkinsPipelineSpecification { + + def lfDocker = null + def defaults = [ + containerPublicRegistry: "docker.io", + containerPushRegistry: "nexus3.example.org", + mvnSettings: "testConfig", + project: "testProject", + ] + + def setup() { + lfDocker = loadPipelineScriptForTest('vars/lfDocker.groovy') + explicitlyMockPipelineVariable('lfCommon') + explicitlyMockPipelineVariable('lfDefaults') + } + + def "Test lfDocker [Should] throw exception [When] mvnSettings is null" () { + setup: + when: + lfDocker({mvnSettings = null}) + then: + thrown Exception + } + + def "Test lfDocker [Should] throw exception [When] project is null" () { + setup: + when: + lfDocker() + then: + thrown Exception + } + + def "Test lfDocker [Should] build docker image [When] called" () { + setup: + def environmentVariables = [ + "DOCKER_FILE_PATH": "", + "DOCKER_BUILD_ARGS": "", + "DOCKER_BUILD_CONTEXT": "" + ] + lfDocker.getBinding().setVariable("env", environmentVariables) + getPipelineMock("lfDefaults.call")() >> { + return defaults + } + explicitlyMockPipelineStep("dockerBuild") + explicitlyMockPipelineStep("docker.build") + when: + lfDocker() + then: + 1 * getPipelineMock("lfCommon.containerRegistryLogin").call(_) >> null + 1 * getPipelineMock("docker.build").call([ + "nexus3.example.org/testProject", + "-f Dockerfile ." + ]) + } + + def "Test lfDocker [Should] build & deploy container [When] branch == 'master'" () { + setup: + def environmentVariables = [ + "DOCKER_FILE_PATH": "", + "DOCKER_BUILD_ARGS": "", + "DOCKER_BUILD_CONTEXT": "" + ] + lfDocker.getBinding().setVariable("env", environmentVariables) + getPipelineMock("lfDefaults.call")() >> { + return defaults + } + explicitlyMockPipelineStep("dockerBuild") + explicitlyMockPipelineStep("docker.build") + when: + lfDocker() + then: + 1 * getPipelineMock("lfCommon.containerRegistryLogin").call(_) >> null + 1 * getPipelineMock("docker.build").call([ + "nexus3.example.org/testProject", + "-f Dockerfile ." + ]) + } + + def "Test lfDocker [Should] build & deploy container [When] branch == 'master'" () { + setup: + def environmentVariables = [ + "DOCKER_FILE_PATH": "", + "DOCKER_BUILD_ARGS": "", + "DOCKER_BUILD_CONTEXT": "" + ] + lfDocker.getBinding().setVariable("env", environmentVariables) + getPipelineMock("lfDefaults.call")() >> { + return defaults + } + explicitlyMockPipelineStep("dockerBuild") + explicitlyMockPipelineStep("docker.build") + when: + lfDocker() + then: + 1 * getPipelineMock("lfCommon.containerRegistryLogin").call(_) >> null + 1 * getPipelineMock("docker.build").call([ + "nexus3.example.org/testProject", + "-f Dockerfile ." + ]) + } +} diff --git a/vars/lfCommon.groovy b/vars/lfCommon.groovy index 3795f42..6dd955b 100644 --- a/vars/lfCommon.groovy +++ b/vars/lfCommon.groovy @@ -59,3 +59,48 @@ def sigulSignDir(signDir, signMode) { sh(script: libraryResource('shell/sigul-configuration-cleanup.sh')) } } + +/** + * Function to login to all needed container registries. + * + * @param settingsFile Maven settings config file ID. + * @param containerRegistry Local container registry. + * @param containerRegistryPorts Ports to use for local container registry. + * @param externalContainerRegistry Path to external registry (generally Docker Hub). + * @param dockerHubEmail Email for logging into external registry (Docker <17.06.0 only). + */ + +def containerRegistryLogin(settingsFile, containerRegistry="", containerRegistryPorts="", + externalContainerRegistry="", dockerHubEmail="") { + // The LF Global JJB Docker Login script looks for information in the following variables: + // $SETTINGS_FILE, $DOCKER_REGISTRY, $REGISTRY_PORTS, $DOCKERHUB_REGISTRY, $DOCKERHUB_EMAIL + // Please refer to the shell script in global-jjb/shell for the usage. + // Most parameters are listed as optional, but without any of them set the script has no operation. + if (!settingsFile) { + error('Project Settings File id (settingsFile) is required for the container registry login script.') + } + + if (containerRegistry && !containerRegistryPorts) { + error('Container registry ports (containerRegistryPorts) are required when registry (containerRegistry) is set.') + } + + if (containerRegistryPorts && !containerRegistry) { + error('Container registry (containerRegistry) is required when registry ports (containerRegistryPorts) are set.') + } + + if (dockerHubEmail && !externalContainerRegistry) { + error('External registry (externalContainerRegistry) is required when Docker Hub Email (dockerHubEmail) is set.') + } + + def envVars = [] + if (containerRegistry) { envVars << "DOCKER_REGISTRY=${containerRegistry}" } + if (containerRegistryPorts) { envVars << "REGISTRY_PORTS=${containerRegistryPorts}" } + if (externalContainerRegistry) { envVars << "DOCKERHUB_REGISTRY=${externalContainerRegistry}" } + if (dockerHubEmail) { envVars << "DOCKERHUB_EMAIL=${dockerHubEmail}" } + + withEnv(envVars){ + configFileProvider([configFile(fileId: settingsFile, variable: 'SETTINGS_FILE')]) { + sh(script: libraryResource('global-jjb-shell/docker-login.sh')) + } + } +} diff --git a/vars/lfDefaults.groovy b/vars/lfDefaults.groovy index edd116a..224fd99 100644 --- a/vars/lfDefaults.groovy +++ b/vars/lfDefaults.groovy @@ -34,6 +34,9 @@ def call(body) { nodeDir: "", nodeVersion: "14.17.5", + + containerPublicRegistry: "docker.io", + containerPushRegistry: "nexus3.example.org", ] return defaults } diff --git a/vars/lfDocker.groovy b/vars/lfDocker.groovy new file mode 100644 index 0000000..4ec3a36 --- /dev/null +++ b/vars/lfDocker.groovy @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (c) 2022 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Method to run Docker builds. Requires "Docker Pipeline" Jenkins plugin. + * Required body values: + * * mvnSettings: Maven settings config file ID + * * project: Name to be given to container + * + * Optional body values (should be defined in lfDefaults): + * * containerPushRegistry: Override default registry to push built image. + * + * @param body Config values to be provided in the form "key = value". + */ +def call(body) { + // Evaluate the body block and collect configuration into the object + def defaults = lfDefaults() + def config = [:] + + if (body) { + body.resolveStrategy = Closure.DELEGATE_FIRST + body.delegate = config + body() + } + + // For duplicate keys, Groovy will use the right hand map's values. + config = defaults + config + + if (!config.mvnSettings || !config.project) { + throw new Exception("Maven settings file id (mvnSettings) and " + + "project ID (project) are required for lfDocker function.") + } + if (config.containerPushRegistry) { + env.CONTAINER_PUSH_REGISTRY = config.containerPushRegistry + } else { + config.containerPushRegistry = "" + } + + lfCommon.containerRegistryLogin(config.mvnSettings) + + /////////////////////////////////// + // Build/Verify Docker Container // + /////////////////////////////////// + dockerBuild(config.project) + + ///////////////////////////// + // Push container on merge // + ///////////////////////////// + if (env.GIT_BRANCH == "main" || env.GIT_BRANCH == "master") { + dockerPush(config.project, config.containerPushRegistry) + } +} + +/** + * Function to build a docker image. + * Optional env variables: + * * DOCKER_BUILD_ARGS: Build-time arguments to pass to Docker. + * * DOCKER_BUILD_CONTEXT: Override default build context of "." (present working directory). + * * DOCKER_FILE_PATH: Override default path to Dockerfile. + * + * @param dockerImageName Image name to build + */ +def dockerBuild(dockerImageName) { + def buildArgString = "" + def dockerfile = "Dockerfile" + def buildContext = "." + + if (env.DOCKER_BUILD_ARGS) { + def buildArgs = [''] // Start with blank entry + env.DOCKER_BUILD_ARGS.split(',').each { buildArgs << it } + buildArgString = buildArgs.join(' --build-arg ') + } + + if (env.DOCKER_FILE_PATH) { + dockerfile = env.DOCKER_FILE_PATH + } + + if (env.DOCKER_BUILD_CONTEXT) { + buildContext = env.DOCKER_BUILD_CONTEXT + } + + docker.build(finalImageName(dockerImageName), "-f ${dockerfile} ${buildArgString} ${buildContext}") +} + +/** + * Function to push a docker image to a registry. + * Optional env variables: + * * CONTAINER_PUSH_REGISTRY: The registry that the image is being pushed to. + * * DOCKER_CUSTOM_TAGS: Space-separated string of additional tags. + * * GIT_COMMIT: Git commit SHA. Should always be part of the build env. + * * VERSION: If version is not being set through another means, it can be passed in. + * + * @param dockerImage Image name to push + * @param registry Registry to push to + * @param latest Boolean indicating if this push should be tagged "latest" + * @param tags List of tags. Used in lieu of env.DOCKER_CUSTOM_TAGS + */ +def dockerPush(dockerImage, registry, latest = true, tags = null) { + if (tags == null) { + tags = getDockerTags(latest) + } + def image = docker.image(finalImageName(dockerImage)) + + docker.withRegistry(registry) { + tags.each { + image.push(it) + } + } +} + +/** + * This function polls multiple possible sources for Docker tags, compiling them + * and returning all tags in a single list. + * Optional env variables: + * * DOCKER_CUSTOM_TAGS: Space-separated string of additional tags. + * * GIT_COMMIT: Git commit SHA. Should always be part of the build env. + * * VERSION: If version is not being set through another means, it can be passed in. + * + * @param latest Boolean to indicate if this should be tagged "latest" + * @param customTags Space-separated string of additional tags + */ +def getDockerTags(latest = true, customTags = env.DOCKER_CUSTOM_TAGS) { + def allTags = [] + + if (env.GIT_COMMIT) { + allTags << "${env.GIT_COMMIT}" + } + + if (latest) { + allTags << "latest" + } + + if (env.VERSION) { + allTags << env.VERSION + } + + if (env.GIT_COMMIT && env.VERSION) { + allTags << "${env.GIT_COMMIT}-${env.VERSION}" + } + + if (customTags) { + customTags.split(' ').each { + allTags << it + } + } + + return allTags +} + +/** + * Function to prepend registry to image name (generally needed when using + * multiple registries). + * Optional env variables: + * * CONTAINER_PUSH_REGISTRY: The registry that the image is being pushed to. + * + * @param imageName Base image name + */ +def finalImageName(imageName) { + def finalDockerImageName = imageName + + // prepend with registry "namespace" if not empty + if (env.CONTAINER_PUSH_REGISTRY && env.CONTAINER_PUSH_REGISTRY != '/') { + finalDockerImageName = "${env.CONTAINER_PUSH_REGISTRY}/${finalDockerImageName}" + } + + finalDockerImageName +} -- 2.16.6