From d1a99147b03c6cd90561f8e70b7af78166bcd4cd Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Wed, 12 May 2021 17:25:45 -0700 Subject: [PATCH] Feat: Add initial Java pipeline functionality This commit introduces functionality equivalent to the global-jjb jobs for maven-verify, maven-merge, and maven-stage. It includes common functions, a script to supply defaults, and unit tests for the new functions. There are also some small changes to older functions in order to utilize these new common functions. Issue: RELENG-3445 Issue: RELENG-3446 Issue: RELENG-3618 Change-Id: I30384d91627f6065f390a0cd6b0b434748daab49 Signed-off-by: Eric Ball --- docs/vars/lfCommon.rst | 39 ++++++ docs/vars/lfDefaults.rst | 25 ++++ docs/vars/lfJava.rst | 26 ++++ .../notes/java-workflow-e103dc06b553a488.yaml | 7 ++ src/test/groovy/LFCommonSpec.groovy | 71 +++++++++++ src/test/groovy/LFInfraShipLogsSpec.groovy | 10 +- src/test/groovy/LFJavaSpec.groovy | 137 +++++++++++++++++++++ tox.ini | 1 + vars/lfCommon.groovy | 61 +++++++++ vars/lfDefaults.groovy | 36 ++++++ vars/lfInfraShipLogs.groovy | 3 +- vars/lfJava.groovy | 112 +++++++++++++++++ 12 files changed, 518 insertions(+), 10 deletions(-) create mode 100644 docs/vars/lfCommon.rst create mode 100644 docs/vars/lfDefaults.rst create mode 100644 docs/vars/lfJava.rst create mode 100644 releasenotes/notes/java-workflow-e103dc06b553a488.yaml create mode 100644 src/test/groovy/LFCommonSpec.groovy create mode 100644 src/test/groovy/LFJavaSpec.groovy create mode 100644 vars/lfCommon.groovy create mode 100644 vars/lfDefaults.groovy create mode 100644 vars/lfJava.groovy diff --git a/docs/vars/lfCommon.rst b/docs/vars/lfCommon.rst new file mode 100644 index 0000000..e7be391 --- /dev/null +++ b/docs/vars/lfCommon.rst @@ -0,0 +1,39 @@ +######## +lfCommon +######## + +Common functions to be used by other scripts. + +Functions +========= + +installPythonTools +------------------ + +This should be run before most jobs, in order to install commonly-used python tools. + +jacocoNojava +------------ + +Workaround for Jenkins not being able to find Java in JaCoCo runs. + +updateJavaAlternatives +---------------------- + +Runs a script to ensure that the preferred version of Java is installed and is +the default when java commands are executed. + +:Required parameters: + + :javaVersion: The version of java to set as the default, e.g. "openjdk11". + +sigulSignDir +------------ + +Signs the specified directory as a single artifact. + +:Required parameters: + + :signDir: Path to directory to be signed (absolute path, or relative to + the current working directory). + :signMode: Serial or parallel. If left blank, the default (serial) is used. diff --git a/docs/vars/lfDefaults.rst b/docs/vars/lfDefaults.rst new file mode 100644 index 0000000..1d242fb --- /dev/null +++ b/docs/vars/lfDefaults.rst @@ -0,0 +1,25 @@ +########## +lfDefaults +########## + +Usage +===== + +These are default values for variables that are frequently used by functions +and scripts within the pipeline library. Calling lfDefaults() will return a map +of the default values. If combining with a user-specified config map, the +user-specified values should be on the right-hand side of the addition, as this +will be the values used in case of a collision. + +.. code-block:: groovy + + def defaults = lfDefaults() + def config = [:] + + if (body) { + body.resolveStrategy = Closure.DELEGATE_FIRST + body.delegate = config + body() + } + + config = defaults + config diff --git a/docs/vars/lfJava.rst b/docs/vars/lfJava.rst new file mode 100644 index 0000000..8e4e327 --- /dev/null +++ b/docs/vars/lfJava.rst @@ -0,0 +1,26 @@ +###### +lfJava +###### + +Parameters +========== + +:Required Parameters: + + :mvnSettings: Jenkins ID of maven settings file to be used by this job + +:Optional Parameters: + + :javaVersion: Java version to use for Maven build + :mvnGlobalSettings: Override default global-settings filename + :mvnGoals: String with maven goals to execute + :mvnVersion: Maven version to use in build + +Usage +===== + +Calling lfJava will prep the agent and then execute a maven build, using the +mvnGoals specified. If the branch is "master" or "main", the maven-deploy script will +be called, deploying artifacts to maven. If triggered by a comment containing +the keyword "stage", the maven-stage script will be run to sign and stage +artifacts for release. diff --git a/releasenotes/notes/java-workflow-e103dc06b553a488.yaml b/releasenotes/notes/java-workflow-e103dc06b553a488.yaml new file mode 100644 index 0000000..34d6c04 --- /dev/null +++ b/releasenotes/notes/java-workflow-e103dc06b553a488.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + The lfJava global var introduces functionality equivalent to the global-jjb + jobs for maven-verify, maven-merge, and maven-stage. This will test new + patch/pull requests, test and deploy artifacts upon merge, and stage artifacts + for release. diff --git a/src/test/groovy/LFCommonSpec.groovy b/src/test/groovy/LFCommonSpec.groovy new file mode 100644 index 0000000..eea8473 --- /dev/null +++ b/src/test/groovy/LFCommonSpec.groovy @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (c) 2019 Intel Corporation +// Copyright (c) 2020 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import com.homeaway.devtools.jenkins.testing.JenkinsPipelineSpecification + +public class LFCommonSpec extends JenkinsPipelineSpecification { + + def lfCommon = null + + def setup() { + lfCommon = loadPipelineScriptForTest("vars/lfCommon.groovy") + } + + def "Test lfCommon [Should] call expected script [When] methods are called" () { + setup: + lfCommon.getBinding().setVariable('WORKSPACE', '/w/test/job/1') + explicitlyMockPipelineStep("withEnv") + getPipelineMock("libraryResource")("shell/python-tools-install.sh") >> { + return "python-tools-install" + } + getPipelineMock("libraryResource")("shell/update-java-alternatives.sh") >> { + return "update-java-alternatives" + } + getPipelineMock("libraryResource")("shell/sigul-configuration.sh") >> { + return "sigul-configuration" + } + getPipelineMock("libraryResource")("shell/sigul-install.sh") >> { + return "sigul-install" + } + when: + lfCommon.installPythonTools() + then: + 1 * getPipelineMock("sh").call([script:"python-tools-install"]) + when: + lfCommon.jacocoNojava() + then: + 1 * getPipelineMock("sh").call("mkdir -p /w/test/job/1/target/classes /w/test/job/1/jacoco/classes") + when: + lfCommon.updateJavaAlternatives("openjdk11") + then: + 1 * getPipelineMock("sh").call([script: + "SET_JDK_VERSION=openjdk11" + "\n" + + "update-java-alternatives" + ]) + when: + lfCommon.sigulSignDir(".", "parallel") + then: + 1 * getPipelineMock('withEnv').call(_) >> { _arguments -> + def envArgs = [ + "SIGN_DIR=.", + "SIGN_MODE=parallel" + ] + assert envArgs == _arguments[0][0] + } + 1 * getPipelineMock("sh").call([script:"sigul-configuration" + "\n" + "sigul-install"]) + } +} diff --git a/src/test/groovy/LFInfraShipLogsSpec.groovy b/src/test/groovy/LFInfraShipLogsSpec.groovy index de19267..18ec81b 100644 --- a/src/test/groovy/LFInfraShipLogsSpec.groovy +++ b/src/test/groovy/LFInfraShipLogsSpec.groovy @@ -23,7 +23,7 @@ public class LFInfraShipLogsSpec extends JenkinsPipelineSpecification { def setup() { lfInfraShipLogs = loadPipelineScriptForTest('vars/lfInfraShipLogs.groovy') - explicitlyMockPipelineVariable('out') + explicitlyMockPipelineVariable('lfCommon') } def "Test lfInfraShipLogs [Should] throw exception [When] logSettingsFile is null" () { @@ -46,9 +46,6 @@ public class LFInfraShipLogsSpec extends JenkinsPipelineSpecification { getPipelineMock("libraryResource")('shell/create-netrc.sh') >> { return 'create-netrc' } - getPipelineMock("libraryResource")('shell/python-tools-install.sh') >> { - return 'python-tools-install' - } getPipelineMock("libraryResource")('shell/sudo-logs.sh') >> { return 'sudo-logs' } @@ -61,6 +58,7 @@ public class LFInfraShipLogsSpec extends JenkinsPipelineSpecification { getPipelineMock("libraryResource")('shell/logs-clear-credentials.sh') >> { return 'logs-clear-credentials' } + getPipelineMock("lfCommon.installPythonTools").call(_) >> null when: 'Only LOGS_SERVER defined' lfInfraShipLogs.getBinding().setVariable('LOGS_SERVER', 'MyLogServer') lfInfraShipLogs.getBinding().setVariable('S3_BUCKET', '') @@ -76,7 +74,6 @@ public class LFInfraShipLogsSpec extends JenkinsPipelineSpecification { assert envArgs == _arguments[0][0] } 1 * getPipelineMock('sh').call([script:'create-netrc']) - 1 * getPipelineMock('sh').call([script:'python-tools-install']) 1 * getPipelineMock('sh').call([script:'sudo-logs']) 1 * getPipelineMock('sh').call([script:'job-cost']) 1 * getPipelineMock('sh').call([script:'logs-deploy']) @@ -97,7 +94,6 @@ public class LFInfraShipLogsSpec extends JenkinsPipelineSpecification { assert envArgs == _arguments[0][0] } 1 * getPipelineMock('sh').call([script:'create-netrc']) - 1 * getPipelineMock('sh').call([script:'python-tools-install']) 1 * getPipelineMock('sh').call([script:'sudo-logs']) 1 * getPipelineMock('sh').call([script:'job-cost']) 1 * getPipelineMock('sh').call([script:'logs-deploy']) @@ -118,7 +114,6 @@ public class LFInfraShipLogsSpec extends JenkinsPipelineSpecification { assert envArgs == _arguments[0][0] } 1 * getPipelineMock('sh').call([script:'create-netrc']) - 1 * getPipelineMock('sh').call([script:'python-tools-install']) 1 * getPipelineMock('sh').call([script:'sudo-logs']) 1 * getPipelineMock('sh').call([script:'job-cost']) 1 * getPipelineMock('sh').call([script:'logs-deploy']) @@ -141,7 +136,6 @@ public class LFInfraShipLogsSpec extends JenkinsPipelineSpecification { assert envArgs == _arguments[0][0] } 0 * getPipelineMock('sh').call([script:'create-netrc']) - 0 * getPipelineMock('sh').call([script:'python-tools-install']) 0 * getPipelineMock('sh').call([script:'sudo-logs']) 0 * getPipelineMock('sh').call([script:'job-cost']) 0 * getPipelineMock('sh').call([script:'logs-deploy']) diff --git a/src/test/groovy/LFJavaSpec.groovy b/src/test/groovy/LFJavaSpec.groovy new file mode 100644 index 0000000..e740eb1 --- /dev/null +++ b/src/test/groovy/LFJavaSpec.groovy @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (c) 2021 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import com.homeaway.devtools.jenkins.testing.JenkinsPipelineSpecification + +public class LFJavaSpec extends JenkinsPipelineSpecification { + + def lfJava = null + def defaults = [ + javaVersion: "openjdk11", + mvnGoals: "clean install", + mvnVersion: "mvn35", + mvnGlobalSettings: "testGlobalConfig", + mvnSettings: "testConfig", + ] + + def setup() { + lfJava = loadPipelineScriptForTest('vars/lfJava.groovy') + explicitlyMockPipelineVariable('lfCommon') + explicitlyMockPipelineVariable('lfDefaults') + } + + def "Test lfJava [Should] throw exception [When] mvnSettings is null" () { + setup: + when: + lfJava({mvnSettings = null}) + then: + thrown Exception + } + + def "Test lfJava [Should] build maven [When] called" () { + setup: + explicitlyMockPipelineStep('withMaven') + getPipelineMock('lfDefaults.call')() >> { + return defaults + } + getPipelineMock("libraryResource")('shell/python-tools-install.sh') >> { + return 'python-tools-install' + } + getPipelineMock("lfCommon.installPythonTools").call(_) >> null + getPipelineMock("lfCommon.jacocoNojava").call(_) >> null + when: + lfJava() + then: + 1 * getPipelineMock('withMaven').call(_) >> { _arguments -> + // Keys are different, but all config values in withMaven should + // match what we passed. + _arguments[0][0].values().each { i -> + assert defaults.values().contains(i) + } + } + 1 * getPipelineMock('sh').call("mvn clean install") + } + + def "Test lfJava [Should] build & deploy maven [When] branch == 'master'" () { + setup: + explicitlyMockPipelineStep("withMaven") + getPipelineMock("lfDefaults.call")() >> { + return defaults + } + getPipelineMock("libraryResource")("shell/python-tools-install.sh") >> { + return "python-tools-install" + } + getPipelineMock("lfCommon.installPythonTools").call(_) >> null + getPipelineMock("lfCommon.jacocoNojava").call(_) >> null + + explicitlyMockPipelineStep("mavenDeploy") + lfJava.getBinding().setVariable("env", ["GIT_BRANCH": "master"]) + + getPipelineMock("libraryResource")("shell/common-variables.sh") >> { + return "common-variables" + } + getPipelineMock("libraryResource")("shell/maven-deploy.sh") >> { + return "maven-deploy" + } + when: + lfJava() + then: + 2 * getPipelineMock('withMaven').call(_) >> { _arguments -> + // Keys are different, but all config values in withMaven should + // match what we passed. + _arguments[0][0].values().each { i -> + assert defaults.values().contains(i) + } + } + 1 * getPipelineMock('sh').call("mvn clean install") + 1 * getPipelineMock('sh').call([script: "common-variables" + "\n" + "maven-deploy"]) + } + + def "Test lfJava [Should] build & stage maven [When] comment contains 'stage'" () { + setup: + explicitlyMockPipelineStep("withMaven") + getPipelineMock('lfDefaults.call')() >> { + return defaults + } + getPipelineMock("libraryResource")('shell/python-tools-install.sh') >> { + return 'python-tools-install' + } + getPipelineMock("lfCommon.installPythonTools").call(_) >> null + getPipelineMock("lfCommon.jacocoNojava").call(_) >> null + + explicitlyMockPipelineStep("mavenStage") + lfJava.getBinding().setVariable("env", ["GITHUB_COMMENT": "stage-release"]) + + getPipelineMock("libraryResource")("shell/common-variables.sh") >> { + return "common-variables" + } + getPipelineMock("libraryResource")("shell/maven-stage.sh") >> { + return "maven-stage" + } + when: + lfJava() + then: + 2 * getPipelineMock('withMaven').call(_) >> { _arguments -> + // Keys are different, but all config values in withMaven should + // match what we passed. + _arguments[0][0].values().each { i -> + assert defaults.values().contains(i) + } + } + 1 * getPipelineMock('sh').call("mvn clean install") + 1 * getPipelineMock('sh').call([script: "common-variables" + "\n" + "maven-stage"]) + } +} diff --git a/tox.ini b/tox.ini index 701d8cf..8dd9362 100644 --- a/tox.ini +++ b/tox.ini @@ -45,6 +45,7 @@ commands = lftools license check-dir -r '.+' vars [testenv:pre-commit] description = Precommit checks for black, gitlint, etc. deps = pre-commit +passenv = HOME commands = pre-commit run --all-files --show-diff-on-failure pre-commit run gitlint --hook-stage commit-msg --commit-msg-filename .git/COMMIT_EDITMSG diff --git a/vars/lfCommon.groovy b/vars/lfCommon.groovy new file mode 100644 index 0000000..3795f42 --- /dev/null +++ b/vars/lfCommon.groovy @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (c) 2021 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Common functions + */ + +// Replaces lf-infra-pre-build +def installPythonTools() { + sh(script: libraryResource('shell/python-tools-install.sh')) +} + +// Replaces lf-jacoco-nojava-workaround +def jacocoNojava() { + sh("mkdir -p $WORKSPACE/target/classes $WORKSPACE/jacoco/classes") +} + +def updateJavaAlternatives(javaVersion) { + bashScript = [ + "SET_JDK_VERSION=$javaVersion", + libraryResource('shell/update-java-alternatives.sh') + ].join("\n") + sh(script: bashScript) + // TODO: Inject /tmp/java.env +} + +def sigulSignDir(signDir, signMode) { + if (signMode == "") { + signMode = "serial" + } + configFileProvider([ + configFile(fileId: 'sigul-config', variable: 'SIGUL_CONFIG'), + configFile(fileId: 'sigul-password', variable: 'SIGUL_PASSWORD'), + configFile(fileId: 'sigul-pki', variable: 'SIGUL_PKI') + ]) { + configAndInstall = [ + libraryResource('shell/sigul-configuration.sh'), + libraryResource('shell/sigul-install.sh') + ].join("\n") + withEnv([ + "SIGN_DIR=$signDir", + "SIGN_MODE=$signMode" + ]) { + sh(script: configAndInstall) + } + sh(script: libraryResource('shell/sigul-configuration-cleanup.sh')) + } +} diff --git a/vars/lfDefaults.groovy b/vars/lfDefaults.groovy new file mode 100644 index 0000000..1540a67 --- /dev/null +++ b/vars/lfDefaults.groovy @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (c) 2021 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Provides default values for common variables within the library. + * + */ +def call(body) { + def defaults = [ + lftoolsVersion: "<1.0.0", + packerVersion: "1.7.2", + jenkinsSshCredential: "jenkins-ssh", + buildDaysToKeep: 30, + buildTimeout: 60, + archiveArtifacts: "", + javaVersion: "openjdk11", + + mvnGoals: "clean install", + mvnVersion: "mvn35", + mvnGlobalSettings: "global-settings", + ] + return defaults +} diff --git a/vars/lfInfraShipLogs.groovy b/vars/lfInfraShipLogs.groovy index 1e6b83d..f86e9d0 100644 --- a/vars/lfInfraShipLogs.groovy +++ b/vars/lfInfraShipLogs.groovy @@ -51,8 +51,7 @@ def call(body) { sh(script: libraryResource('shell/create-netrc.sh')) } - echo 'Running shell/python-tools-install.sh' - sh(script: libraryResource('shell/python-tools-install.sh')) + lfCommon.installPythonTools() echo 'Running shell/sudo-logs.sh' sh(script: libraryResource('shell/sudo-logs.sh')) diff --git a/vars/lfJava.groovy b/vars/lfJava.groovy new file mode 100644 index 0000000..dca9d1a --- /dev/null +++ b/vars/lfJava.groovy @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (c) 2021 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * Method to run Java jobs. + * Required body values: + * * mvnSettings: Maven settings config file ID + * Optional body values (should be defined in lfDefaults): + * * javaVersion + * * mvnGlobalSettings + * * mvnGoals + * * mvnVersion + * + * @param body Config values to be provided in the form "key = value". + */ +def call(body) { + // Evaluate the body block and collect configuration into the object + def defaults = lfDefaults() + def config = [:] + + if (body) { + body.resolveStrategy = Closure.DELEGATE_FIRST + body.delegate = config + body() + } + + // For duplicate keys, Groovy will use the right hand map's values. + config = defaults + config + + if (!config.mvnSettings) { + throw new Exception("Maven settings file id (mvnSettings) is " + + "required for lfJava function.") + } + + //////////////////////// + // Default parameters // + //////////////////////// + archiveArtifacts = """**/*.log +**/hs_err_*.log +**/target/**/feature.xml +**/target/failsafe-reports/failsafe-summary.xml +**/target/surefire-reports/*-output.txt""" + + lfCommon.installPythonTools() + lfCommon.jacocoNojava() + + withMaven( + maven: config.mvnVersion, + jdk: config.javaVersion, + mavenSettingsConfig: config.mvnSettings, + globalMavenSettingsConfig: config.mvnGlobalSettings, + ) { + sh "mvn ${config.mvnGoals}" + } + + if (env.GIT_BRANCH == "main" || env.GIT_BRANCH == "master") { + mavenDeploy(config) + } + + // TODO: Make this generic, rather than Github-specific. A function in + // lfCommon that will take comments from different providers and check them + // would be ideal. + if (env.GITHUB_COMMENT =~ /stage/) { + mavenStage(config) + } +} + +def mavenDeploy(config) { + deployScript = [ + libraryResource('shell/common-variables.sh'), + libraryResource('shell/maven-deploy.sh') + ].join("\n") + + withMaven( + maven: config.mvnVersion, + jdk: config.javaVersion, + mavenSettingsConfig: config.mvnSettings, + globalMavenSettingsConfig: config.mvnGlobalSettings, + ) { + sh(script: deployScript) + } +} + +def mavenStage(config) { + lfCommon.sigulSignDir(".", "") + stageScript = [ + libraryResource('shell/common-variables.sh'), + libraryResource('shell/maven-stage.sh') + ].join("\n") + + withMaven( + maven: config.mvnVersion, + jdk: config.javaVersion, + mavenSettingsConfig: config.mvnSettings, + globalMavenSettingsConfig: config.mvnGlobalSettings, + ) { + sh(script: stageScript) + } +} -- 2.16.6