From dc8816ee880a4d21fd6b6b866f851d2dc8b9768a Mon Sep 17 00:00:00 2001 From: Eric Ball Date: Fri, 22 Jan 2021 15:18:37 -0800 Subject: [PATCH] Add lfParallelCostCapture function This is a function to be run as a "post" call at the end of each stage within a parallel block, which will gather cost data from dynamically-allocated build nodes. It outputs a file titled "stack-cost" and stashes it. lfInfraShipLogs has been modified to unstash this file, which will automatically be picked up by the job-cost.sh script. Issue: RELENG-3207 Change-Id: I284cd8a2f878aed4e09dacf57fe47587abeffd1b Signed-off-by: Eric Ball --- docs/vars/lfParallelCostCapture.rst | 47 ++++++++++++++++++++++ global-jjb | 2 +- ...add-parallel-cost-capture-feebf1de5b4f3ec6.yaml | 9 +++++ vars/lfInfraShipLogs.groovy | 7 ++++ vars/lfParallelCostCapture.groovy | 34 ++++++++++++++++ 5 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 docs/vars/lfParallelCostCapture.rst create mode 100644 releasenotes/notes/add-parallel-cost-capture-feebf1de5b4f3ec6.yaml create mode 100644 vars/lfParallelCostCapture.groovy diff --git a/docs/vars/lfParallelCostCapture.rst b/docs/vars/lfParallelCostCapture.rst new file mode 100644 index 0000000..5f8b788 --- /dev/null +++ b/docs/vars/lfParallelCostCapture.rst @@ -0,0 +1,47 @@ +##################### +lfParallelCostCapture +##################### + +Parameters +========== + +None. + +Usage +===== + +This function is designed to run at the end of a parallel stage that has spun +up a temporary node. It will pull in the cost data, and stash it in a file +titled "stack-cost". This can then be picked up by the job-cost.sh script. + +.. warning:: + Calling this function requires that the Lockable Resources Plugin is + installed. + +An example of the intended implementation: + +.. code-block:: groovy + + parallel { + stage('1') { + node {newNode1} // Only add post stage if a new node is being used + stages {} + post { + always { + lfParallelCostCapture() + } + } + } + stage('2') { + node {newNode2} + stages {} + post { + always { + lfParallelCostCapture() + } + } + } + } + +The calling pipeline must manually unstash the "stack-cost" file. For LF +Pipelines, this is handled by the lfInfraShipLogs function. diff --git a/global-jjb b/global-jjb index e3b47c7..53f811d 160000 --- a/global-jjb +++ b/global-jjb @@ -1 +1 @@ -Subproject commit e3b47c77a181b5f5bd3a79802a46188adb6173ce +Subproject commit 53f811d91411bf26a4acf00a9244c6ea0f4510d5 diff --git a/releasenotes/notes/add-parallel-cost-capture-feebf1de5b4f3ec6.yaml b/releasenotes/notes/add-parallel-cost-capture-feebf1de5b4f3ec6.yaml new file mode 100644 index 0000000..3ebdc6e --- /dev/null +++ b/releasenotes/notes/add-parallel-cost-capture-feebf1de5b4f3ec6.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + New global var lfParallelCostCapture. This is a function to be run as a + "post" call at the end of each stage within a parallel block, which will + gather cost data from dynamically-allocated build nodes. It outputs a file + titled "stack-cost" and stashes it. lfInfraShipLogs has been modified to + unstash this file, which will automatically be picked up by the job-cost.sh + script. diff --git a/vars/lfInfraShipLogs.groovy b/vars/lfInfraShipLogs.groovy index 3113d0a..1e6b83d 100644 --- a/vars/lfInfraShipLogs.groovy +++ b/vars/lfInfraShipLogs.groovy @@ -55,6 +55,13 @@ def call(body) { sh(script: libraryResource('shell/python-tools-install.sh')) echo 'Running shell/sudo-logs.sh' sh(script: libraryResource('shell/sudo-logs.sh')) + + // Check for stashed "stack-cost" file, used to get cost data from + // parallel builds. + try { + unstash "stack-cost" + } catch(Exception e) {} + echo 'Running shell/job-cost.sh' sh(script: libraryResource('shell/job-cost.sh')) diff --git a/vars/lfParallelCostCapture.groovy b/vars/lfParallelCostCapture.groovy new file mode 100644 index 0000000..41e89f4 --- /dev/null +++ b/vars/lfParallelCostCapture.groovy @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: Apache-2.0 +// +// Copyright (c) 2019 Intel Corporation +// Copyright (c) 2020 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +def call() { + sh(script: libraryResource('shell/job-cost.sh')) + cost_str = sh(script: "cat $WORKSPACE/archives/cost.csv | cut -d, -f6", returnStdout: true) + + lock("${BUILD-TAG}-stack-cost") { + try { + unstash "stack-cost" + stack_cost = sh(script: "cat stack-cost | awk '{print \$2}", returnStdout: true) + } catch(Exception e) { + stack_cost = 0 + } + + cost = (cost_str as float) + (stack_cost as float) + sh("echo total: ${cost} > stack-cost") + stash includes: "**/stack-cost", name: "stack-cost" + } +} -- 2.16.6