Add lfParallelCostCapture function

This is a function to be run as a "post" call at the end of each
stage within a parallel block, which will gather cost data from
dynamically-allocated build nodes. It outputs a file titled
"stack-cost" and stashes it. lfInfraShipLogs has been modified to
unstash this file, which will automatically be picked up by the
job-cost.sh script.

Issue: RELENG-3207
Change-Id: I284cd8a2f878aed4e09dacf57fe47587abeffd1b
Signed-off-by: Eric Ball <eball@linuxfoundation.org>

diff --git a/docs/vars/lfParallelCostCapture.rst b/docs/vars/lfParallelCostCapture.rst
new file mode 100644 (file)
index 0000000..5f8b788
--- /dev/null
+++ b/docs/vars/lfParallelCostCapture.rst
@@ -0,0 +1,47 @@
+#####################
+lfParallelCostCapture
+#####################
+
+Parameters
+==========
+
+None.
+
+Usage
+=====
+
+This function is designed to run at the end of a parallel stage that has spun
+up a temporary node. It will pull in the cost data, and stash it in a file
+titled "stack-cost". This can then be picked up by the job-cost.sh script.
+
+.. warning::
+    Calling this function requires that the Lockable Resources Plugin is
+    installed.
+
+An example of the intended implementation:
+
+.. code-block:: groovy
+
+    parallel {
+        stage('1') {
+            node {newNode1}  // Only add post stage if a new node is being used
+            stages {<all stages>}
+            post {
+                always {
+                    lfParallelCostCapture()
+                }
+            }
+        }
+        stage('2') {
+            node {newNode2}
+            stages {<all stages>}
+            post {
+                always {
+                    lfParallelCostCapture()
+                }
+            }
+        }
+    }
+
+The calling pipeline must manually unstash the "stack-cost" file. For LF
+Pipelines, this is handled by the lfInfraShipLogs function.

diff --git a/global-jjb b/global-jjb
index e3b47c7..53f811d 160000 (submodule)
--- a/global-jjb
+++ b/global-jjb
@@ -1 +1 @@
-Subproject commit e3b47c77a181b5f5bd3a79802a46188adb6173ce
+Subproject commit 53f811d91411bf26a4acf00a9244c6ea0f4510d5

diff --git a/releasenotes/notes/add-parallel-cost-capture-feebf1de5b4f3ec6.yaml b/releasenotes/notes/add-parallel-cost-capture-feebf1de5b4f3ec6.yaml
new file mode 100644 (file)
index 0000000..3ebdc6e
--- /dev/null
+++ b/releasenotes/notes/add-parallel-cost-capture-feebf1de5b4f3ec6.yaml
@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    New global var lfParallelCostCapture. This is a function to be run as a
+    "post" call at the end of each stage within a parallel block, which will
+    gather cost data from dynamically-allocated build nodes. It outputs a file
+    titled "stack-cost" and stashes it. lfInfraShipLogs has been modified to
+    unstash this file, which will automatically be picked up by the job-cost.sh
+    script.

diff --git a/vars/lfInfraShipLogs.groovy b/vars/lfInfraShipLogs.groovy
index 3113d0a..1e6b83d 100644 (file)
--- a/vars/lfInfraShipLogs.groovy
+++ b/vars/lfInfraShipLogs.groovy
@@ -55,6 +55,13 @@ def call(body) {
             sh(script: libraryResource('shell/python-tools-install.sh'))
             echo 'Running shell/sudo-logs.sh'
             sh(script: libraryResource('shell/sudo-logs.sh'))
+
+            // Check for stashed "stack-cost" file, used to get cost data from
+            // parallel builds.
+            try {
+                unstash "stack-cost"
+            } catch(Exception e) {}
+
             echo 'Running shell/job-cost.sh'
             sh(script: libraryResource('shell/job-cost.sh'))
 

diff --git a/vars/lfParallelCostCapture.groovy b/vars/lfParallelCostCapture.groovy
new file mode 100644 (file)
index 0000000..41e89f4
--- /dev/null
+++ b/vars/lfParallelCostCapture.groovy
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// Copyright (c) 2019 Intel Corporation
+// Copyright (c) 2020 The Linux Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+def call() {
+    sh(script: libraryResource('shell/job-cost.sh'))
+    cost_str = sh(script: "cat $WORKSPACE/archives/cost.csv | cut -d, -f6", returnStdout: true)
+
+    lock("${BUILD-TAG}-stack-cost") {
+        try {
+            unstash "stack-cost"
+            stack_cost = sh(script: "cat stack-cost | awk '{print \$2}", returnStdout: true)
+        } catch(Exception e) {
+            stack_cost = 0
+        }
+
+        cost = (cost_str as float) + (stack_cost as float)
+        sh("echo total: ${cost} > stack-cost")
+        stash includes: "**/stack-cost", name: "stack-cost"
+    }
+}