From: Tim Johnson <tijohnson@linuxfoundation.org>
Date: Fri, 8 Nov 2019 17:11:01 +0000 (-0800)
Subject: Get build cost estimates
X-Git-Tag: v0.49.0~5
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=3937b8cb2c1b6ba0997d5aeba1b38085e57a7d2f;p=releng%2Fglobal-jjb.git

Get build cost estimates

Generate and archive files containing cost, time & job info. If this is
a Openstack job call 'lftools openstack stack cost' to retrieve and
include costs from the stack members. Added two new scripts:
openstack-stack-delete.sh & job-cost.sh. The 'openstack-stack-delete
script' will retrieve the costs for the stack and delete the stack. The
'job-cost' will generate the job cost file which will include the costs
for the build agent and any Openstack members.

Issue: RELENG-1867
Change-Id: I658f5b42f90a28010e6f98f2f3c3ebb166192c30
Signed-off-by: Tim Johnson <tijohnson@linuxfoundation.org>
---

diff --git a/jjb/lf-macros.yaml b/jjb/lf-macros.yaml
index 69848114..e505d879 100644
--- a/jjb/lf-macros.yaml
+++ b/jjb/lf-macros.yaml
@@ -78,6 +78,8 @@
           - ../shell/python-tools-install.sh
       - shell: !include-raw:
           - ../shell/sudo-logs.sh
+      - shell: !include-raw:
+          - ../shell/job-cost.sh
       - shell: !include-raw:
           - ../shell/logs-deploy.sh
       - shell: !include-raw:
diff --git a/jjb/lf-openstack-heat.yaml b/jjb/lf-openstack-heat.yaml
index 4ca5e8f6..0797e34f 100644
--- a/jjb/lf-openstack-heat.yaml
+++ b/jjb/lf-openstack-heat.yaml
@@ -36,8 +36,5 @@
                     files:
                       - file-id: clouds-yaml
                         target: "$HOME/.config/openstack/clouds.yaml"
-                - shell: |
-                    #!/bin/bash -l
-                    echo "Deleting $OS_STACK_NAME"
-                    lftools openstack --os-cloud "$OS_CLOUD" stack delete "$OS_STACK_NAME"
+                - shell: !include-raw-escape: ../shell/openstack-stack-delete.sh
           mark-unstable-if-failed: false
diff --git a/releasenotes/notes/get-job-cost-be3f4061e9cbdc90.yaml b/releasenotes/notes/get-job-cost-be3f4061e9cbdc90.yaml
new file mode 100644
index 00000000..b6c02f76
--- /dev/null
+++ b/releasenotes/notes/get-job-cost-be3f4061e9cbdc90.yaml
@@ -0,0 +1,19 @@
+---
+features:
+  - |
+    Generate Job cost information Each job it will archive a CSV file
+    (cost.csv). It will contain a single CSV record containing the following
+    fields: JobName , BuildNumber , Date , InstanceType , Uptime , Cost1 , Cost2
+    The Date field can be sorted as a string and is readable by your favorite
+    spreadsheet.
+    The Date/Time is GMT.
+    The Uptime is uptime of the build agent  (secs).
+    The Cost1 field is the cost($$) of the build node & Cost2 is cost associated
+    with the stack. If the job is not a Openstack job, then Cost2 is '0'.  The
+    project cost file will based on the year (cost-2019.csv).
+
+upgrade:
+  - |
+    The openstack-stack-delete.sh script installs the latest tagged version of
+    lftools and it uses that to get the stack cost. Any version of lftools >=
+    v0.29.0 will contain the required changes to get the stack cost.
diff --git a/shell/job-cost.sh b/shell/job-cost.sh
new file mode 100644
index 00000000..2264728d
--- /dev/null
+++ b/shell/job-cost.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo "---> build-cost.sh"
+
+set -euf -o pipefail
+
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+lf-activate-venv python-openstackclient
+
+if [[ -z ${JOB_NAME:-} ]]; then
+    lf-echo-error "Required Env Variable Unset/Empty: JOB_NAME"
+    exit 1
+fi
+
+# Get the cost of the Openstack agents. The 'stack-cost' file is created when
+# the 'lftools openstack stack cost' command is called from
+# 'openstack-stack-delete.sh' script. The 'stack-cost' file will only be created
+# if this is an openstack job.
+if [[ -f stack-cost ]]; then
+    echo "DEBUG: $(cat stack-cost)"
+    echo "INFO: Retrieving Stack Cost..."
+    if ! stack_cost=$(fgrep "total: " stack-cost | awk '{print $2}'); then
+        echo "ERROR: Unable to retrieve Stack Cost, continuing anyway"
+        stack_cost=0
+    fi
+else
+    echo "INFO: No Stack..."
+    stack_cost=0
+fi
+
+# Retrieve the current uptime (in seconds)
+uptime=$(cat /proc/uptime | awk '{print $1}')
+# Convert to integer by truncating fractional part' and round up by one
+((uptime=${uptime%\.*}+1))
+
+instance_type=$(curl -s http://169.254.169.254/latest/meta-data/instance-type)
+
+echo "INFO: Retrieving Pricing Info for: $instance_type"
+url="https://pricing.vexxhost.net/v1/pricing/$instance_type/cost?seconds=$uptime"
+jason_block=$(curl -s $url)
+
+cost=$(jq .cost <<< $jason_block)
+resource=$(jq .resource <<< $jason_block | tr -d '"')
+
+# Archive the cost date
+mkdir -p $WORKSPACE/archives/cost
+
+echo "INFO: Archiving Costs"
+
+# Set the timestamp in GMT
+# This format is readable by spreadsheet and is easily sortable
+date=$(TZ=GMT date +'%Y-%m-%d %H:%M:%S')
+
+cat << EOF > $WORKSPACE/archives/cost.csv
+$JOB_NAME,$BUILD_NUMBER,$date,$resource,$uptime,$cost,$stack_cost
+EOF
+
diff --git a/shell/openstack-stack-delete.sh b/shell/openstack-stack-delete.sh
new file mode 100644
index 00000000..6124c7e2
--- /dev/null
+++ b/shell/openstack-stack-delete.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2019 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+echo "---> openstack-stack-delete.sh"
+
+set -eufo pipefail
+
+# shellcheck disable=SC1090
+source ~/lf-env.sh
+
+lf-activate-venv lftools[openstack] python-openstackclient
+
+echo "INFO: Retrieving stack cost for: $OS_STACK_NAME"
+if ! lftools openstack --os-cloud $OS_CLOUD stack cost $OS_STACK_NAME > stack-cost; then
+    echo "WARNING: Unable to get stack costs, continuing anyway"
+    echo "total: 0" > stack-cost
+else
+    echo "DEBUG: Successfully retrieved stack cost: $(cat stack-cost)"
+fi
+
+# Delete the stack even if the stack-cost script fails
+lftools openstack --os-cloud "$OS_CLOUD" stack delete "$OS_STACK_NAME" \
+    | echo "INFO: $(cat)"
+