From: Tim Johnson <tijohnson@linuxfoundation.org>
Date: Mon, 9 Dec 2019 16:13:09 +0000 (-0800)
Subject: Add Script to scrape cost data
X-Git-Tag: v0.51.0~8
X-Git-Url: https://gerrit.linuxfoundation.org/infra/gitweb?a=commitdiff_plain;h=3702b8680891a0392a53a2e2ce8060fb75b2bbdf;p=releng%2Fglobal-jjb.git

Add Script to scrape cost data

Issue: RELENG-2555
Change-Id: I46e719f88721fd8cf0060181297d2655995e47f7
Signed-off-by: Tim Johnson <tijohnson@linuxfoundation.org>
---

diff --git a/releasenotes/notes/scrape-job-cost-2a43bea721775985.yaml b/releasenotes/notes/scrape-job-cost-2a43bea721775985.yaml
new file mode 100644
index 00000000..6994dfe9
--- /dev/null
+++ b/releasenotes/notes/scrape-job-cost-2a43bea721775985.yaml
@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    New script has been added: scrape-job-cost. This script will be executed by
+    cron as nexus on the Nexus Server for each project. It will extract cost
+    data from the nexus directory for each Jenkins Builder (production &
+    sandbox). The cost data for each builder will be appended to separate cost
+    files located in ~nexus/cost on the nexus server. The files will be named
+    sandbox-YYYY.csv and production-YYYY.csv.
diff --git a/shell/scrape-job-cost.sh b/shell/scrape-job-cost.sh
new file mode 100755
index 00000000..9bd5205e
--- /dev/null
+++ b/shell/scrape-job-cost.sh
@@ -0,0 +1,125 @@
+#! /bin/bash
+# SPDX-License-Identifier: EPL-1.0
+##############################################################################
+# Copyright (c) 2020 The Linux Foundation and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License v1.0
+# which accompanies this distribution, and is available at
+# http://www.eclipse.org/legal/epl-v10.html
+##############################################################################
+
+# This script will be run on the Nexus Server for each project.  Typically each
+# project will have multiple nexus silos. This script will find all the job cost
+# files (job_name/job_num/cost.csv) in each silo and append them to the annual
+# cost file (~nexus/cost/$silo.YYYY.csv~nexus/bin . It will then delete all the
+# job cost files.
+
+# Because this file meant to be run by cron, I have restricted normal info
+# logging messages on a single line. Error may be multi-line.
+#
+# Each cost file contains one or more CSV records in the following format:
+#
+#   JobName , BuildNumber , Date , InstanceType , Uptime , Cost1 , Cost2
+#
+#   Date format: '%Y-%m-%d %H:%M:%S'
+#
+##############################################################################
+#
+#  Testing/Validation
+#
+#  You can validate this script by running as yourself on the Nexus server. You
+#  should not have write permission anywhere in the Silo Directory. The
+#  Silo Cost File from your test will be created/updated in:
+#      ~/cost/$silo-$year.csv. If you run multiple times, duplicate records
+#      will be created.
+#
+#  To enable debug set envionment variable DEBUG=true. If this is done on the
+#  command-line, you  do not have to edit this file.
+debug=${DEBUG:-false}
+$debug && echo "DEBUG Enabled"
+#
+##############################################################################
+
+set -eufo pipefail
+
+function get-year-list()
+{
+    # Grab the years for each cost record use sort | uniq to get the
+    # list of unique years found
+    local list
+    list=$(awk -F',' '{print $3}' "$cost_file_records" \
+               | awk -F'-' '{print $1}' | sort | uniq)
+    echo "$list"
+}
+
+###########  End of Function Definitions  ######################################
+
+if [[ $# != 2 ]]; then
+    echo "usage: $(basename "$0") silo silo_dir"
+    exit 1
+fi
+
+# The Silo Dir is top-level directory that will contain the job directories
+# which will contain the cost files (cost.csv)
+silo=$1
+silo_dir=$2
+
+cost_file_records=/tmp/cost-file-records$$
+cost_file_list=/tmp/cost-file-list$$
+# The directory where the annual cost file will be located
+cost_dir=~/cost
+[[ -d $cost_dir ]] || mkdir $cost_dir
+
+# The Silo Directory for sandbox will get deleted periodically, so
+# gracefully handle that
+if [[ -d $silo_dir ]]; then
+   cd "$silo_dir"
+else
+    echo  "$(date +'%Y-%m-%d %H:%M') No Silo Directory, nothing to do"
+    exit 0
+fi
+
+find . -maxdepth 3 -name cost.csv > $cost_file_list
+xargs cat < $cost_file_list | \
+    sort --field-separator=',' --key=3  > $cost_file_records
+num_of_records=$(wc -l < $cost_file_records)
+echo -n "$(date +'%Y-%m-%d %H:%M') Records: $num_of_records "
+
+if [[ $num_of_records == 0 ]]; then
+    echo "Nothing to do"
+    set +f
+    rm -rf /tmp/cost-file-* || true
+    exit 0
+fi
+
+# Append each entry to the silo cost file based on date
+year_list=$(get-year-list)
+for year in $year_list; do
+    echo -n "cost-$year.csv: $(grep -Fc ",$year-" $cost_file_records) "
+    grep -F ",$year-" $cost_file_records >>    \
+         "$cost_dir/$silo-$year.csv"
+done
+
+# Rename the job cost files (make them hidden)
+while read -r p; do
+    job_dir=$(dirname "$p")
+    $debug || (cd "$job_dir" ; mv cost.csv .cost.csv)
+done < $cost_file_list
+
+rm -r $cost_file_list $cost_file_records
+
+echo -n "Sorting: "
+# Sort the silo cost file by 'date' (column 3)
+for year in $year_list; do
+    echo -n "cost-$year.csv "
+    sort --field-separator=',' --key=3  \
+         -o "$cost_dir/$silo-$year.csv" \
+         "$cost_dir/$silo-$year.csv"
+done
+
+set +f
+rm -rf /tmp/cost-file-* || true
+
+# Keep track of time initally
+echo "Complete $SECONDS Secs"