2 # SPDX-License-Identifier: EPL-1.0
3 ##############################################################################
4 # Copyright (c) 2020 The Linux Foundation and others.
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Eclipse Public License v1.0
8 # which accompanies this distribution, and is available at
9 # http://www.eclipse.org/legal/epl-v10.html
10 ##############################################################################
12 # This script will be run on the Nexus Server for each project. Typically each
13 # project will have multiple nexus silos. This script will find all the job cost
14 # files (job_name/job_num/cost.csv) in each silo and append them to the annual
15 # cost file (~nexus/cost/$silo.YYYY.csv~nexus/bin . It will then delete all the
18 # Because this file meant to be run by cron, I have restricted normal info
19 # logging messages on a single line. Error may be multi-line.
21 # Each cost file contains one or more CSV records in the following format:
23 # JobName , BuildNumber , Date , InstanceType , Uptime , Cost , StackCost
25 # Date format: '%Y-%m-%d %H:%M:%S'
26 # Cost format: '%.2f' (1.29)
28 ##############################################################################
32 # You can validate this script by running as yourself on the Nexus server. You
33 # should not have write permission anywhere in the Silo Directory. The
34 # Silo Cost File from your test will be created/updated in:
35 # ~/cost/$silo-$year.csv. If you run multiple times, duplicate records
38 # To enable debug set envionment variable DEBUG=true. If this is done on the
39 # command-line, you do not have to edit this file.
41 $debug && echo "DEBUG Enabled"
43 ##############################################################################
47 function get-year-list()
49 # Grab the years for each cost record use sort | uniq to get the
50 # list of unique years found
52 list=$(awk -F',' '{print $3}' "$cost_file_records" \
53 | awk -F'-' '{print $1}' | sort | uniq)
57 ########### End of Function Definitions ######################################
59 if [[ $# != 2 ]]; then
60 echo "usage: $(basename "$0") silo silo_dir"
64 # The Silo Dir is top-level directory that will contain the job directories
65 # which will contain the cost files (cost.csv)
69 cost_file_records=/tmp/cost-file-records$$
70 cost_file_list=/tmp/cost-file-list$$
71 # The directory where the annual cost file will be located
73 [[ -d $cost_dir ]] || mkdir $cost_dir
75 # The Silo Directory for sandbox will get deleted periodically, so
76 # gracefully handle that
77 if [[ -d $silo_dir ]]; then
80 echo "$(date +'%Y-%m-%d %H:%M') No Silo Directory, nothing to do"
84 find . -maxdepth 3 -name cost.csv > $cost_file_list
85 xargs cat < $cost_file_list | \
86 sort --field-separator=',' --key=3 > $cost_file_records
87 num_of_records=$(wc -l < $cost_file_records)
88 echo -n "$(date +'%Y-%m-%d %H:%M') Records: $num_of_records "
90 if [[ $num_of_records == 0 ]]; then
93 rm -rf /tmp/cost-file-* || true
97 # Append each entry to the silo cost file based on date
98 year_list=$(get-year-list)
99 for year in $year_list; do
100 echo -n "cost-$year.csv: $(grep -Fc ",$year-" $cost_file_records) "
101 grep -F ",$year-" $cost_file_records >> \
102 "$cost_dir/$silo-$year.csv"
105 # Rename the job cost files (make them hidden)
107 job_dir=$(dirname "$p")
108 $debug || (cd "$job_dir" ; mv cost.csv .cost.csv)
109 done < $cost_file_list
111 rm -r $cost_file_list $cost_file_records
114 # Sort the silo cost file by 'date' (column 3)
115 for year in $year_list; do
116 echo -n "cost-$year.csv "
117 sort --field-separator=',' --key=3 \
118 -o "$cost_dir/$silo-$year.csv" \
119 "$cost_dir/$silo-$year.csv"
123 rm -rf /tmp/cost-file-* || true
125 # Keep track of time initally
126 echo "Complete $SECONDS Secs"