2 # SPDX-License-Identifier: EPL-1.0
3 ##############################################################################
4 # Copyright (c) 2020 The Linux Foundation and others.
6 # All rights reserved. This program and the accompanying materials
7 # are made available under the terms of the Eclipse Public License v1.0
8 # which accompanies this distribution, and is available at
9 # http://www.eclipse.org/legal/epl-v10.html
10 ##############################################################################
12 # This script will be run on the Nexus Server for each project. Typically each
13 # project will have multiple nexus silos. This script will find all the job cost
14 # files (job_name/job_num/cost.csv) in each silo and append them to the annual
15 # cost file (~nexus/cost/$silo.YYYY.csv~nexus/bin . It will then delete all the
18 # Because this file meant to be run by cron, I have restricted normal info
19 # logging messages on a single line. Error may be multi-line.
21 # Each cost file contains one or more CSV records in the following format:
23 # JobName , BuildNumber , Date , InstanceType , Uptime , Cost1 , Cost2
25 # Date format: '%Y-%m-%d %H:%M:%S'
27 ##############################################################################
31 # You can validate this script by running as yourself on the Nexus server. You
32 # should not have write permission anywhere in the Silo Directory. The
33 # Silo Cost File from your test will be created/updated in:
34 # ~/cost/$silo-$year.csv. If you run multiple times, duplicate records
37 # To enable debug set envionment variable DEBUG=true. If this is done on the
38 # command-line, you do not have to edit this file.
40 $debug && echo "DEBUG Enabled"
42 ##############################################################################
46 function get-year-list()
48 # Grab the years for each cost record use sort | uniq to get the
49 # list of unique years found
51 list=$(awk -F',' '{print $3}' "$cost_file_records" \
52 | awk -F'-' '{print $1}' | sort | uniq)
56 ########### End of Function Definitions ######################################
58 if [[ $# != 2 ]]; then
59 echo "usage: $(basename "$0") silo silo_dir"
63 # The Silo Dir is top-level directory that will contain the job directories
64 # which will contain the cost files (cost.csv)
68 cost_file_records=/tmp/cost-file-records$$
69 cost_file_list=/tmp/cost-file-list$$
70 # The directory where the annual cost file will be located
72 [[ -d $cost_dir ]] || mkdir $cost_dir
74 # The Silo Directory for sandbox will get deleted periodically, so
75 # gracefully handle that
76 if [[ -d $silo_dir ]]; then
79 echo "$(date +'%Y-%m-%d %H:%M') No Silo Directory, nothing to do"
83 find . -maxdepth 3 -name cost.csv > $cost_file_list
84 xargs cat < $cost_file_list | \
85 sort --field-separator=',' --key=3 > $cost_file_records
86 num_of_records=$(wc -l < $cost_file_records)
87 echo -n "$(date +'%Y-%m-%d %H:%M') Records: $num_of_records "
89 if [[ $num_of_records == 0 ]]; then
92 rm -rf /tmp/cost-file-* || true
96 # Append each entry to the silo cost file based on date
97 year_list=$(get-year-list)
98 for year in $year_list; do
99 echo -n "cost-$year.csv: $(grep -Fc ",$year-" $cost_file_records) "
100 grep -F ",$year-" $cost_file_records >> \
101 "$cost_dir/$silo-$year.csv"
104 # Rename the job cost files (make them hidden)
106 job_dir=$(dirname "$p")
107 $debug || (cd "$job_dir" ; mv cost.csv .cost.csv)
108 done < $cost_file_list
110 rm -r $cost_file_list $cost_file_records
113 # Sort the silo cost file by 'date' (column 3)
114 for year in $year_list; do
115 echo -n "cost-$year.csv "
116 sort --field-separator=',' --key=3 \
117 -o "$cost_dir/$silo-$year.csv" \
118 "$cost_dir/$silo-$year.csv"
122 rm -rf /tmp/cost-file-* || true
124 # Keep track of time initally
125 echo "Complete $SECONDS Secs"