#! /s/std/bin/perl -w


# make-email-graph: Grind over the lab maintained archives of
# condor-users and generate a nice graph.
#
# 2007-05-23 bt 	 - Modified for test data plot images
# 2006-09-19 adesmet - Original creation

# Notes:
#
# GNUPlot is really, really annoying.  PNG output doesn't antialias
# the lines, so it's all jaggy.  Also, it's non-obvious how to
# get a given output size.  EPS output works pretty well and
# generates nice output when you convert it to PNG, but _you can't
# pick your plot colors_.  Insanity!


use Date::Manip;
use File::Temp qw(tempfile);
use File::Spec;
use Time::Local;
use Carp;
use strict;

# will be processed.
#my $ARCHIVE_DIR = '/p/list-archives/condor-users';

# First date to graph.  Useful to filter out the occasional
# insane message claiming to be from Midnight, Jan 1, 1970.
my $FIRST_DATE = '1 June 2005 00:00:00';
my $originaldate;

# Width of output in pixels. Height is based on ratio
my $WIDTH = 1000;

# Approximate ratio of height to width.  May not end up
# exactly this ratio; blame GNUplot
#my $RATIO = 0.3;
my $RATIO = 0.3;

# Scale factor for graph as whole
# Setting it larger makes lines thinner and fonts smaller
my $SCALE = 2;

# Constants.  Ignore them.
my $BIN_DAY = 'DAY';
my $BIN_WEEK = 'WEEK';
my $BIN_MONTH = 'MONTH';
my $BIN_YEAR = 'YEAR';

# How to cluster messages.  Supported options are the above $BIN_* settings.
my $BINSIZE = $BIN_MONTH;

# Path to GNUPlot
my $GNUPLOT = '/s/gnuplot-4.2.0/bin/gnuplot';

# Path to ImageMagick's convert
my $CONVERT = 'convert';

# Path to mv
my $MV = '/bin/cp';

# What's the smallest rational PNG file we could generate.  Used to detect
# corruption.  At the moment we're generating files about 90k
my $MIN_PNG_SIZE = 10_000;

################################################################################
################################################################################
#
#  NO USER SERVICABLE PARTS BELOW
#
################################################################################
################################################################################
my $image_source = "";
my $failure_details = 0;
my $details = "";
my $morextics = "";
my $title = "";

exit main();

sub main {

	$failure_details = 0;
	my $final_destination;
	my $binset = 0;

	for(my $i = 0; $i < @ARGV; $i++) {
		if($ARGV[$i] eq '--weekly') { 
			$BINSIZE = $BIN_WEEK;
			if($binset) {
				print STDERR "Warning: Multiple durations set.  Only the last one will take effect.\n";
			}
			$binset = 1;
		} elsif($ARGV[$i] eq '--daily') { 
				$BINSIZE = $BIN_DAY;
				if($binset) {
				print STDERR "Warning: Multiple durations set.  Only the last one will take effect.\n";
			}
			$binset = 1;
		} elsif($ARGV[$i] eq '--monthly') { 
			$BINSIZE = $BIN_MONTH;
			if($binset) {
				print STDERR "Warning: Multiple durations set.  Only the last one will take effect.\n";
			}
			$binset = 1;
		} elsif($ARGV[$i] eq '--yearly') { 
			$BINSIZE = $BIN_YEAR;
			if($binset) {
				print STDERR "Warning: Multiple durations set.  Only the last one will take effect.\n";
			}
			$binset = 1;
		} elsif($ARGV[$i] eq '--output') {
			$i++;
			if($i > @ARGV) {
				usage();
				fatal_error("--output must be followed with a filename to write the resulting image to.");
			}
			if(defined $final_destination) {
				print STDERR "Warning: Multple output files specificed.  Only the last one will take effect.";
			}
			$final_destination = $ARGV[$i];
		} elsif($ARGV[$i] eq '--input') {
			$i++;
			if($i > @ARGV) {
				usage();
				fatal_error("--input must be followed with a filename to read the image data from.");
			}
			$image_source = $ARGV[$i];
		} elsif($ARGV[$i] eq '--title') {
			$i++;
			if($i > @ARGV) {
				usage();
				fatal_error("--title must be followed by title of plot.");
			}
			$title = $ARGV[$i];
		} elsif($ARGV[$i] eq '--detail') {
			$i++;
			if($i > @ARGV) {
				usage();
				fatal_error("--detail must be followed with a kind of plot.");
			}
			$details = $ARGV[$i];
		} elsif($ARGV[$i] eq '--xtics') {
			$i++;
			if($i > @ARGV) {
				usage();
				fatal_error("--xtics must be followed with a file with labels for kind of plot.");
			}
			print "XTICS = $ARGV[$i]\n";
			$morextics = $ARGV[$i];
		} elsif($ARGV[$i] eq '--firstdate') {
			$i++;
			if($i > @ARGV) {
				usage();
				fatal_error("--firstdate must be followed by first date to plot from.");
			}
			$FIRST_DATE = $ARGV[$i] . " 00:00:00";
		} else {
			usage();
			fatal_error("Unknown argument '$ARGV[$i]'.");
		}
	}

	if(not defined $final_destination) {
		usage();
		fatal_error("You must specific an --output file.");
	}

	if($details eq "autotestyear") {
		$FIRST_DATE = '1 June 2007 00:00:00';
		$WIDTH = 500;
	}

	$originaldate = $FIRST_DATE;

	my $firstdate = parse_date($FIRST_DATE);

	#print "<<<<<<<<< Firstdate $firstdate >>>>>>>>>>>>>>>>>>\n";

	$firstdate = bindate($BINSIZE, $firstdate);

	# Find source files.
	my @files = ("$image_source");

	# Read source files 
	# read temp date date, total, passed, failed
	my %total; # messages over time.
	my %staff; # from cs.wisc.edu, over time.
	my($datafh, $datafile) = tempfile_or_die(".data");
	#print "Datafile is $datafile\n";
	foreach my $file (@files) {
		local *IN;
		open IN, $file or fatal_error("Unable to open \"$file\" for reading.");
		my $msg;
		my @dataline;
		my $date;
		my $line;
		my $first = 1;
		while(<IN>) {
			chomp;
			$line = $_;
			#print "working on <<$line>>\n";
			@dataline = split /,/, $line;
			$first = 1;
			foreach my $item (@dataline) {
				if($first) {
					if($details eq "weeklydownloadlogs") {
						print $datafh "$item ";
					} else {
						$date = parse_date($item);
						$date = bindate($BINSIZE, $date);
						#print "BinDate is <<$date>>\n";
						my $epoch_date = epoch_date($date);
						#print "EpochDate is <<$date>>\n";
						print $datafh "$epoch_date ";
					}
					$first = 0;
				} else {
					print $datafh "$item ";
				}
					#print  "$epoch_date $dataline[1] $dataline[2] $dataline[3]\n";
					#print $datafh "$epoch_date $dataline[1] $dataline[2] $dataline[3]\n";
			}
			print $datafh "\n";
		}
		close IN;
		close $datafh;
	}

	# Invoke GNUPlot
	my($epsfh, $epsfile) = tempfile_or_die(".eps");
	close $epsfh;

	my $cmds;
	$cmds = gnuplot_commands(
		title => $title,
		epsfile => $epsfile,
		startdate => epoch_date($firstdate),
		datafile => $datafile,
		binsize => describe_binsize($BINSIZE),
		);

	#print "FULL PLOT command:<<$cmds>>\n";

	local *GP;
	open GP, "|-", $GNUPLOT or fatal_error("unable to run $GNUPLOT because of $!");
	print GP $cmds;
	close GP;
	if($? != 0) {
		fatal_error("$GNUPLOT failed to convert '$datafile' into '$epsfile' using provided commands\n");
	}

	unlink($datafile);


	# Convert to a PNG
	#my($pngfh, $pngfile) = tempfile("email-graph-XXXXXX", DIR => File::Spec->tmpdir(), SUFFIX => '.jpg');
	my($pngfh, $pngfile) = tempfile("condor-nmiplots-XXXXXX", DIR => File::Spec->tmpdir(), SUFFIX => '.png');
	close $pngfh;
	if(system($CONVERT, '-density', '300', $epsfile, '-trim', '-resize', $WIDTH, '-colors', '256', $pngfile) != 0) {
		fatal_error("Problem using $CONVERT to convert '$epsfile' to '$pngfile'.");
	}
	unlink($epsfile);

	# Final sanity check
	if((-s $pngfile) < $MIN_PNG_SIZE) {
		fatal_error("'$pngfile' isn't at least $MIN_PNG_SIZE.  There may be a problem.");
	}

	if( not chmod(0644, $pngfile) ) {
		fatal_error("Unable to fix permissions of '$pngfile'.");
	}

	# Copy to destination
	if(system($MV, $pngfile, $final_destination) != 0) {
		fatal_error("Problem moving '$pngfile' to '$final_destination'.");
	}
	unlink($pngfile);


	return 0;
}

sub usage {
	print <<END;
Usage:
$0 {--daily|--monthly|--weekly|--yearly} --output FILENAME --input FILENAME [ --detail ]
 - "detail" option gets a graph generated on test failure details
 - Defaults to weekly
END
}

sub epoch_date {
	return UnixDate($_[0], "%s");
}

sub fatal_error {
	die "FATAL ERROR: @_\n";
}

sub internal_error {
	print STDERR "INTERNAL ERROR: @_\n";
	confess();
}

sub parse_date {
	local($_) = @_;
	my $date = ParseDate($_);
	if(defined $date and length $date) { return $date; }
	# Possibly confused by "GMT+2"?
	s/GMT\+(\d\d)$/+${1}00/;
	s/GMT\+(\d)$/+0${1}00/;
	$date = ParseDate($_);
	if(defined $date and length $date) { return $date; }
	$date = ParseDate(substr($_,0,25)); # Sun, 16 Nov 2003 23:23:34
	if(defined $date and length $date) { return $date; }
	$date = ParseDate(substr($_,0,20)); # 16 Nov 2003 23:23:34
	if(defined $date and length $date) { return $date; }
	fatal_error("Unable to parse date: $_\n");
}

sub bindate {
	my($binsize, $date) = @_;
	if($binsize eq $BIN_DAY) {
		return Date_GetPrev($date, undef, 1, 0, 0, 0); # To midnight
	}
	if($binsize eq $BIN_WEEK) {
		return Date_GetPrev($date, "Sun", 2, 0, 0, 0); # To midnight
	}
	if($binsize eq $BIN_MONTH) {
		$date = Date_GetPrev($date, undef, 1, 0, 0, 0); # To midnight
		my $tmp = UnixDate($date, "%P");
		$tmp =~ s/(......)..(........)/${1}01$2/;
		return parse_date($tmp);
	}
	if($binsize eq $BIN_YEAR) {
		$date = Date_GetPrev($date, undef, 1, 0, 0, 0); # To midnight
		my $tmp = UnixDate($date, "%P");
		$tmp =~ s/(....)....(........)/${1}0101$2/;
		return parse_date($tmp);
	}
	fatal_error("Unknown binning size of $binsize.");
}

sub describe_binsize {
	my($binsize) = @_;
	if($binsize eq $BIN_DAY) { return 'day'; }
	if($binsize eq $BIN_WEEK) { return 'week'; }
	if($binsize eq $BIN_MONTH) { return 'month'; }
	if($binsize eq $BIN_YEAR) { return 'year'; }
	fatal_error("Unknown binning size of $binsize.");
}

sub gnuplot_escape {
	local($_) = @_;
	s/\\/\\\\/g;
	s/"/\\"/g;
	return $_;
}

sub tempfile_or_die {
	my($ext) = @_;
	my($fh, $filename) = tempfile("condor-nmiplots-XXXXXX", DIR => File::Spec->tmpdir(), SUFFIX => '.data');
	if(not defined $filename or (length($filename) == 0)) {
		fatal_error("Unable to create a temporary filename!");
	}
	return($fh, $filename);
}

sub gnuplot_commands {
	my $myret = "";
	my(%args) = @_;
	my $title = $args{title};
	if(not defined $title) { internal_error("gnuplot_commands needs a 'title'"); }
	$title = gnuplot_escape($title);

	my $epsfile = $args{epsfile};
	if(not defined $epsfile) { internal_error("gnuplot_commands needs a 'epsfile'"); }
	$epsfile = gnuplot_escape($epsfile);

	my $startdate = $args{startdate};
	if(not defined $startdate) { internal_error("gnuplot_commands needs a 'startdate'"); }

	my $binsize = $args{binsize};
	if(not defined $binsize) { internal_error("gnuplot_commands needs a 'binsize'"); }
	$binsize = ucfirst($binsize);
	$binsize = gnuplot_escape($binsize);

	my $datafile = $args{datafile};
	if(not defined $datafile) { internal_error("gnuplot_commands needs a 'datafile'"); }
	$datafile = gnuplot_escape($datafile);

	# Pre Setup
	my $y2label = "";
	my $xtics = "";

	if($details eq "weeklydownloadlogs") {
		$y2label = "Download Results per Week";
		#$title = "Condor Download Plots";
		#$xtics = "691200";
	} elsif($details eq "downloadlogs") {
		$y2label = "Download Results per Day";
		#$title = "Condor Download Plots";
		$xtics = "691200";
	} elsif($details eq "uniquedownloadlogs") {
		$y2label = "Unique Downloads per Day";
		#$title = "Unique Condor Downloads";
		$xtics = "691200";
	} elsif($details eq "totaldownloadlogs") {
		$y2label = "Accumulated Downloads";
		#$title = "Total Condor Downloads";
		$xtics = "691200";
	} elsif($details eq "times") {
		#$title = "Condor Build & Test Times";
		$y2label = "Hours";
		$xtics = "345600";
	} else {
		$y2label = "Tests per $binsize";
		$xtics = "345600";
	}

	# Load the common setup

	$myret = $myret . "set terminal postscript color eps 24\n";
	$myret = $myret . "set title \"$title\"\n";
	$myret = $myret . "set output \"$epsfile\"\n";
	$myret = $myret . "set key outside top right\n";
	$myret = $myret . "set size ratio $RATIO $SCALE,$SCALE\n";
	
	#colors
	#$myret = $myret . "set key outside right top\n";
	$myret = $myret . "#sea green\n";
	$myret = $myret . "set style line 1  lt rgb \"#2e8b57\" \n";
	$myret = $myret . "#firebrick\n";
	$myret = $myret . "set style line 2  lt rgb \"#b22222\" \n";
	$myret = $myret . "#yellow1\n";
	$myret = $myret . "set style line 3  lt rgb \"#ffff00\" \n";
	$myret = $myret . "#orange - tests\n";
	$myret = $myret . "set style line 4  lt rgb \"#ffa500\" \n";
	$myret = $myret . "#blue3 - framework\n";
	$myret = $myret . "set style line 5  lt rgb \"#0000cd\" \n";
	$myret = $myret . "#gold1 - platform\n";
	$myret = $myret . "set style line 6  lt rgb \"#ffd700\" \n";
	$myret = $myret . "#cyan - condor\n";
	$myret = $myret . "set style line 7  lt rgb \"#00ffff\" \n";
	$myret = $myret . "#Hot Pink - condor\n";
	$myret = $myret . "set style line 8  lt rgb \"#ff69b4\" \n";
	$myret = $myret . "#honeydew 3 - condor\n";
	$myret = $myret . "set style line 9  lt rgb \"#C1CDC1\" \n";

	#$myret = $myret . "set boxwidth 0.75 relative\n";
	#$myret = $myret . "set style fill solid 1.00 border -1\n";
	$myret = $myret . "set style fill solid .75 border -1\n";
	$myret = $myret . "set style histogram rowstacked title offset character 0, 0, 0\n";
	$myret = $myret . "set datafile missing \'-\'\n";
	$myret = $myret . "set style data histograms\n";

	$myret = $myret . "set y2label \"$y2label\"\n";
	
	$myret = $myret . "# day in seconds 86400\n";
	$myret = $myret . "# week in seconds 604800\n";
	$myret = $myret . "# Quarters in seconds\n";
	$myret = $myret . "# set xtics 7884000\n";
	#$myret = $myret . "set xtics rotate 90\n";
	#$myret = $myret . "set xtics $xtics\n";
	$myret = $myret . "# monthly (1/3 of a quarter)\n";
	#$myret = $myret . "set mxtics 4\n";

	$myret = $myret . "set mytics 2\n";
	#$myret = $myret . "set xrange [\"$startdate\":]\n";
	#$myret = $myret . "set grid \n";

	if($details eq "weeklydownloadlogs") {
		$RATIO = 0.5;
		$SCALE= 2;
		$myret = $myret . "set size ratio $RATIO $SCALE,$SCALE\n";
		$myret = $myret . "set xtics rotate by -45\n";
		$myret = $myret . "set xdata time\n";
		#$myret = $myret . "set timefmt \"\%s\"\n";
		$myret = $myret . "set timefmt \"\%B \%d \%Y\"\n";
		$myret = $myret . "set format x \"\%b\\n%d\\n\%y\"\n";
		#$myret = $myret . "set grid xtics\n";
		$myret = $myret . "set xrange [\"$originaldate\":*]\n";
		#$myret = $myret . "set xtics $xtics\n";
		if( -f $morextics ){
		 	open(XTICS,"<$morextics") || die "tried to open<$morextics>:$!\n";
			while(<XTICS>){
				chomp();
				$morextics = $_;
				print "XTICS = $morextics\n";
				$myret = $myret . "set xtics $morextics\n";
				$myret = $myret . "set grid \n";
				$myret = $myret . "show xrange\n";
				#$myret = $myret . "show xtics\n";
			}
		}
	} else {
		$myret = $myret . "set xdata time\n";
		$myret = $myret . "set timefmt \"\%s\"\n";
		$myret = $myret . "set format x \"\%b\\n%d\\n\%y\"\n";
		$myret = $myret . "set xrange [\"$startdate\":]\n";
		$myret = $myret . "set xtics $xtics\n";
	}

	if($details eq "analdetail") {
		$myret = $myret . "plot \"$datafile\" using 1:9 with boxes   ls 5 title \"Framework\", \"$datafile\" using 1:7 with boxes   ls 6 title \"Platform\", \"$datafile\" using 1:5 with boxes   ls 7 title \"Condor\", \"$datafile\" using 1:3 with boxes ls 4 title \"Tests\"\n";
	} elsif($details eq "autobuilds") {
		$myret = $myret . "plot \"$datafile\" using 1:6 with boxes   ls 1 title \"Good Builds\", \"$datafile\" using 1:4 with boxes   ls 2 title \"Failed Builds\", \"$datafile\" using 1:2 with boxes   ls 3 title \"Pending\"\n";
	} elsif($details eq "autotests") {
		$myret = $myret . "plot \"$datafile\" using 1:6 with boxes   ls 1 title \"Good Tests\", \"$datafile\" using 1:4 with boxes   ls 2 title \"Failed Tests\", \"$datafile\" using 1:2 with boxes   ls 3 title \"Pending\"\n";
	} elsif($details eq "autotestyear") {
		$myret = $myret . "plot \"$datafile\" using 1:6 with boxes   ls 1 title \"Good Tests\", \"$datafile\" using 1:4 with boxes   ls 2 title \"Failed Tests\", \"$datafile\" using 1:2 with boxes   ls 3 title \"Pending\"\n";
	} elsif($details eq "times") {
		$myret = $myret . "plot \"$datafile\" using 1:5 with boxes   ls 6 title \"Test Times\", \"$datafile\" using 1:3 with boxes   ls 5 title \"Build Times\", \"$datafile\" using 1:8 with boxes   ls 7 title \"Test Times/10\", \"$datafile\" using 1:6 with boxes   ls 4 title \"Build Times/10\"\n";
	} elsif($details eq "testsperplatform") {
		$myret = $myret . "plot \"$datafile\" using 1:2 with boxes   ls 1 title \"Tests per Platform\"\n";
	} elsif($details eq "downloadlogs") {
		$myret = $myret . "plot \"$datafile\" using 1:3 with boxes   ls 5 title \"Binary\", \"$datafile\" using 1:2 with boxes   ls 9 title \"Source\"\n";
	} elsif($details eq "weeklydownloadlogs") {
		$myret = $myret . "plot \"$datafile\" using 1:5 with boxes   ls 5 title \"Binary\", \"$datafile\" using 1:4 with boxes   ls 9 title \"Source\"\n";
	} elsif($details eq "uniquedownloadlogs") {
		$myret = $myret . "plot \"$datafile\" using 1:3 with boxes   ls 5 title \"Unique\"\n";
	} elsif($details eq "totaldownloadlogs") {
		$myret = $myret . "plot \"$datafile\" using 1:3 with boxes   ls 5 title \"Binary\",  \"$datafile\" using 1:2 with boxes   ls 9 title \"Source\"\n";
	} else {
		$myret = $myret . "plot \"$datafile\" using 1:2 with boxes ls 1 title \"Passed Tests\", \"$datafile\" using 1:4 with boxes ls 2 title \"Failed Tests\", \"$datafile\" using 1:10 with boxes ls 5 title \"Framework Lost Tests\", \"$datafile\" using 1:8 with boxes ls 7 title \"Condor Issues\", \"$datafile\" using 1:6 with boxes ls 6 title \"Platform Issues\"\n";
	}
	return($myret);
}
