#! /usr/bin/env perl
##**************************************************************
##
## Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
## University of Wisconsin-Madison, WI.
## 
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at
## 
##    http://www.apache.org/licenses/LICENSE-2.0
## 
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
##**************************************************************
##
## This test will submit 10000 sleep jobs to get a longer negotiation
## cycle and when they are all submitted will do four things:
##  1. Find the next start of a negotiation cycle
##  2. run a condor_userprio
##	3. find the next end of a negotation cycle
##  4. determine if condor_userprio happened during negotiation cycle
##
##**************************************************************

use CondorTest;
use CondorUtils;
use Check::SimpleJob;
use Check::CondorLog;


my $ClusterId = 0;
my $expect_return = 0;

	#NEGOTIATE_ALL_JOBS_IN_CLUSTER = True

my $testname = "job_services_during_neg_cycle";

#testreq: personal
my $config = <<CONDOR_TESTREQ_CONFIG;
	DAEMON_LIST = MASTER,SCHEDD,COLLECTOR,NEGOTIATOR,STARTD
	MAX_NEGOTIATOR_LOG = 150 Mb
	MAX_SCHEDD_LOG = 150 Mb
	SCHEDD_DEBUG = D_COMMAD:1
	SIGNIFICANT_ATTRIBUTES = Procid
	START = False
	NEGOTIATOR_DEBUG = D_COMMAND, D_MATCH
	NUM_CPUS = 12
	DEBUG = D_FULLDEBUG
CONDOR_TESTREQ_CONFIG
#endtestreq


TLOG "$testname test started\n";

my $count = 0;
my $targetjobcount = 10000;

my $on_execute = sub {
};

my $on_abort = sub {
};

my $GetClusterId = sub {
	my $cid = shift;
	$ClusterId = $cid;
	TLOG "Requested Cluster Got $ClusterId\n";
	##########################
	#
	# Convoluted: This function is paased to RunCheck which uses it in RunTest in the fourth
	# position which is always an undefined variable EXCEPT when it is a call back routine 
	# to pass out the cluster Id.
	#
	##########################
	#runcmd("condor_q",{emit_output => 1});
};

my $result = 1;
my $test_name = "job_services_during_neg_cycle";

my $matchedbegin = "";
my $matchednextend = "";
my $matchedendafterbegin = "";
my $matchbeginafterend = "";
my $matchedfinalfinish = "";

my $matchonbegin = 0;
my $matchonnextend = 0;
my $matchedendafterbegin = 0;
my $matchedbeginafterend = 0;
my $matchonfinalfinish = 0;
my $match = "";

my $failure = "HitRetryLimit";

my $on_match = sub {
	$match = shift;
	TLOG "Matched: $match\n\n";
	if($matchonbegin == 1) {
		$matchedbegin = $match;
	} elsif($matchonnextend == 1) {
		$matchednextend = $match;
	} elsif($matchonendafterbegin == 1) {
		$matchedendafterbegin = $match;
	} elsif($matchedbeginafterend == 1) {
		$matchbeginafterend = $match;
	} elsif ($matchonfinalfinish == 1) {
		$matchedfinalfinish = $match;
	}
};

my $on_submit = sub {
	$count += 1;
	#TLOG "Submitted count <$count>\n";
	my $leftover = ($count % 1000);
	if($leftover == 0) {
		TLOG "$count submitted\n";
	}
	if($count == $targetjobcount) {
	TLOG "************************************ All Jobs Submitted ***********************************\n";
		#runToolNTimes("condor_q -better $ClusterId.10000",1,1);
		# they will never match because start = false
		# wait for the start of the next negotiation cycle
		$matchonbegin = 1;
		CondorLog::RunCheckMultiple(
			daemon => "Negotiator",
			match_new => "true",
			match_timeout => 600,
			match_regexp => "Started Negotiation Cycle",
			match_callback => $on_match,
		);
		$matchonbegin = 0;
		TLOG "Looking for new - Started Negotiation Cycle - ";
		if($matchedbegin eq $failure) {
			print "failure\n";
		} else {
			print "OK\n";
		}

		# do my condor_userprios
		#my @prioutput = `condor_userprio`;
		#my @condorreconfig = `condor_reconfig`;
		runToolNTimes("condor_userprio",1,0);
		runToolNTimes("condor_reconfig",1,0);

		runToolNTimes("condor_reconfig",1,0);
		runToolNTimes("condor_userprio",5,0);

		# find the one  finish after our begin
		TLOG "RunCheckMultiple find Finished Negotiation Cycle after $matchedbegin\n";
		$matchonendafterbegin = 1;
		CondorLog::RunCheckMultiple(
			daemon => "Negotiator",
			match_new => "false",
			match_timeout => 600,
			match_regexp => quotemeta($matchedbegin), # prefix with \Q to avoid interpreting literal chars as regex chars
			match_after_regexp => "Finished Negotiation Cycle",
			match_callback => $on_match,
		);
		$matchonendafterbegin = 0;
		TLOG "Looking for the Finish which follows my Begin - Finished Negotiation Cycle - ";
		if($matchedendafterbegin eq $failure) {
			print "$failure\n";
		} else {
			print "OK\n";
		}

		# Were all the condor_userprios done within the cycle?
		CondorLog::RunCheckMultiple(
			daemon => "Negotiator",
			match_new => "false",
			match_timeout => 600,
			match_instances => 1,
			match_regexp => quotemeta($matchedbegin),
			match_between_regexp => "Return from HandleReq <GET_PRIORITY_commandHandler>",
			match_after_regexp => quotemeta($matchedendafterbegin),
			match_callback => $on_match,
		);
		TLOG "Looking for how many Return from HandleReq: GET_PRIORITY... come between: $matchedbegin and $matchedendafterbegin - ";
		if($match eq $failure) {
			print "$failure\n";
		} else {
			print "OK got $match\n";
		}

		CheckExpectations("Condor_userprio handlings during cycle",$match,6);

		# Did the reconfig wait for the end of the cycle
		CondorLog::RunCheckMultiple(
			daemon => "Negotiator",
			match_new => "false",
			match_timeout => 600,
			match_instances => 1,
			match_regexp => quotemeta($matchedbegin),
			match_between_regexp => "Got SIGHUP.  Re-reading config files.",
			match_after_regexp => quotemeta($matchedendafterbegin),
			match_callback => $on_match,
		);

		TLOG "Looking for how many ...Got SIGHUP.  Re-reading config files.... between '$matchedbegin' and '$matchedendafterbegin' - ";
		if($match eq $failure) {
			print "$failure\n";
		} else {
			print "OK got $match\n";
		}

		CheckExpectations("Condor_reconfig handlings during cycle",$match,0);


		runToolNTimes("condor_rm -all",1,0);
		runToolNTimes("condor_rm -forcex -all",1,0);
		CondorTest::EndTest();
		exit(0);
	}
};

# submit a bunch of jobs and when the submitted count reaches target count
# we will use RunCheckMultiple to find and return by a callback with the found line.

# Then we have to figure out what happened with a series of events done within
# that last on_submit event.

$result = SimpleJob::RunCheck(
	test_name => "$test_name",
	timeout => 1800,
	on_abort => $on_abort,
	on_submit => $on_submit,
	on_execute => $on_execute,
	queue_sz => $targetjobcount,
	GetClusterId => $GetClusterId,
	duration => 3600,
);

TLOG "******************** Test to do more within a negotiation cycle is $result\n";

CondorTest::EndTest();


sub CheckExpectations
{
	my $ad = shift;
	my $got = shift;
	my $expected = shift;

	CondorTest::debug("CheckExpectations: $ad Got,$got Expected,$expected\n",2);
	if($got != $expected) {
		if($ad =~ /Condor_userprio handlings/) {
			if(($got >= 1) && ($got <= $expected)) {
				CondorTest::RegisterResult(1, "test_name", $testname);
			}
		} else {
			CondorTest::RegisterResult(0, "test_name", $testname);
			die "$ad: Expected <$expected> but only got <$got>\n";
		}
	} else {
		CondorTest::RegisterResult(1, "test_name", $testname);
	}
}

# sub RunNTimes
# {
# 	my $cmd = shift;
# 	my $goal = shift;
# 	my $wantoutput = shift;

# 	my $count = 0;
# 	my $stop = 0;
# 	my @cmdout = ();
# 	my $date;
# 	$stop = $goal;

# 	while($count < $stop) {
# 		@cmdout = ();
#		$date = scalar(localtime());
#		print "$date $cmd $count\n";
# 		@cmdout = `$cmd`;

# 		if(defined $wantoutput) {
# 			foreach my $line (@cmdout) {
# 				print "$line\n";
# 			}
# 		}
# 		$count += 1;
# 	}
# }
