#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          to be called from test21.30
#          Tests if the MaxJobs limit is enforced
############################################################################
# Copyright (C) 2012 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################

# Test MaxJobs limit
proc inc21_30_7 { wait_reason } {

	global salloc scontrol srun acct re_word_str bin_sleep jobmatch job_id1 job_id2 job_id3 number exit_code maxjobs_num

	log_info "Starting MaxJob limit test (Within: inc21.30.7)"
	set check_num $maxjobs_num

	set jobmatch 0
	for {set inx 0} {$inx <= $check_num} {incr inx} {
		set test_reason 0
		spawn $salloc --account=$acct -t1 -n1 $srun $bin_sleep 60
		expect {
			-re "Granted job allocation ($number)" {
				set job_id($inx) $expect_out(1,string)
				if {$inx < $check_num} {
					log_debug "Job $inx $job_id($inx) was submitted. (inc21.30.7)"
					incr jobmatch
				} else {
					log_error "Job $inx $job_id($inx) wasn't suppose to work but it did, limit was $check_num? (Within: inc21.30.7)"
					set exit_code 1
				}
			}
			-re "job ($number) queued and waiting for resources" {
				set job_id($inx) $expect_out(1,string)
				if {$inx >= $check_num} {
					log_debug "Job $inx $job_id($inx) is waiting for resources. This is expected (Within: inc21.30.7)"
					incr jobmatch
				} else {
					set test_reason 1
				}
			}
			-re "error" {
				log_error "Job allocation should not have failed. (Within: inc21.30.7)"
				set exit_code 1
			}
			timeout {
				log_error "salloc is not responding. (Within: inc21.30.7)"
				set exit_code 1
			}
			eof {
				wait
			}
		}

#		Job is waiting, check if the reason is the limit we are testing for
		if {$test_reason != 0 && $job_id($inx) != 0} {
			spawn $scontrol show job $job_id($inx)
			expect {
				-re "Reason=($re_word_str)" {
					if {[string compare $expect_out(1,string) $wait_reason]} {
						log_error "Job $inx $job_id($inx) was suppose to start but it didn't, limit was $check_num? (Within: inc21.30.7)"
						set exit_code 1
					} else {
						incr jobmatch
					}
					exp_continue
				}
				timeout {
					log_error "scontrol is not responding. (Within: inc21.30.7)"
					set exit_code 1
				}
				eof {
					wait
				}
			}
		}
	}

	set target_jobs [expr $check_num + 1]
	if {$jobmatch != $target_jobs} {
		log_error "Not all jobs were not submitted ($jobmatch != $target_jobs). (Within: inc21.30.7)"
		set exit_code 1
	}

	# Checks the job state
	if [info exists job_id($check_num)] {
		if {![check_job_state $job_id($check_num) PENDING]} {
			log_error "Job was not on the desired state (Within: inc21.30.7)"
			set exit_code 1
		}
	} else {
		log_error "Didn't attempt to start enough jobs. (Within: inc21.30.7)"
		set exit_code 1
	}

	for {set inx 0} {$inx <= $check_num} {incr inx} {
		if [info exists job_id($inx)] {
			# Cancels the jobs
			cancel_job $job_id($inx)
		}
	}
}
