#!/usr/bin/env expect
############################################################################
# Purpose: Test normalized factors
############################################################################
# Copyright (C) 2019 SchedMD LLC.
# Written by Brian Christiansen <brian@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting

set timeout	60

set ta1         "test$test_id-account.1"
set ta2         "test$test_id-account.2"
set tq1         "test$test_id-qos.1"
set tq2         "test$test_id-qos.2"

set p1         "test$test_id-part.1"
set p2         "test$test_id-part.2"

set config_path  ""

set jobid_list ""

set cwd "[$bin_pwd]"

set access_err 0

set user [get_my_user_name]

#
# Check accounting config and bail if not found.
#
if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
	skip "This test can't be run without a usable AccountStorageType"
}
if {[get_config_param "PriorityType"] ne "priority/multifactor"} {
	skip "This test can't be run without a usable PriorityType"
}

#
# This test needs to be modified to use the core counts rather than CPU counts
#
set select_type_params [get_select_type_params ""]
if { [string first "CR_ONE_TASK_PER_CORE" $select_type_params] != -1 } {
	skip "This test can't be run with SelectTypeParameters=CR_ONE_TASK_PER_CORE"
}

proc clean_assocs {} {
	global ta1 ta2 tq1 tq2

	remove_acct "" $ta1
	remove_acct "" $ta2
	remove_qos $tq1
	remove_qos $tq2
}

proc cleanup { } {
	global config_file jobid_list

	cancel_job $jobid_list
	clean_assocs

	restore_conf $config_file
	reconfigure
}


proc part_cpu_cnt { partition } {
	global sinfo number

	set cpu_cnt 1
	spawn $sinfo -h -o "%C" -p $partition --state=idle
	expect {
		-re "($number)(K?)\/($number)(K?)" {
			set cpu_cnt $expect_out(3,string)
			if {[string compare $expect_out(4,string) ""]} {
				set cpu_cnt [expr $cpu_cnt * 1024]
			}

			exp_continue
		}
		timeout {
			fail "sinfo not responding"
		}
		eof {
			wait
		}
	}
	return $cpu_cnt
}

proc sub_job { args state } {

	global number sbatch test_id bin_sleep jobid_list

	set jobid 0
	set cmd "$sbatch -J$test_id -o/dev/null -e/dev/null --exclusive --wrap \"$bin_sleep 999\""
	append cmd " $args"

	spawn {*}$cmd
	expect {
		-re "Submitted batch job ($number)" {
			set jobid $expect_out(1,string)
			exp_continue
		}
		timeout {
			fail "sbatch not responding"
		}
		eof {
			wait
		}
	}

	if {$jobid == 0} {
		fail "Did not get sbatch jobid"
	}

	if {[wait_for_job $jobid $state] != 0} {
		fail "Error starting job $jobid"
	}

	lappend jobid_list $jobid
	return $jobid
}

proc sprio_args { args regex } {
	global sprio

	wait_for -timeout 5 -poll .1 {
		[regexp $regex [run_command_output -fail "$sprio $args"]] == 1
	} { }
}

proc scontrol { args regex match_cnt } {
	global scontrol
	set matches 0

	spawn $scontrol {*}$args
	expect {
		-re $regex {
			incr matches
			exp_continue
		}
		timeout {
			fail "scontrol not responding"
		}
		eof {
			wait
		}
	}

	if {$matches != $match_cnt} {
		fail "scontrol $args failed to match '$regex' $match_cnt times"
	}
}

proc update_job { jobid prio {error ""} } {
	global scontrol

	set matches 0

	spawn $scontrol update jobid=$jobid priority=$prio
	set tmp_re ".*"
	if {$error != ""} {
		set tmp_re $error
	}
	expect {
		-re $tmp_re {
			incr matches
		}
		timeout {
			fail "scontrol not responding"
		}
		eof {
			wait
		}
	}

	if {($error != "") && !$matches} {
		fail "Didn't get expected error '$error'"
	}
}

proc create_part { name nodes prio } {
	global scontrol

	spawn $scontrol create partitionname=$name priorityjobfactor=$prio nodes=$nodes
	expect {
		-re "error" {
			fail "scontrol found error"
		}
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}

	set found 0
	spawn $scontrol show partitionname=$name
	expect {
		-re "PartitionName=$name" {
			set found 1
			exp_continue
		}
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}

	if {$found == 0} {
		fail "scontrol did not create partition $name"
	}
}

proc update_part { name args } {
	global scontrol

	set cmd "$scontrol update partitionname=$name"
	append cmd " $args"

	spawn {*}$cmd
	expect {
		-re "error" {
			fail "scontrol found error"
		}
		timeout {
			fail "scontrol is not responding"
		}
		eof {
			wait
		}
	}
}

################################################################

if {[info exists env(SPRIO_FORMAT)]} {
    unset env(SPRIO_FORMAT)
}

set config_path [get_conf_path]
set config_file $config_path/slurm.conf

save_conf $config_file

set weight 4000000
set weight_half [expr $weight / 2]

# max is uint16_t
set part_weight 60000
set part_weight_half [expr $part_weight / 2]

exec $bin_sed -i "s/^\\(PriorityWeight.*\\)/#\\1/Ig" $config_file
exec $bin_sed -i "s/^\\(PriorityFlags.*\\)/#\\1/Ig" $config_file
set prio_conf "
PriorityWeightAssoc     = $weight
PriorityWeightJobSize   = 1
PriorityWeightPartition = $part_weight
PriorityWeightQOS       = $weight
PriorityWeightTRES=cpu=$weight
"
exec $bin_echo $prio_conf >> $config_file
reconfigure -fail


set idle_node [lindex [get_nodes_by_state] 0]
if {$idle_node == ""} {
	fail "No idle nodes"
}
log_info "idle: $idle_node"
create_part $p1 $idle_node $part_weight
create_part $p2 $idle_node $part_weight_half

set cpu_cnt [part_cpu_cnt $p1]
if {[expr fmod($cpu_cnt, 2)] != 0} {
	skip "This test requires an even number of cpus per node ($idle_node has $cpu_cnt cpus)"
}
set cpu_cnt_half [expr $cpu_cnt / 2]

clean_assocs

array set qos_req {}
set qos_req(priority) $weight
if {[add_qos $tq1 [array get qos_req]]} {
	fail "Unable to create qos $tq1"
}
set qos_req(priority) $weight_half
if {[add_qos $tq2 [array get qos_req]]} {
	fail "Unable to create qos $tq2"
}

array set acct_req {}
set acct_req(qos) "$tq1,$tq2"
set acct_req(priority) $weight
if {[add_acct $ta1 [array get acct_req]]} {
	fail "Unable to create account $ta1"
}
set acct_req(priority) $weight_half
if {[add_acct $ta2 [array get acct_req]]} {
	fail "Unable to create account $ta2"
}

array set user_req {}
set user_req(account) "$ta1,$ta2"
if {[add_user $user [array get user_req]]} {
	fail "Unable to add user $user"
}

log_info "test1"
scontrol "show assoc account=$ta1 flags=assoc" "(Account=$ta1 UserName= Partition= Priority=$weight)|(Account=$ta1 UserName=$user\\S+ Partition= Priority=$weight)" 2
scontrol "show assoc account=$ta2 flags=assoc" "(Account=$ta2 UserName= Partition= Priority=$weight_half)|(Account=$ta2 UserName=$user\\S+ Partition= Priority=$weight_half)" 2
scontrol "show assoc qos=$tq1 flags=qos" "Priority=$weight" 1
scontrol "show assoc qos=$tq2 flags=qos" "Priority=$weight_half" 1

set jobid1 [sub_job "-n$cpu_cnt -p$p1 -A$ta1 --qos=$tq1" RUNNING]
set jobid2 [sub_job "-n$cpu_cnt -p$p1 -A$ta1 --qos=$tq1" PENDING]
set jobid3 [sub_job "-n$cpu_cnt_half -p$p2 -A$ta2 --qos=$tq2" PENDING]

sprio_args "-j $jobid2 -o \"%.15i %10B %10P %10Q %12T\"" "$jobid2\\s+$weight\\s+$part_weight\\s+$weight\\s+cpu=$weight"
sprio_args "-n -j $jobid2 -o \"%.15i %10b %10p %10q %12t\"" "$jobid2\\s+1\\.0+\\s+1\\.0+\\s+1\\.0+\\s+cpu=1\\.0+"

sprio_args "-j $jobid3 -o \"%.15i %10B %10P %10Q %12T\"" "$jobid3\\s+$weight_half\\s+$part_weight_half\\s+$weight_half\\s+cpu=$weight_half"
sprio_args "-n -j $jobid3 -o \"%.15i %10b %10p %10q %12t\"" "$jobid3\\s+0\\.50+\\s+0\\.50+\\s+0\\.50+\\s+cpu=0\\.5+"

log_info "test2"
# update priorities to high water mark, job should get full weights because the
# new high wark will be adjusted.
array set acct_mod_desc {}
array set acct_mod_acct_vals {}
array set acct_mod_assoc_vals {}
set acct_mod_assoc_vals(priority) $weight_half
if {[mod_acct $ta1 [array get acct_mod_desc] [array get acct_mod_assoc_vals] [array get acct_mod_acct_vals]] } {
	fail "Unable to modify account $ta1"
}
if {[mod_qos $tq1 [array get acct_mod_assoc_vals]] } {
	fail "Unable to modify qos $tq1"
}

scontrol "show assoc account=$ta1 flags=assoc" "(Account=$ta1 UserName= Partition= Priority=$weight_half)|(Account=$ta1 UserName=$user\\S+ Partition= Priority=$weight_half)" 2
scontrol "show assoc qos=$tq1 flags=qos" "Priority=$weight_half" 1

update_part $p1 "priorityjobfactor=$part_weight_half"

set jobid [sub_job "-n$cpu_cnt -p$p2 -A$ta2 --qos=$tq2" PENDING]
sprio_args "-j $jobid -o \"%.15i %10B %10P %10Q %12T\"" "$jobid\\s+$weight\\s+$part_weight\\s+$weight\\s+cpu=$weight"
sprio_args "-n -j $jobid -o \"%.15i %10b %10p %10q %12t\"" "$jobid\\s+1\\.0+\\s+1\\.0+\\s+1\\.0+\\s+cpu=1\\.0+"


log_info "test3"
# move back to higher wait to test max priorities and normalized values being reconfigured.
array set acct_mod_desc {}
array set acct_mod_acct_vals {}
array set acct_mod_assoc_vals {}
set acct_mod_assoc_vals(priority) $weight
if {[mod_acct $ta1 [array get acct_mod_desc] [array get acct_mod_assoc_vals] [array get acct_mod_acct_vals]] } {
	fail "Unable to modify account $ta1"
}
if {[mod_qos $tq1 [array get acct_mod_assoc_vals]] } {
	fail "Unable to modify qos $tq1"
}

scontrol "show assoc account=$ta1 flags=assoc" "(Account=$ta1 UserName= Partition= Priority=$weight)|(Account=$ta1 UserName=$user\\S+ Partition= Priority=$weight)" 2
scontrol "show assoc qos=$tq1 flags=qos" "Priority=$weight" 1

update_part $p1 "priorityjobfactor=$part_weight"

set jobid [sub_job "-n$cpu_cnt_half -p$p2 -A$ta2 --qos=$tq2" PENDING]
sprio_args "-j $jobid -o \"%.15i %10B %10P %10Q %12T\"" "$jobid\\s+$weight_half\\s+$part_weight_half\\s+$weight_half\\s+cpu=$weight_half"
sprio_args "-n -j $jobid -o \"%.15i %10b %10p %10q %12t\"" "$jobid\\s+0\\.50+\\s+0\\.50+\\s+0\\.50+\\s+cpu=0\\.5+"

log_info "test4"
# test setting priority directly on a job
set jobid [sub_job "-n$cpu_cnt_half -p$p2 -A$ta2 --qos=$tq2" PENDING]
update_job $jobid 999

sprio_args "-j $jobid -o \"%.15i %.10Y %.10S %.10A %.10B %.10F %.10J %.10P %.10Q %.11N %.20T\"" "$jobid\\s+999\\s+0\\s+0\\s+0\\s+0\\s+0\\s+0\\s+0\\s+0"
sprio_args "-n -j $jobid -o \"%.15i %.10Y %.10S %.10A %.10B %.10F %.10J %.10P %.10Q %.11N %.20T\"" "$jobid\\s+999\\s+0\\s+0\\s+0\\s+0\\s+0\\s+0\\s+0\\s+0"
