#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Test sacctmgr update job [condition] set newwckey="newkey"
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2019 SchedMD LLC
# Written by Albert Gil <albert.gil@schedmd.com>
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals
source ./globals_accounting

set test_id       "21.41"
set test_name     "test21_41"
set exit_code     0

# globals
set access_err    0
set timeout 120

print_header $test_id

set sql_in_clus1  "$test_name-clus1-in.sql"
set sql_rem_clus1 "$test_name-clus1-rem.sql"
set sql_in_clus2  "$test_name-clus2-in.sql"
set sql_rem_clus2 "$test_name-clus2-rem.sql"

set cluster1      [format "%s%s" $test_name "clus1"]
set cluster2      [format "%s%s" $test_name "clus2"]
set account1      [format "%s%s" $test_name "acct1"]
set account2      [format "%s%s" $test_name "acct2"]
set user1         [get_my_user_name]
set user2         "root"
set wckey1        [format "%s%s" $test_name "wckey1"]
set wckey2        [format "%s%s" $test_name "wckey2"]
set wckey_new     [format "%s%s" $test_name "wckeynew"]

set clusters      [format "%s,%s" $cluster1 $cluster2]
set accounts      [format "%s,%s" $account1 $account2]
set users         [format "%s,%s" $user1    $user2]
set wckeys        [format "%s,%s" $wckey1   $wckey2]

# 8 jobs to test combinations of 2 accounts, 2 users and 2 wckeys
# same job ids will be used for the 2 clusters
set jobid01       "1001"
set jobid02       "1002"
set jobid03       "1003"
set jobid04       "1004"
set jobid05       "1005"
set jobid06       "1006"
set jobid07       "1007"
set jobid08       "1008"

###############################################################################
#
# Proc: Loads the remove sql entries and remove the added users, accounts and
#       clusters.
#
###############################################################################
proc clean_it {} {
	global exit_code bin_rm
	global sql_in_clus1 sql_in_clus2
	global sql_rem_clus1 sql_rem_clus2
	global clusters accounts users

	incr exit_code [archive_load $sql_rem_clus1]
	incr exit_code [archive_load $sql_rem_clus2]
	remove_user    $clusters $accounts $users
	remove_acct    "" $accounts
	remove_cluster $clusters

	exec $bin_rm $sql_in_clus1 $sql_in_clus2 $sql_rem_clus1 $sql_rem_clus2
}

###############################################################################
#
# Proc: Calls cleant_it, print test result and exits the test.
#
###############################################################################
proc end_it {} {
	global exit_code test_id

	if {$exit_code == 0} {
		clean_it
		print_success $test_id
	} else {
		print_failure $test_id
	}
	exit $exit_code
}

###############################################################################
#
# Proc: Creates the sql files to load 8 test jobs, and to remove them.
#       Each job is a unique combination of 2 accounts, 2 users and 2 wckeys
#
# Input: cluster: cluster name to use
#        sql_in : filename to save the creation file
#        sql_rem: filename to save the remove file
#
# TODO: Error handling.
###############################################################################
proc create_sql {cluster sql_in sql_rem} {

	global bin_rm
	global end_it
	global exit_code test_name
# 	global cluster1 cluster2
	global account1 account2
	global wckey1 wckey2
	global user1acct1 user1acct2
	global user2acct1 user2acct2
	global user1wckey1 user1wckey2
	global user2wckey1 user2wckey2
	global jobid01 jobid02 jobid03 jobid04 jobid05 jobid06 jobid07 jobid08

	set node_list     [format "%s%s" $cluster "\[0-1\]"]
	set job_name      "$test_name\_job"
	set cluster_cpus  4

	set uid [get_my_uid]
	set gid [get_my_gid]

	# Mon Dec 31 23:00:00 2007
	set start_date "Mon Dec 31 23:00:00 2007"
	set period_start [exec date -d $start_date +%s]

	# Thu Jan 31 23:59:59 2008
	set end_date "Thu Jan 31 23:59:59 2008"
	set period_end [exec date -d $end_date +%s]

	# DON'T MESS WITH THIS UNLESS YOU REALLY UNDERSTAND WHAT YOU ARE DOING!!!!!
	# THIS COULD SERIOUSLY MESS UP YOUR DATABASE IF YOU ALTER THIS INCORRECTLY
	# JUST A FRIENDLY REMINDER ;)

	# create the in sql file
	exec $bin_rm -f $sql_in
	set file [open $sql_in "w"]

	# put in the cluster for back in the day before accounting was made here for us we are using 'Tue Jan  1 00:00:00 2008' = 1199174400 as the start
	puts $file "insert into cluster_event_table (node_name, cluster, tres, period_start, period_end, reason, cluster_nodes) values"
	puts $file "('', '$cluster', '1=$cluster_cpus', $period_start, $period_end, 'Cluster processor count', '$node_list' )"
	puts $file "on duplicate key update period_start=VALUES(period_start), period_end=VALUES(period_end);"

	# add jobs
	puts $file "insert into job_table (jobid, associd, wckey, wckeyid, uid, gid, `partition`, blockid, cluster, account, eligible, submit, start, end, suspended, name, track_steps, state, comp_code, priority, req_cpus, tres_alloc, nodelist, kill_requid, qos, deleted) values"
	puts $file "('$jobid01', '$user1acct1', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid02', '$user1acct1', '$wckey2', '$user1wckey2', '$uid', '$gid', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid03', '$user2acct1', '$wckey1', '$user2wckey1',    '0',    '0', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid04', '$user2acct1', '$wckey2', '$user2wckey2',    '0',    '0', 'debug', '', '$cluster', '$account1', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"

	puts $file "('$jobid05', '$user1acct2', '$wckey1', '$user1wckey1', '$uid', '$gid', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid06', '$user1acct2', '$wckey2', '$user1wckey2', '$uid', '$gid', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid07', '$user2acct2', '$wckey1', '$user2wckey1',    '0',    '0', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "('$jobid08', '$user2acct2', '$wckey2', '$user2wckey2',    '0',    '0', 'debug', '', '$cluster', '$account2', 1199138400, 1199138400, 1199138400, 1199139600, '0', '$job_name', '0', '3', '0', '2', 2, '1=2', '$node_list', '0', '0', '0')"
	puts $file "on duplicate key update id=LAST_INSERT_ID(id), eligible=VALUES(eligible), submit=VALUES(submit), start=VALUES(start), end=VALUES(end), associd=VALUES(associd), tres_alloc=VALUES(tres_alloc), wckey=VALUES(wckey), wckeyid=VALUES(wckeyid);";
	close $file

	# create the remove sql file
	exec $bin_rm -f $sql_rem
	set file [open $sql_rem "w"]

	puts $file [format "%s%s%s" "truncate table \"" $cluster "_event_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_job_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_step_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_usage_month_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_assoc_usage_month_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_day_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_hour_table\";"]
	puts $file [format "%s%s%s" "truncate table \"" $cluster "_wckey_usage_month_table\";"]
	close $file
}


###############################################################################
#
# Proc: Runs the desired "sacctmgr update job" command and checkd the expected
#       output.
#
# Returns: 0 if result was the expected, non-zero otherwise.
#
# Input: sacctmgr_args: the "where" args to the update job command.
#        expected_jobs: the list of job ids expected to be updated.
#        sql_rem: filename to save the remove file
#
###############################################################################
proc test_job_update {sacctmgr_args expected_jobs} {
	global sacct sacctmgr number
	global clusters wckey_new

	set matches_right 0
	set matches_wrong 0
	set rc 0

	set cmd_str "$sacctmgr -i update job set newwckey=$wckey_new where $sacctmgr_args"
	set my_pid [eval spawn [lindex $cmd_str 0] {*}[lrange $cmd_str 1 end]]
	expect {
		-re "($number) submitted at" {
			set jobid $expect_out(1,string)
			if {"$jobid" in $expected_jobs} {
				incr matches_right
			} else {
				incr matches_wrong
			}
			exp_continue
		}
		timeout {
			log_error "sacctmgr update job not responding"
			slow_kill $my_pid
			incr rc
			return $rc
		}
		eof {
			wait
		}
	}
	if {$matches_right != [llength $expected_jobs] || $matches_wrong != 0} {
		log_error "Expected job updates wrong $matches_right $matches_wrong"
		incr rc
		return $rc
	}

	set matches_right 0
	set matches_wrong 0
	set my_pid [eval spawn $sacct -a -n --cluster=$clusters --wckey=$wckey_new --start=2000-01-01 --end=now --format=jobid]
	expect {
		-re "($number)" {
			set jobid $expect_out(1,string)
			if {"$jobid" in $expected_jobs} {
				incr matches_right
			} else {
				incr matches_wrong
			}
			exp_continue
		}
		timeout {
			log_error "sacct not responding"
			slow_kill $my_pid
			incr rc
			return $rc
		}
		eof {
			wait
		}
	}
	if {$matches_right != [llength $expected_jobs] || $matches_wrong != 0} {
		log_error "Expected job updates wrong $matches_right $matches_wrong"
		incr rc
		return $rc
	}
	return $rc
}

proc _run_test { update_str expected_jobs } {
	global sql_rem_clus1 sql_rem_clus2 sql_in_clus1 sql_in_clus2
	global exit_code

	incr exit_code [archive_load $sql_rem_clus1]
	incr exit_code [archive_load $sql_rem_clus2]
	incr exit_code [archive_load $sql_in_clus1]
	incr exit_code [archive_load $sql_in_clus2]

	if { $exit_code } {
		end_it
	}

	incr exit_code [test_job_update $update_str $expected_jobs]

	if { $exit_code } {
		log_error "Update failed with: $update_str"
		end_it
	}
}

#
# Check configuration
# TODO: chack also slurm.conf!!
#
if { [test_account_storage] == 0 } {
	log_warn "This test can't be run without a usable AccountStorageType."
	exit 0
}

set wc_key_track [check_wckey_tracking_slurmdbd]
if { $wc_key_track == 0 } {
	log_warn "This test can't be run without TrackWCKey set in slurmdbd.conf."
	exit 0
}

if { [string compare [check_accounting_admin_level] "Administrator"] } {
	log_warn "This test can't be run without being an Accounting administrator.\nUse: sacctmgr mod user \$USER set admin=admin."
	exit 0
}

# Cluster Request
array set clus_req {}

# Account Request
array set acct_req {}
set acct_req(cluster) $clusters

# User Request
array set user_req {}
set user_req(cluster) $clusters
set user_req(account) $accounts
set user_req(wckey)   $wckeys

#
# Remove previous test clusters, accounts and users
#
remove_user    $clusters $accounts $users
remove_acct    "" $accounts
remove_cluster $clusters

# Add cluster
incr exit_code [add_cluster $cluster1 [array get clus_req]]
incr exit_code [add_cluster $cluster2 [array get clus_req]]
if { $exit_code } {
	end_it
}

# Add accounts
incr exit_code [add_acct "$account1,$account2" [array get acct_req]]
if { $exit_code } {
	end_it
}

# Add users
incr exit_code [add_user $users [array get user_req]]
if { $exit_code } {
	end_it
}

#
# Get the association ids
# TODO: Assuming cluster1 and cluster2 has the same assoc ids
#
set user1acct1 0
set user1acct2 0
set user2acct1 0
set user2acct2 0
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format="User,account,id"]
expect {
	-re "There was a problem" {
		log_error "there was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user1.$account1.($number)." {
		set user1acct1 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr list associations not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format="User,account,id"]
expect {
	-re "There was a problem" {
		log_error "there was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user1.$account2.($number)." {
		set user1acct2 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr list associations not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format="User,account,id"]
expect {
	-re "There was a problem" {
		log_error "there was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user2.$account1.($number)." {
		set user2acct1 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr list associations not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}
set my_pid [eval spawn $sacctmgr -n -p list assoc users=$users account=$accounts cluster=$cluster1 format="User,account,id"]
expect {
	-re "There was a problem" {
		log_error "there was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user2.$account2.($number)." {
		set user2acct2 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr list associations not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {!$user1acct1 || !$user1acct2 || !$user2acct1 || !$user2acct2} {
	log_error "Didn't get one of the user associations $user1acct1 $user1acct2 $user2acct1 $user2acct2."
	incr exit_code 1
	end_it
}

#
# Get the wckeys ids
# TODO: Assuming cluster1 and cluster2 has the same wckeys ids
#
set user1wckey1 0
set user2wckey1 0
set user1wckey2 0
set user2wckey2 0
set my_pid [eval spawn $sacctmgr -n -p list wckeys users=$users cluster=$clusters format="user,wckey,id"]
expect {
	-re "There was a problem" {
		log_error "There was a problem with the sacctmgr command"
		incr exit_code 1
	}
	-re "$user1.$wckey1.($number)." {
		set user1wckey1 $expect_out(1,string)
		exp_continue
	}
	-re "$user2.$wckey1.($number)." {
		set user2wckey1 $expect_out(1,string)
		exp_continue
	}
	-re "$user1.$wckey2.($number)." {
		set user1wckey2 $expect_out(1,string)
		exp_continue
	}
	-re "$user2.$wckey2.($number)." {
		set user2wckey2 $expect_out(1,string)
		exp_continue
	}
	timeout {
		log_error "sacctmgr wckeys not responding"
		slow_kill $my_pid
		incr exit_code 1
	}
	eof {
		wait
	}
}

if {!$user1wckey1 || !$user2wckey1 || !$user1wckey2 || !$user2wckey2} {
	log_error "Didn't get one of the user wckeys $user1wckey1 $user2wckey1 $user1wckey2 $user2wckey2."
	incr exit_code 1
	end_it
}

#
# Create the SQL files
#
create_sql $cluster1 $sql_in_clus1 $sql_rem_clus1
create_sql $cluster2 $sql_in_clus2 $sql_rem_clus2

#
# Tests
#

# 1 cluster1, 1 user1
_run_test "cluster=$cluster1 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 1 user2
_run_test "cluster=$cluster1 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 users
_run_test "cluster=$cluster1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid05 $jobid06 $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster2, 1 user1
_run_test "cluster=$cluster2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster2, 1 user2
_run_test "cluster=$cluster2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster2, 2 users
_run_test "cluster=$cluster2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid05 $jobid06 $jobid03 $jobid04 $jobid07 $jobid08]


# 1 cluster1, 1 account1, 1 user1
_run_test "cluster=$cluster1 account=$account1 user=$user1" [list $jobid01 $jobid02]

# 1 cluster1, 1 account1, 1 user2
_run_test "cluster=$cluster1 account=$account1 user=$user2" [list $jobid03 $jobid04]

# 1 cluster1, 1 account1, 2 users
_run_test "cluster=$cluster1 account=$account1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2, 1 user1
_run_test "cluster=$cluster1 account=$account2 user=$user1" [list $jobid05 $jobid06]

# 1 cluster1, 1 account2, 1 user2
_run_test "cluster=$cluster1 account=$account2 user=$user2" [list $jobid07 $jobid08]

# 1 cluster1, 1 account2, 2 users
_run_test "cluster=$cluster1 account=$account2 user=$user1,$user2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 1 user1
_run_test "cluster=$cluster1 account=$account1,$account2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 2 accounts, 1 user2
_run_test "cluster=$cluster1 account=$account1,$account2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 2 users
_run_test "cluster=$cluster1 account=$account1,$account2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 wckey1, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey1 user=$user1" [list $jobid01 $jobid05]

# 1 cluster1, 1 wckey1, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey1 user=$user2" [list $jobid03 $jobid07]

# 1 cluster1, 1 wckey1, 2 users
_run_test "cluster=$cluster1 wckey=$wckey1 user=$user1,$user2" [list $jobid01 $jobid05 $jobid03 $jobid07]

# 1 cluster1, 1 wckey2, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey2 user=$user1" [list $jobid02 $jobid06]

# 1 cluster1, 1 wckey2, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey2 user=$user2" [list $jobid04 $jobid08]

# 1 cluster1, 1 wckey2, 2 users
_run_test "cluster=$cluster1 wckey=$wckey2 user=$user1,$user2" [list $jobid02 $jobid06 $jobid04 $jobid08]

# 1 cluster1, 2 wckeys, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user1" [list $jobid01 $jobid02 $jobid05 $jobid06]

# 1 cluster1, 2 wckeys, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user2" [list $jobid03 $jobid04 $jobid07 $jobid08]

# 1 cluster1, 2 wckeys, 2 users
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1account1, 1 wckey1, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user1" [list $jobid01]

# 1 cluster1, 1account1, 1 wckey1, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user2" [list $jobid03]

# 1 cluster1, 1account1, 1 wckey1, 2 users
_run_test "cluster=$cluster1 wckey=$wckey1 account=$account1 user=$user1,$user2" [list $jobid01 $jobid03]

# 1 cluster1, 1account1, 1 wckey2, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user1" [list $jobid02]

# 1 cluster1, 1account1, 1 wckey2, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user2" [list $jobid04]

# 1 cluster1, 1account1, 1 wckey2, 2 users
_run_test "cluster=$cluster1 wckey=$wckey2 account=$account1 user=$user1,$user2" [list $jobid02 $jobid04]

# 1 cluster1, 1account1, 2 wckeys, 1 user1
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user1" [list $jobid01 $jobid02]

# 1 cluster1, 1account1, 2 wckeys, 1 user2
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user2" [list $jobid03 $jobid04]

# 1 cluster1, 1account1, 2 wckeys, 2 users
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2 account=$account1 user=$user1,$user2" [list $jobid01 $jobid02 $jobid03 $jobid04]


# 1 cluster1, 1 account1
_run_test "cluster=$cluster1 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2
_run_test "cluster=$cluster1 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts
_run_test "cluster=$cluster1 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 1 account1
_run_test "cluster=$cluster2 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster2, 1 account2
_run_test "cluster=$cluster2 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 2 accounts
_run_test "cluster=$cluster2 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 wckey1
_run_test "cluster=$cluster1 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster1, 1 wckey2
_run_test "cluster=$cluster1 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster1, 2 wckeys
_run_test "cluster=$cluster1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 1 wckey1
_run_test "cluster=$cluster2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster2, 1 wckey2
_run_test "cluster=$cluster2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster2, 2 wckeys
_run_test "cluster=$cluster2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster1, 1 account1, 1 wckey1
_run_test "cluster=$cluster1 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03]

# 1 cluster1, 1 account1, 1 wckey2
_run_test "cluster=$cluster1 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04]

# 1 cluster1, 1 account1, 2 wckeys
_run_test "cluster=$cluster1 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster1, 1 account2, 1 wckey1
_run_test "cluster=$cluster1 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07]

# 1 cluster1, 1 account2, 1 wckey2
_run_test "cluster=$cluster1 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08]

# 1 cluster1, 1 account2, 2 wckeys
_run_test "cluster=$cluster1 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster1, 2 accounts, 1 wckey1
_run_test "cluster=$cluster1 account=$account1,$account2 wckey=$wckey1" [list  $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster1, 2 accounts, 1 wckey2
_run_test "cluster=$cluster1 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster1, 2 accounts, 2 wckeys
_run_test "cluster=$cluster1 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 1 cluster2, 1 account1, 1 wckey1
_run_test "cluster=$cluster2 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03]

# 1 cluster2, 1 account1, 1 wckey2
_run_test "cluster=$cluster2 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04]

# 1 cluster2, 1 account1, 2 wckeys
_run_test "cluster=$cluster2 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04]

# 1 cluster2, 1 account2, 1 wckey1
_run_test "cluster=$cluster2 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07]

# 1 cluster2, 1 account2, 1 wckey2
_run_test "cluster=$cluster2 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08]

# 1 cluster2, 1 account2, 2 wckeys
_run_test "cluster=$cluster2 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid06 $jobid07 $jobid08]

# 1 cluster2, 2 accounts, 1 wckey1
_run_test "cluster=$cluster2 account=$account1,$account2 wckey=$wckey1" [list  $jobid01 $jobid03 $jobid05 $jobid07]

# 1 cluster2, 2 accounts, 1 wckey2
_run_test "cluster=$cluster2 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08]

# 1 cluster2, 2 accounts, 2 wckeys
_run_test "cluster=$cluster2 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]


# 2 clusters, 1 account1
_run_test "cluster=$cluster1,$cluster2 account=$account1" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid01 $jobid02 $jobid03 $jobid04]

# 2 clusters, 1 account2
_run_test "cluster=$cluster1,$cluster2 account=$account2" [list $jobid05 $jobid06 $jobid07 $jobid08 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 2 accounts
_run_test "cluster=$cluster1,$cluster2 account=$account1,$account2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08 $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 1 wckey1
_run_test "cluster=$cluster1,$cluster2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid05 $jobid07 $jobid01 $jobid03 $jobid05 $jobid07]

# 2 clusters, 1 wckey2
_run_test "cluster=$cluster1,$cluster2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid06 $jobid08 $jobid02 $jobid04 $jobid06 $jobid08]

# 2 clusters, 2 wckeys
_run_test "cluster=$cluster1,$cluster2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08 $jobid01 $jobid02 $jobid03 $jobid04 $jobid05 $jobid06 $jobid07 $jobid08]

# 2 clusters, 1 account1, 1 wckey1
_run_test "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey1" [list $jobid01 $jobid03 $jobid01 $jobid03]

# 2 clusters, 1 account1, 1 wckey2
_run_test "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey2" [list $jobid02 $jobid04 $jobid02 $jobid04]

# 2 clusters, 1 account1, 2 wckeys
_run_test "cluster=$cluster1,$cluster2 account=$account1 wckey=$wckey1,$wckey2" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid02 $jobid04 $jobid02 $jobid04]

# 2 clusters, 1 account2, 1 wckey1
_run_test "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey1" [list $jobid05 $jobid07 $jobid05 $jobid07]

# 2 clusters, 1 account2, 1 wckey2
_run_test "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey2" [list $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 1 account2, 2 wckeys
_run_test "cluster=$cluster1,$cluster2 account=$account2 wckey=$wckey1,$wckey2" [list $jobid05 $jobid07 $jobid05 $jobid07 $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 2 accounts, 1 wckey1
_run_test "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey1" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid05 $jobid07 $jobid05 $jobid07]

# 2 clusters, 2 accounts, 1 wckey2
_run_test "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey2" [list $jobid02 $jobid04 $jobid02 $jobid04 $jobid06 $jobid08 $jobid06 $jobid08]

# 2 clusters, 2 accounts, 2 wckeys
_run_test "cluster=$cluster1,$cluster2 account=$account1,$account2 wckey=$wckey1,$wckey2" [list $jobid01 $jobid03 $jobid01 $jobid03 $jobid05 $jobid07 $jobid05 $jobid07 $jobid02 $jobid04 $jobid02 $jobid04 $jobid06 $jobid08 $jobid06 $jobid08]

#
# Finish
#
end_it