#!/usr/bin/env expect
############################################################################
# Purpose: Test of Slurm functionality
#          Verify srun behaves when its controlling terminal disappears.
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2006 The Regents of the University of California.
# Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
# Written by Christopher J. Morrone <morrone2@llnl.gov>
# CODE-OCEC-09-009. All rights reserved.
#
# This file is part of Slurm, a resource management program.
# For details, see <https://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# Slurm is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with Slurm; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id              "1.55"
set test_script          "test$test_id.sh"
set step_id              0

print_header $test_id

#
# Create a test script to be launch by srun
#
exec $bin_rm -f $test_script
make_bash_script $test_script {
	echo "Running"

	sleep 5

	for ((i = 0; i < 100; i++)); do
		cat /etc/hosts
	done
}

#
# Launch the test script
#
set timeout $max_job_delay
set jobid  0
set stepid 0
set job_running 0
set srun_pid [spawn $srun --unbuffered -v -n1 -t1 ./$test_script]
expect {
	-re "launching (($number)\.$step_id)" {
		set stepid $expect_out(1,string)
		set jobid $expect_out(2,string)
		exp_continue
	}
	"Running" {
		set job_running 1
	}
	timeout {
		send_user "\nFAILURE: Failed to launch test program through srun\n"
		slow_kill $srun_pid
		exit 1
	}
	eof {
		wait
	}
}
if {$jobid == 0 || $job_running == 0} {
	send_user "\nFAILURE: Failed to launch test program through srun\n"
	exit 1
}

#
# Kill srun's stdio streams
#
send_user "Test saw stepid $stepid\n"
set file [exp_open]
close $file

#
# Lets see if the job step finishes normally
#
set running "seed"
set i 0
while {[string compare $running ""]} {
	set running [exec $squeue --noheader --states=running --steps=$stepid]
	sleep 1
	if {$i == 30} {
		send_user "\nFAILURE: job step is not completing\n"
		cancel_job $jobid
		sleep 5
		exec kill -9 $srun_pid
		exit 1
	}
	incr i 1
}
send_user "Test sees step is gone, srun should have exited as well.\n"

#
# And finally check to see if srun is still hanging around (it should
# have exited by now) and job has completed
#
if [catch {exec kill -0 $srun_pid}] {
	send_user "\nsrun command is terminated, as desired\n"
	set exit_code 0
} else {
	send_user "\nFAILURE: srun is still running after job exits!\n"
	set exit_code 1
}
spawn $squeue --noheader --jobs=$jobid
expect {
	-re "R" {
		send_user "\nFAILURE: job not completed!\n"
		set exit_code 1
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: squeue not responding\n"
		set exit_code 1
		exit 1
	}
	eof {
		wait
	}
}

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
	exec $bin_rm -f $test_script
	exit 0
} else {
	cancel_job $jobid
	exit 1
}