#!/usr/bin/perl
use strict;
use Socket;
use Sys::Hostname;
my $host = hostname();

my $phost_ip =  gethostbyname($host);
my $host_ip;
if (defined $phost_ip) {
    $host_ip = inet_ntoa($phost_ip);
} else {
    $host_ip = "";
}

#usage eoshapl manager-node1 manager-node2 alias failovertime [debug]

$| = 1;

system("echo $$ >& /var/run/eosha.pid");

my $logfile = (shift @ARGV) or exit -1;
my $target1 = (shift @ARGV) or exit -1;
my $target2 = (shift @ARGV) or exit -1;
my $aliasname = (shift @ARGV) or exit -1;
my $failovertime = (shift @ARGV) or exit -1;
my $debug = (shift @ARGV or 0);

my $now;

sub HUP_handler {
    close STDOUT;
    close STDERR;

    open STDOUT, ">> $logfile";
    open STDERR,  ">> $logfile";

    $SIG{'HUP'} = 'HUP_handler';
    now();
    printf STDOUT "$now log rotated ...";

}

$SIG{'HUP'} = 'HUP_handler';

my $mgmstatus="";
my $mqstatus="";
my $eossyncstatus="";
my $eosshtimeout=10;

my $mastermgm=0;
my $mastermq=0;
my $masterfed=0;
my $eossync=0;


sub now {
    $now = `date`;
    chomp $now;
    $now .= " ";
    $now .= `date +%s`;
    chomp $now;
    return $now;
}

# Variable for checking if service manager is systemd
my $SYSTEMD = `pidof systemd`;

# Function for checking status of EOS services
sub service_status {
	if ( $SYSTEMD = 1 ) {
		$mastermgm = system("env CONSOLETYPE=serial systemctl status eos\@mgm >&/dev/null");
		$mastermq  = system("env CONSOLETYPE=serial systemctl status eos\@mq >&/dev/null");
		$masterfed = system("env CONSOLETYPE=serial systemctl status eos\@fed >&/dev/null");
		$eossync   = system("env CONSOLETYPE=serial systemctl status eossync@* >&/dev/null");
	} else {
		$mastermgm = system("env CONSOLETYPE=serial /etc/rc.d/init.d/eos status mgm >&/dev/null");
		$mastermq  = system("env CONSOLETYPE=serial /etc/rc.d/init.d/eos status mq >&/dev/null");
		$masterfed = system("env CONSOLETYPE=serial /etc/rc.d/init.d/eos status fed >&/dev/null");
		$eossync   = system("env CONSOLETYPE=serial /etc/rc.d/init.d/eossync status >&/dev/null");
	}
}

my $cnt=0;
printf "$now ===================================================================\n";
printf "$now Started EOSHA - EOS High Availability\n";
printf "$now ___________________________________________________________________\n";

while(1) {
    now();
    service_status();

    $mastermgm = ($mastermgm >> 8);
    $mastermq  = ($mastermq >> 8);
    $masterfed = ($masterfed >> 8);
    $eossync   = ($eossync >> 8);

    if (!$mastermgm) {
	$mgmstatus="<up>";
    } else {
	$mgmstatus="<down>";
    }

    if (!$mastermq) {
	$mqstatus="<up>";
    } else {
	$mqstatus="<down>";
    }

    if (!$eossync) {
	$eossyncstatus="<up>";
    } else {
	$eossyncstatus="<down>";
    }

    if ($debug || (!$cnt)) {print "$now hostname=$host [$host_ip]  aliases=$aliasname\n";}

	service_status();

    $mastermgm = ($mastermgm >> 8);
    $mastermq  = ($mastermq >> 8);
    $masterfed = ($masterfed >> 8);
    $eossync   = ($eossync >> 8);

    if ( $mastermgm && ($mastermgm != 2)) {
	now();
	printf "$now ===================================================================\n";
	printf("$now restart dead MGM ...\n");
	printf "$now ___________________________________________________________________\n";
	# service dead ... let's restart
	if ( $SYSTEMD = 1 ) {
		my $startupmgm = system("env CONSOLETYPE=serial systemctl restart eos\@mgm >> $logfile 2>&1 ");
	} else {
		my $startupmgm = system("env CONSOLETYPE=serial /etc/rc.d/init.d/eos restart mgm >> $logfile 2>&1 ");
	}
    }

    if ( $mastermq && ($mastermq != 2)) {
	now();
	printf "$now ===================================================================\n";
	printf("$now restart dead MQ ...\n");
	printf "$now ___________________________________________________________________\n";
	# service dead ... let's restart
	if ( $SYSTEMD = 1 ) {
		my $startupmq = system("env CONSOLETYPE=serial systemctl restart eos\@mq >> $logfile 2>&1 ");
	} else {
		my $startupmq = system("env CONSOLETYPE=serial /etc/rc.d/init.d/eos restart mq >> $logfile 2>&1 ");
	}
    }

    if ( $masterfed ) {
	now();
	printf "$now ===================================================================\n";
	    printf("$now (re-)start FED ...\n");
	printf "$now ___________________________________________________________________\n";
	# service dead ... let's restart
	if ( $SYSTEMD = 1 ) {
		my $startupfed = system("env CONSOLETYPE=serial systemctl restart eos\@fed >> /dev/null 2>&1 ");
	} else {
		my $startupfed = system("env CONSOLETYPE=serial /etc/rc.d/init.d/eos restart fed >> /dev/null 2>&1 ");
	}
    }

    if ( $eossync ) {
	now();
	printf "$now ===================================================================\n";
	printf("$now (re-)start EOS SYNC ...\n");
	printf "$now ___________________________________________________________________\n";
	# service dead ... let's restart
	if ( $SYSTEMD = 1 ) {
		my $startupsync = system("env CONSOLETYPE=serial systemctl restart eossync@* >> $logfile 2>&1 ");
	} else {
		my $startupsync = system("env CONSOLETYPE=serial /etc/rc.d/init.d/eossync restart >> $logfile 2>&1 ");
	}
    }
    # check that the MGM is operational
    if (!$mastermgm) {
	my $mgmdead=system("eossh-timeout -t $eosshtimeout xrd localhost stat / >& /dev/null");
	$mgmdead = ($mgmdead >> 8);
	if ($mgmdead) {
	    now();
	    printf "$now ===================================================================\n";
	    printf "$now local alive test gave a timeout after $eosshtimeout seconds ...\n";
	    printf "$now ___________________________________________________________________\n";
	    if ( $ENV{'EOS_NOTIFY'} ne "" ) {
		my $timefile=`find /var/lock/subsys/eos.mgmtimeout -mmin -10 -mmin +0 2>/dev/null`;
		if ( "$timefile" eq "/var/lock/subsys/eos.mgmtimeout" ) {
		    printf "$now MGM didn't respond within $eosshtimeout seconds .. [ not sending mail notification for some time ... ]\n";
		} else {
		    system("echo $now MGM on $host\\(me\\) did not reply within timeout of $eosshtimeout seconds ... | $ENV{EOS_NOTIFY}");
		    system("touch /var/lock/subsys/eos.mgmtimeout");
		    printf "$now MGM didn't respond within $eosshtimeout seconds .. [ sending mail via $ENV{'EOS_NOTIFY'} ]\n";
		}
	    }
	}
    }

    sleep(5);
    $cnt++;
}
