#!/usr/gnu/bin/perl

$help = <<ENDHELP;
	Usage:  queriermonitor (no arguments)
	Purpose: Restart permanent server applications if they have died,
		but only if informix is running.
	If informix is not running, kill applications.
	Also restarts the Cacher if it has died.

	It is machine-specific!! Make sure it is correct for the machine
	you are targeting (esp11, hpcuppb, spp170.external.hp.com)
	In particular, the values of $path, INFORMIXDIR, INFORMIXSERVER
	are machine-specific.
	
	queriermonitor is designed to be executed as a crontab job every few
	minutes.  It checks to see if the designated processes are running, and if
	not, restarts them.
	
	10/10/2000:  For the application programs, do a more sophisticated
		check, since these programs can be running but 'hung'.  The
		new check is to run the proxy cgi program, and wait for an
		exit status.
			0 : all is well
			4 : application not running.  Restart.
			5 : application running but hung. Kill and restart
	
	Redirect STDOUT to a file, to log activity.
ENDHELP

$progroot = "/dbbrowse/WinrefBoot";

%InformixRoot = 
(	
	name => 'informix',
	pattern => 'oninit',
	procname => 'informix',
	children => 
	[
		{
			name   => 'dsrouter',
			pattern => 'dbbrowse.*dsrouter',
			procname => 'dsrouter.cgi',
# NO LONGER NEEDED	proxy => "$progroot/DEALSEARCH/toperlsock.cgi"
		},
		{
			name => 'submittalform',
			pattern => 'dbbrowse.*submittalform',
			procname => 'submittalform.cgi',
			proxy => "$progroot/NEWSUBMITTAL/submittalproxy.cgi"
		},
		{
			name => 'trrouter',
			pattern => 'dbbrowse.*trrouter',
			procname => 'trrouter.cgi',
		}
	]
);

%CacherRoot = 
(	
	name   => 'cacher',
	pattern => 'Cacher',
	procname => 'Cacher.pl',
	children => 
	[
		{
			name => 'dsrouter',
			pattern => 'dbbrowse.*dsrouter',
			procname => 'dealsearch.cgi',
		},
		{
			name => 'submittalform',
			pattern => 'dbbrowse.*submittalform',
			procname => 'submittalform.cgi',
		},
		{
			name => 'trrouter',
			pattern => 'dbbrowse.*trrouter',
			procname => 'trrouter.cgi',
		}
	]
);

if (@ARGV) {
	print $help;
	exit(0);
}


#-----------------------------------------------#
#  descend process chain, killing all processes	#
#  which depend on a killed parent process		#
#-----------------------------------------------#
&CascadeKill(\%InformixRoot);
if (&IsRunning(\%InformixRoot))
{
	&Restart($InformixRoot{children});
}
else
{
	die "Informix is not running!!: $!\n";
}

&CascadeKill(\%CacherRoot);
if (!&IsRunning(\%CacherRoot))
{
	&cacher_restart();		# we can restart the Cacher
}
if (&IsRunning(\%CacherRoot))
{
	&Restart($CacherRoot{children});
}

sub CascadeKill
{
	my $href = shift;
	my $running = &IsRunning($href);
	if (!$running)
	{
		foreach my $child (@{ $href->{children} })
		{
			&Kill($child);
		}
	}
	foreach my $child (@{ $href->{children} })
	{
		&CascadeKill($child);
	}
}

sub IsRunning
{
	my $href = shift;
	my $pid = 0;
	
	my $cmd = 'ps -ef | grep ' . $href->{pattern};
	open (RSLT, "$cmd |") or die "Can't open pipe to $cmd:$!\n";
	my @results = <RSLT>;
	close(RSLT);
	my $procname = $href->{procname};
	my @found = grep { /$procname/ } @results;
	
	if (@found)
	{
		my ($owner, $rest);
		($owner, $pid, $rest) = split(" ",$found[0]);
	}

	if ($pid && exists($href->{proxy}))
	{
		$Strategy = <<STRATEGY;
		rm proxy.out
		toperlsock.cgi > proxy.out 2>&1
		wait for exit status of above
		if exit status is 0, all is ok
		if exit status is 4, dealsearch was not running
		if exit status is 5, dealsearch is running but hung
		if exit status <> 0 
			kill process
STRATEGY
		my $outfile = 'proxy.out';
		unlink($outfile);

		my $proxy = $href->{proxy};
		delete $href->{proxy};		# only do this check once per run
		
#		printf ("Evaluating %s with %s...\n",$href->{name},$proxy);
		system("$proxy >$outfile 2>&1");
		$exit_value = $? >> 8;
		$signal_value = $? & 127;
		$core_dumped = $? & 128;

		printf ("exitvalue=%d, signal=%d, coredumped=%d\n",$exit_value,
			$signal_value, $core_dumped)
			if ($exit_value + $signal_value + $core_dumped);
			
		if ($exit_value != 0)
		{
			Kill($href);
			$pid = 0;
		}
	}
	
	return $pid;
}

sub Kill
{
	my $href = shift;
	my $pid = &IsRunning($href);
	if ($pid)
	{
		$cmd = "kill $pid";
		system($cmd);
		print  localtime() . ", Killed " . $href->{procname} . ", pid=$pid.\n";
	}	
}

sub Restart
{
	my $aref = shift;
	foreach my $child (@$aref)
	{
		if (!IsRunning($child))
		{
			eval($child->{name} . '_restart');
		}
		if (IsRunning($child))
		{
			&Restart($child->{children});
		}
	}
}

sub querier_restart
{
	my $path = "$progroot/SOCKSERVER";
	my $cmd = "cd $path; export INFORMIXDIR=/informix; export INFORMIXSERVER=hpsfcom_online; nohup ./Querier.pl > /dev/null & \n";
	print $cmd;
	system($cmd);
	print  localtime() . ", Restarting Querier...\n";
	print "sleeping 20 seconds...\n";
	sleep(20);		# give Querier a chance to come up before launching
					# process which depend on (connect to) it
}

sub cacher_restart
{
	my $path = "$progroot/SOCKSERVER";
	my $cmd = "cd $path; nohup ./Cacher.pl > /dev/null & \n";
	print $cmd;
	system($cmd);
	print  localtime() . ", Restarting Cacher...\n";
}	

sub dsrouter_restart
{
	my $path = "$progroot/DEALSEARCH";
	my $cmd = "cd $path; nohup ./dsrouter.cgi > /dev/null 2>&1 & \n";
	print $cmd;
	system($cmd);
	print  localtime() . ", Restarting DSrouter...\n";
}	

sub submittalform_restart
{
	my $path = "$progroot/NEWSUBMITTAL";
	my $cmd = "cd $path; nohup ./submittalform.cgi > /dev/null 2>&1 & \n";
	print $cmd;
	system($cmd);
	print  localtime() . ", Restarting Submittalform...\n";
}	

sub trrouter_restart
{
	my $path = "$progroot/TRACKER";
	my $cmd = "cd $path; nohup ./trrouter.cgi > /dev/null 2>&1 & \n";
	print $cmd;
	system($cmd);
	print  localtime() . ", Restarting TRrouter...\n";
}