#!/usr/bin/perl -w

# TODOs:
# - incorporate the column_ranges() function from scut to create ranges
# - query active nodes from qhost via STDIN consumption of:
#   qhost |grep amd64 | grep -v ' - '

=head1 NAME
clustexec.pl

=head1 SYNOPSIS

  % {sudo} clustexec.pl 'remote command' MACHINELIST {FORK}

=head1 DESCRIPTION

clustexec.pl automates the remote execution of commands to a cluster.
Depending on the options, it will execute the commands serially or
in parallel.  If the former, the output of each node will be echoed
to STDOUT.  With the 'FORK' option, it will operate in parallel and
pipe the output of the commands into a newly created dir, each file
labeled with the IP# of the node on which it has executed.

It can also do primitve analysis on the output files , clustering
the files by 'wc', and 'md5sum' to show which groups of outputs are
identical or nearly so.  The files can then be viewed via 'Midnight
Commander' (mc), in a new xterm or the same text terminal.

type 'clustexec.pl' alone to dump the full usage text.

=head1 COMMAND-LINE OPTIONS

Usage: {sudo} clustexec.pl 'remote command' options
where the options are, in order:

[NACS|ICS|BOTH|FILE:/path/to/file] .... REQUIRED
{FORK} ................................ OPTIONAL

The first option defines which nodes get the command.
NACS - all the nodes in the ADC subcluster (a64-101...167; at this time)
ICS  - all the nodes in the ICS subcluster (a64-001...040; at this time)
BOTH - both the above
FILE:/path/to/file - takes IP#s from a file defined by '/path/to/file'


=head1 AUTHOR
Harry Mangalam <hjm@tacgi.com>, <harry.mangalam@uci.edu>
Copyright (c) 2009, 2010 UC Irvine

=head1 HISTORY
This was first implemented as a bash script until it got too long.

=cut

use strict;
use Socket;
use Env;
use Env qw(HOME);

use vars qw( $HOST $CMD $DATEDIR $FORK $DATE $DATEDIR $SHORT_CMD $PWD
$HOSTLIST $IPFILE $CLUSTER $IP_NMBR $md5list  $wclist %md5h %wch $HOME
@md5 @wc $tmp @L $N $DEBUG $line
);

# call as host_loop($HOST, $CMD, $DATEDIR, $FORK)
sub host_loop($$$$) {

	$HOST = shift;
	$CMD = shift;
	$DATEDIR = shift;
	$FORK = shift;
	my $IP_NMBR = "";
	my $HOSTNAME = "";
	#print "\n";
	#print "HOST -> $HOST\n";
	if ($HOST =~ /[a-z]/) { # then it's a hostname, so have to look up the IP#
	    #print "looking up IP#..\n";
	    my $packed_ip = gethostbyname($HOST);
	    if (defined $packed_ip) { $IP_NMBR = inet_ntoa($packed_ip); }
	    else {$IP_NMBR = "IP UNDEFINED";}
	    $HOSTNAME = $HOST;
        } else { # it's already an IP#, so look up the hostname
	    #print "looking up HOSTNAME..\n";
	    $HOSTNAME = gethostbyaddr(inet_aton($HOST), AF_INET)
	       or die "Can't resolve $HOST: $!\n";
            #print "HOSTNAME = $HOSTNAME\n";
	    $IP_NMBR = $HOST; # and copy for output.
	}
#	print ".\n";
	print "\n$HOSTNAME [$IP_NMBR]: \n";
#        print "..\n";
	my $PING_RESPONSE = `ping -c1 $HOST`;
#	 print "...\n";
	if ( $PING_RESPONSE =~ /100% packet loss/ ) {
		print "                              .... unresponsive..\n";
	} elsif ($FORK eq "FORK") {
	     if ( -d $DATEDIR ){
#	       print ".... $DATEDIR/$HOST \n";
	       system("ssh $HOST $CMD &> $DATEDIR/$HOST &");
	       sleep 0.25;
	     } else { die "ERROR: Dir $DATEDIR doesn't exist.\n"; }
	 } else {
		system("ssh $HOST $CMD"); # this should dump the command output to STDOUT.
	 }
}



sub usage {
   #print "HOME ENV = $HOME\n";
   #my $rr = <STDIN>;
   my $helpfile="$HOME/clustexec_help.tmp";
   my $helptxt = "";
   open HLP, ">$helpfile" or die "Can't open the summary file\n";
   $helptxt = <<HELP;

SUMMARY
=======
clustexec.pl automates the remote execution of commands to a cluster.
Depending on the options, it will execute the commands serially or
in parallel.  If the former, the output of each node will be echoed
to STDOUT.  With the 'FORK' option, it will operate in parallel and
pipe the output of the commands into a newly created dir, each file
labeled with the IP# of the node on which it has executed.

It can also do primitive analysis on the output files , clustering
the files by 'wc', and 'md5sum' to show which groups of outputs are
identical or nearly so.  The files can then be viewed via 'Midnight
Commander' (mc), in a new xterm or the same text terminal.


USAGE
=====
clustexec.pl (alone, it produces this help)

{sudo} clustexec.pl 'remote command' options

('remote command' MUST be quoted.)

where the options are, in order:

[NACS|ICS|BOTH|FILE:/path/to/file] .... REQUIRED
{FORK} ................................ OPTIONAL

The first option defines which nodes get the command.
NACS - all the nodes in the ADC subcluster (a64-101...167; at this time)
ICS  - all the nodes in the ICS subcluster (a64-001...040; at this time)
BOTH - both the above
FILE:/path/to/file - takes IP#s from a file defined by '/path/to/file'
   (see INPUT FILE FORMAT below)

If you append the argument 'FORK' to the commandline, the 'remote command'
will fork on each instance, not waiting for the 1st to complete.
Therefore each command issued with the 'FORK' version has to be
self-contained (can't require a response).

For example, if installing the RPMs with yum, the yum command has to
include the '-y' flag:
   sudo clustexec.pl 'yum -y install [RPM list]' NACS FORK

If you use the FORK command (as above)), instead of producing the stdout/err
immediately, a new subdir will be created with the name of the format:
   REMOTE_CMD-(20 chars of the command)-time_date
and the output for each node will be directed into a separate file
named for the IP number or hostname (whichever the input spec was).

It will execute whatever you assign to 'remote command' on all the hosts
that result from the iteration.


INPUT FILE FORMAT
=================
If using the 'FILE:/path/to/file' option, the file must be in the format
of 1 hostname or IP# per line, optionally appended by the number of mpd
slots so you can feed it the mpd.hosts file
ie
---------------
a64-013
a64-014
a64-015
a64-017
---------------
       or
---------------
a64-013:2
a64-014:2
a64-015:2
a64-017:2
---------------


OUTPUT
======
If executed without the 'FORK', it will produce a list of stanzas
corresponding to the nodes requested:
--------------------------------
a64-101 [192.168.0.10]:
 <output from the command>

a64-102 [192.168.0.11]:
 <output from the command>
--------------------------------

If 'FORK' is appended, it will produce only the header, with the output
being redirected into the REMOTE_CMD.. directory noted.

In both cases, if the node doesn't respond to a single ping, it will say:
--------------------------------
a64-130 [192.168.0.39]:
                              .... unresponsive..
--------------------------------

VERIFICATION
============
At the end of the run, it will summarize the output by word count (wc)
and md5 checksum (md5sum) to cluster the files by identical and
'nearly identical' output.  This data is written to a file named "Summary"
in the target directory, and can optionally be viewed with 'less' from
the program.  The same summary data is also printed to STDOUT.
There's a final option to view the files created via 'Midnight Commander'
in a new xterm or the current text term.

AUTHOR
------
Harry Mangalam <hjm\@tacgi.com>, <harry.mangalam\@uci.edu>
Copyright (c) 2009 UC Irvine


!!!!!!!!!!!!!!!!  Be careful what you ask for. !!!!!!!!!!!!!!!!!!!!

HELP

print HLP $helptxt;
close HLP;

system("less -S $helpfile");
unlink $helpfile;
exit(0);
}


#print "# of args = $#ARGV \n";
#my $i = 0;
#foreach my $i (0..$#ARGV){ print "arg[$i] = $ARGV[$i]\n"};

if ($#ARGV < 1){
   usage();
   die "ERROR: You need more options.\n"
}

$CMD =     $ARGV[0];
$CLUSTER = $ARGV[1];
$FORK =    $ARGV[2];
$PWD = `pwd`; chomp $PWD;

$DATE=`date +"%T_%F" | sed 's/:/./g' `; chomp $DATE;

$DEBUG = 0;

$SHORT_CMD = substr($CMD,0,20); # chop it
#print "CMD  = $CMD \n";
#print "SHORT CMD  = $SHORT_CMD \n";
#$SHORT_CMD =~ s/\|//g; # get rid of '|' as it might lead to unpleasant surprises
#$SHORT_CMD =~ s/ /_/g; # and sub _ for spaces
#$SHORT_CMD =~ s/\//::/g; # and sub '::' for '/'
#print "SHORT CMD  = $SHORT_CMD \n";
#my $tmp=<STDIN>;
#$SHORT_CMD =~ s/;/-/g; # and sub '::' for ';'
$SHORT_CMD =~ s/[\|\ \/\;\~\!\@\#\$\%\^\&\*\(\)\{\}\[\]\{\}]/-/g; # and sub '::' for ';'
#print "SHORT CMD  = $SHORT_CMD \n";
#$tmp=<STDIN>;

if ($DEBUG){$DATEDIR = "REMOTE_CMD-yum_list_installed-17.09.42_2009-10-01";
} else {$DATEDIR = "REMOTE_CMD-" . $SHORT_CMD . "-" . $DATE;}

#print "DATEDIR  = $DATEDIR \n";
if (!$DEBUG){
if (defined $FORK && $FORK eq "FORK"){
	print "INFO: Creating dir [$DATEDIR]..";
	if (mkdir $DATEDIR) {print "..success\n";}
	else {die "ERROR: Can't mkdir [$DATEDIR] at [$PWD]!\n";}
} else {$FORK = "NULL";}

if ($CLUSTER eq "BOTH" || $CLUSTER eq "NACS") {
	print <<"NACSNODES";

======================================================
        Processing NACS nodes
======================================================

NACSNODES

   foreach my $iter (10..76) {
    $HOST="10.255.78." . $iter;
	 host_loop($HOST, $CMD, $DATEDIR, $FORK);
   }
}

if ($CLUSTER eq "BOTH" || $CLUSTER eq "ICS") {
	print <<"ICSNODES";

======================================================
        Processing ICS nodes
======================================================

ICSNODES

   foreach my $iter (5 .. 44) {
	 $HOST="10.255.89." . $iter;
	 host_loop($HOST, $CMD, $DATEDIR, $FORK);
   }
}

# For FILE:/path/to/file
if ( substr($CLUSTER,0,4) eq "FILE"){
	$IPFILE = substr($CLUSTER,5);
	print <<"FILENODES";

======================================================
Processing Nodes from [$IPFILE]
======================================================

FILENODES

	if (-f $IPFILE && -r $IPFILE ) {
		print "INFO: Input file [$IPFILE] exists and is readable..\n";
	} else {
		die "ERROR: [$IPFILE] either doesn't exist or isn't readable by you..";
	}
	open(IPF, "<$IPFILE");
	while (<IPF>){
		$HOST = $_; chomp $HOST;
		if ($HOST =~ /:/){$HOST =~ s/:[0-9]*//;}
		host_loop($HOST, $CMD, $DATEDIR, $FORK);
	}
}
if ($CLUSTER ne "BOTH" && $CLUSTER ne "NACS" && $CLUSTER ne "ICS" && substr($CLUSTER,0,4) ne "FILE"){
	print "ERROR: I think you may have made a typo. Here's how to use it.. \n";
	usage();
	exit(1);
}

} # DEBUG stanza skipped

local $| = 1;

print <<WAITAMIN;

  -------------============== CAUTION ==============-------------
Since this is going to be a variable length process with a number of nodes
writing into the dir over a network, you might want to wait a few seconds
before you continue to let all the nodes complete and the file handles close.

If not, the analysis may well be faulty as it will analyze the CURRENT file
state, not the FINAL file state.

Hit [Enter] when you want to complete the analysis of the results.

WAITAMIN

my $tmp = <STDIN>;


if ($FORK eq "FORK"){
   print "\n\nYou can find the results of your command in the dir\n\t[ $DATEDIR ]\n\n";
   $md5list = `cd $DATEDIR; md5sum * | sort`;
   chomp $md5list;
   $wclist = `cd $DATEDIR; wc * | grep -v total | sort -g`;
   chomp $wclist;

   $N = @md5 = split(/\n/,$md5list);
   #print "DEBUG: Got [$N] entries for array md5\n";
   #for (my $e=0; $e<$N;$e++){
   #   print "DLOOP: $md5[$e]\n";
   #}

   foreach $line (@md5) {
      chomp $line;
      #print "DEBUG: md5 line = $line\n";
      $N = @L = split(/\s+/,$line);
      if ($N != 2){print "ERROR: Unexpected # of fields [$N] splitting md5 input line: [$line]\n";}
      else {
	 $md5h{$L[0]} .= "$L[1]" . " "; # $md5{md5sum} = add to host list
	 #print "DEBUG: md5h entry for [ $L[0] ]: [ $md5h{$L[0]} ] \n";
      }
   }
   # and now for the wc data
   $N = @wc = split(/\n/,$wclist);
   #print "Got [$N] entries for array wc\n";

   foreach my $line (@wc) {
      chomp $line;
      $N = @L = split(/\s+/,$line);
      if ($N != 5){print "ERROR: Unexpected # of fields [$N] splitting wc input line: [$line]\n";}
      else {$wch{$L[4]} = $L[1] . " " . $L[2]. " " . $L[3];} # $wch[host] = ''line_c  word_c  char_c'
      #print "DEBUG: (wc entry) $wch{$L[4]}\n";
   }
   # write analysis to disk in $DATEDIR
   open OUT, ">$DATEDIR/Summary" or die "Can't open the summary file\n";
   print OUT "Analysis of contents for files in $DATEDIR\n";
   print OUT "========================================================================\n";
   print OUT " line / word / chars  |              md5 sum                 |  hosts ->\n";
   foreach my $key (keys %md5h){
      my $n = my @l = split(/\s+/, $md5h{$key});
      printf OUT "%20s  %37s    %s\n", $wch{$l[0]}, $key, $md5h{$key}; # wc md5 host_list
      #printf     "%20s  %37s    %s\n", $wch{$l[0]}, $key, $md5h{$key};
      #my $tmp = <STDIN>;
   }
   print "\n\nSummary printed to $DATEDIR/Summary.\n\n";
   print "View in 'less'? [Yn]";
   my $tmp = <STDIN>;
   if ($tmp !~ /[Nn]/){
      system("less -S $DATEDIR/Summary");
   }

   # and now spit it back in better form.
   print "\nAnalysis of contents for files in $DATEDIR\n";
   print "========================================================================\n";
   print " line / word / chars  |              md5 sum                 |  hosts ->\n";
   foreach my $key (keys %md5h){
      my $n = my @l = split(/\s+/, $md5h{$key});
      printf "%20s  %37s    %s\n", $wch{$l[0]}, $key, $md5h{$key}; # wc md5 host_list
   }

   if (-e "/usr/bin/mc" && -x "/usr/bin/mc"){
      print "\n\nWould you like to view the results with 'mc'? [Yn]  ";
      $tmp = <STDIN>;
      if (($tmp !~ /[Nn]/) && (-x "/usr/bin/xterm")) {
	 print "\nin a new [x]term or via this [t]ext terminal? [Tx]?";
	 $tmp = <STDIN>;
	 if ($tmp =~ /[xX]/){ system("cd $DATEDIR; xterm -geometry 100x40 -e mc"); }
	 else {system("cd $DATEDIR; mc;")}
      } elsif ($tmp !~ /[Nn]/) {system("cd $DATEDIR; mc;")}
      else {print "\nHave it your own way.  Bye!\n\n";}
   }
}

