#! /usr/bin/env perl

#################################################################
#
# Purpose: Perform Grid2003 site verification checks on remote
#          host(s) and report results.  These checks include:
#
#          * checking official Grid2003 database entries
#          * checking MDS information by connecting to the
#            LDAP server running on head nodes
#          * authentication to remote site using credentials
#            of person running this script
#          * gsiftp from/to localhost to/from remote site
#          * execution of "hello, world" script
#          * examination of remote grid-mapfile
#          * checking for running monitoring services
#          * verification of runnable environment for VO users
#            and application installers
#
# TODO/WISHLIST:
#  * fix display of "Hello World" errors (eg: auth ok, but stdout filename
#    not created)
#  * subroutine-ize remaining checks
#  * colorize responses?  (green=good, red=bad, uncolored=neutral)
#  * reduce globus-job-run calls (directory existence, space, write)
#  * add "--vo=..." to allow filtering by sponsoring VO from MDS
#  * add "--jobmanager=..." to allow globus-job-run commands to use
#    a user-specified jobmanager.
#
#################################################################

#
# This script depends upon a few CPAN modules that may not
# be included in your perl distribution:
#
# DBI, Net::LDAP, Net::LDAP::Filter
#
# If they are not, and you do not wish to install them 
# globally (or are not able to due to privileges), you can 
# still run this script by 
# 
# 1) installing them wherever you have write access
# 2) EITHER
#    * Add that directory to your PERL5LIB environment variable.
#    OR
#    * Uncomment the BEGIN block below.
#      Replace "YOUR_PERSONAL_PERLDIR" with the path to your
#      module installation(s).  You may add as many "push"
#      statements as you like to the BEGIN block.
#
#BEGIN { push(@INC, "YOUR_PERSONAL_PERLDIR"); }

#
# $tests->{STATUS}{$testname} = SUCCESS, FAILURE, or UNTESTED
# $tests->{STDOUT}{$testname} = text output to be display
# $tests->{STDERR}{$testname} = text output to be display
#
# $opts->{TESTNAME}                  name of current test
#
# $opts->{FLAGS}{VERBOSE}            verbosity flag
# $opts->{FLAGS}{DEBUG}              debug flag
#
# $opts->{GATEKEEPER_HOST}           hostname of remote gatekeeper service
# $opts->{GATEKEEPER_PORT}           port number of remote gatekeeper service
# $opts->{GSIFTP_HOST}               hostname of remote gsiftp service
# $opts->{GSIFTP_PORT}               port number of remote gsiftp service
# $opts->{MDS_HOST}                  hostname of remote MDS service
# $opts->{MDS_PORT}                  port number of remote MDS service
#
# The $opts->{LOCAL_FILES} hash has the following keys:
#
#   SERVICES         Local filename of remote /etc/services
#   INETD_CONF       Local filename of remote /etc/inetd.conf or xinetd.conf,
#                    as appropriate.  For xinetd.conf, any includedir 
#                    directives are processed and substituted.
#   GRID3_INFO_CONF  Local filename of remote grid3-info.conf
#   GRID_INFO_CONF   Local filename of remote grid3-info.conf
#
# The $opts->{REMOTE_FILES} hash has the following keys:
#
#   INETD_CONF      Path to remote inet super-server configuration file.
#
# The $opts->{FILE_HASH} hash has the following keys:
#
#
# $opts->{FILE_HASH}{INETD_CONF}     Representation of the remote site's
#                                    inet super-server configuration file
#                                    (/etc/inetd.conf or /etc/xinetd.conf).
#                                    For xinetd.conf, any includedir 
#                                    directives are processed and substituted.
#                                    The hash contains the following keys:
#
#                                    {service_name}{protocol}{socket_type}
#                                    {service_name}{protocol}{wait}
#                                    {service_name}{protocol}{user}
#                                    {service_name}{protocol}{server}
#                                    {service_name}{protocol}{server_args}
#
#                                    Additional keys exist for each attribute
#                                    in xinetd configurations.
#
# $opts->{FILE_HASH}{SERVICES}       Representation of the remote site's
#                                    Internet network services list.
#                                    The hash contains the following keys:
#
#                                    {service_name}{protocol}{PORT}
#                                    {service_name}{protocol}{ALIASES}
#
# $opts->{JM_TYPES}{$jm} = $type;
# $opts->{SCHED_PATHS}{$type} = $path;
#
#################################################################

use Getopt::Long;
use File::Basename;
use Sys::Hostname;
use Socket;
use IO::File;
use IO::Socket::INET;
use Net::LDAP;
use Net::LDAP::Filter;
use File::Temp;
use File::Compare;

$| = 1;

#
# Define some constants
#

# STATUS constants
use constant UNTESTED => 'UNTESTED';
use constant SUCCESS  => 'PASS';
use constant FAILURE  => 'FAIL';

# Other constants
use constant UNKNOWN  => 'UNKNOWN';
use constant YES      => 'YES';
use constant NO       => 'NO';

#
# get_fqdn
#
# Return the fully qualified domain name associated with the input.
#
sub get_fqdn($)
	     {
    my $in = shift;

    if ( ! defined $in ) { return undef; }

    my $out = $in;

    if ( lc($in) eq 'localhost' )
      {
	$out = hostname();
	my $address = gethostbyname($out)
	      or die "Couldn't resolve $out: $!";
	$out = gethostbyaddr($address, AF_INET)
	      or die "Couldn't re-resolve $out: $!";
      }

    return $out;
  }


#
# get_service_name
#
# Return the name of the service running on the supplied port with the
# supplied protocol from the remote site's Internet network services list.
#
sub get_service_name($$$)
		     {
    my ($opts, $port, $prot) = @_;

    if ( ! defined $opts ) { return undef; }
    if ( ! defined $port ) { return undef; }
    if ( ! defined $prot ) { $prot = 'tcp'; }

    if ( ! exists $opts->{'FILE_HASH'}{'SERVICES'} ) { return undef; }

    my $svcname = undef;

    # XXX could be faster
    foreach my $svc ( keys %{$opts->{'FILE_HASH'}{'SERVICES'}} )
      {
	if ( exists $opts->{'FILE_HASH'}{'SERVICES'}{$svc}{$prot}{'PORT'} &&
	          ($opts->{'FILE_HASH'}{'SERVICES'}{$svc}{$prot}{'PORT'} == $port) )
	  {
	        $svcname = $svc;
		    last;
	      }
      }

    return $svcname;
  }


#
# get_inetd_service_config
#
# Return a hash reference containing the remote site's inetd/xinetd 
# config for the service name and protocol supplied.
#
sub get_inetd_service_config($$$)
			     {
    my ($opts, $svcname, $prot) = @_;

    if ( ! defined $opts ) { return undef; }
    if ( ! defined $svcname ) { return undef; }
    if ( ! defined $prot ) { $prot = 'tcp'; }

    if ( ! exists  $opts->{'FILE_HASH'}{'INETD_CONF'} ) { return undef; }

    if ( exists $opts->{'FILE_HASH'}{'INETD_CONF'}{$svcname}{$prot} )
      {
	return $opts->{'FILE_HASH'}{'INETD_CONF'}{$svcname}{$prot};
      }

    return undef;
  }


#
# run_remote_cmds
#
# Run commands at the remote site using its gatekeeper service.
# Return value is a hash reference containing keys:
#
# Cmd              - The command run
# ExitStatus       - Exit status of globus-job-run
# RemoteExitStatus - Exit status of command executed remotely
# Stdout           - Standard output of remote command
# Stderr           - Standard error of remote command
#
sub run_remote_cmds($$)
  {
    my ($cmds, $ctx) = @_;

    my @a;

    if ( ! defined $cmds || scalar(@$cmds) == 0 ) { return \@a; }
    if ( ! defined $cmds || scalar(@$cmds) == 0 ) { return \@a; }

    my $host = undef;
    if ( exists $ctx->{'GATEKEEPER_HOST'} )
      {
	$host = $ctx->{'GATEKEEPER_HOST'};
      }

    if ( ! defined $host ) { return \@a; }

    my $port = 2119;
    if ( exists $ctx->{'GATEKEEPER_PORT'} )
      {
	$port = $ctx->{'GATEKEEPER_PORT'};
      }

    my $dest = $host;
    if ( defined $port ) { $dest .= ":$port" }

    # Note: To avoid having to authenticate once for each supplied
    # command, which is very expensive, we glom all the commands 
    # together and sort it out later.
    use File::Temp qw(tempfile unlink0 );
    use vars qw($outfh $outf);
    
    my $template = 'outXXXXXX';
#    my $dir      = '$ctx->{\'TMPDIR\'}';
    my $dir      = '/tmp';
    ($outfh, $outf) = tempfile($template, DIR => $dir);


#    my $outfh = new File::Temp(
#			              TEMPLATE => 'outXXXXXX',
#			              DIR      => $ctx->{'TMPDIR'},
#			              UNLINK   => 0,
#			             );
#    my $outf = $outfh->filename;

#    my $errfh = new File::Temp(
#			              TEMPLATE => 'errXXXXXX',
#			              DIR      => $ctx->{'TMPDIR'},
#			       UNLINK   => 0,       
#			             );
#    my $errf = $errfh->filename;
    use File::Temp qw(tempfile unlink0 );
    use vars qw($errfh $errf);
    
    my $template = 'errXXXXXX';
    my $dir      = '/tmp';
    ($errfh, $errf) = tempfile($template, DIR => $dir);

    my $c = "globus-job-run $dest/jobmanager-fork /bin/sh -c \"";

    foreach my $cmd (@$cmds)
      {
	my $h = {};

	$h->{'Cmd'}              = undef;
	$h->{'ExitStatus'}       = undef;
	$h->{'RemoteExitStatus'} = undef;
	$h->{'Stdout'}           = undef;
	$h->{'Stderr'}           = undef;


	$h->{'Cmd'} = "globus-job-run $dest/jobmanager-fork /bin/sh -c \"$cmd\"";

	$c .= "$cmd; echo RemoteExitStatus \\\$?; echo MARK 1>&2; ";

	push @a, $h;
      }

    $c .= "\" >> $outf 2>> $errf";

    my $rc = system($c);

    foreach my $i (0 .. $#a)
      {
	$a[$i]{'ExitStatus'} = $rc;
      }

    $outfh = new IO::File;

    if ( $outfh->open("< $outf") )
      {
	# Remote command exit status lives in stdout file; sort that out
	my $testnum = 0;
	my @stdout;

	while ( my $line = $outfh->getline )
	  {
	        chomp($line);
		    if ( $line =~ /^RemoteExitStatus/ )
		      {
			$a[$testnum]{'Stdout'} = join "\n", @stdout;
			($a[$testnum]{'RemoteExitStatus'} = $line) =~ 
			      s/^RemoteExitStatus\s+//;
			splice @stdout;
			$testnum++;
		      }
		    else
		      {
			push @stdout, $line;
			    }
	      }
      }

    unlink $outf;

    $errfh = new IO::File;

    if ( $errfh->open("< $errf") )
      {
	my $testnum = 0;
	my @stderr;

	while ( my $line = $errfh->getline )
	  {
	        chomp($line);
		    if ( $line =~ /^MARK/ )
		      {
			$a[$testnum]{'Stderr'} = join "\n", @stderr;
			splice @stderr;
			$testnum++;
		      }
		    else
		      {
			push @stderr, $line;
			    }
	      }
      }

    unlink $errf;

    return \@a;
  }


#
# hashify_xinetd_services
#
# Return a hash reference representing xinetd-style services contained
# in the given file.
#
sub hashify_xinetd_services($)
			    {
    my $f = shift;

    if ( ! defined $f ) { return undef; }

    my %h;

    my $fh = new IO::File;

    if ( $fh->open("< $f") )
      {
	my $svcname = undef;

	while ( my $line = $fh->getline )
	  {
	        chomp($line);
		    $line =~ s/^\s+//;
		    $line =~ s/\s+$//;

		if    ( ! defined $line || length($line) == 0 ) { next;}
		elsif ( $line =~ m/^\#/ ) { next; }
		    elsif ( $line =~ m/defaults\s*/ )
		      {
			$svcname = 'defaults';
			next;
		      }
		    elsif ( $line =~ m/\}/ )
		      {
			$svcname = undef;
			next;
		      }
		    elsif ( $line =~ m/^service\s+.*$/ )
		      {
			$svcname = $line;
			$svcname =~ s/^service\s+(\S+)/$1/;
			$h{$svcname} = undef;
			next;
		      }
		    elsif ( $line =~ m/\{/ )
		      {

			my %svctmp;

			while ( $line !~ m/^\}/ )
			  {
			        $line = $fh->getline;
				    chomp($line);
				    $line =~ s/^\s+//;
				    $line =~ s/\s+$//;

				if    ( ! defined $line || length($line) == 0 ) { next;}
				elsif ( $line =~ m/^\#/ ) { next; }
				elsif ( $line =~ m/^\}/ ) { last; }

				    my $key = $line;
				    $key =~ s/^(\S+)\s+.*/$1/;
				    my $op  = $line;
				    $op  =~ s/^\S+\s+(\S+)\s+.*/$1/;
				    my $val = $line;
				    $val =~ s/^\S+\s+\S+\s+(.*)$/$1/;

				    if ( $op eq '=' )
				      {
					$svctmp{$key} = $val;
				      }
				    elsif ( $op eq '+=' )
				      {
					$svctmp{$key} .= ' ' . $val;
				      }
			      }

			my $protocol = 'tcp';
			if ( exists $svctmp{'protocol'} )
			  {
			        $protocol = $svctmp{'protocol'};
				    delete $svctmp{'protocol'};
			      }

			foreach my $key ( keys %svctmp )
			  {
			        $h{$svcname}{$protocol}{$key} = $svctmp{$key};
			      }
		      }
	      }
      }

    # Copy the defaults into other service descriptions
    # XXX 'tcp' for 'defaults' is a hack - FIXME XXX
    if ( exists $h{'defaults'}{'tcp'} )
      {
	foreach my $key ( keys %{$h{'defaults'}{'tcp'}} )
	  {
	        foreach my $svc (keys %h)
		  {
		    if ( $svc eq 'defaults' ) { next; }

		    foreach my $prot (keys %{$h{$svc}})
		      {
			    if ( ! defined $h{$svc}{$prot}{$key} )
			      {
				$h{$svc}{$prot}{$key} = $h{'defaults'}{'tcp'}{$key};
			      }
			  }
		  }
	      }

	undef $h{'defaults'};
      }

    return \%h;
  }


#
# hashify_inetd_services
#
# Return a hash reference representing inetd-style services contained
# in the given file.
#
sub hashify_inetd_services($)
			   {
    my $f = shift;

    if ( ! defined $f ) { return undef; }

    my %h;

    my $fh = new IO::File;

    if ( $fh->open("< $f") )
      {
	while ( my $line = $fh->getline )
	  {
	        chomp($line);
		    $line =~ s/^\s+//;
		    $line =~ s/\s+$//;

		if    ( ! defined $line || length($line) == 0 ) { next;}
		elsif ( $line =~ m/^\#/ ) { next; }
	      }
      }

    return \%h;
  }


#
# hashify_inet_conf
#
# Return a hash representing the given Internet Services Daemon 
# configuration file.
#
sub hashify_inet_conf($$)
  {
    my ($f_style, $f_local) = @_;

    if ( ! defined $f_style ) { return; }
    if ( ! defined $f_local )  { return; }

    my ($base, $path) = fileparse($f_style);

    my $href = undef;
    if ( $base eq "xinetd.conf" )
      {
	$href = hashify_xinetd_services($f_local);
      }
    elsif ( $base eq "inetd.conf" )
      {
	$href = hashify_inetd_services($f_local);
      }

    return $href;
  }


#
# hashify_grid_mapfile
#
# Return a hash reference representing the contents of a grid-mapfile.
#
sub hashify_grid_mapfile($)
			 {
    my $f = shift;

    if ( ! defined $f || ! -f $f ) { return undef; }

    open(GMAP, "< $f");

    my %h;

    while ( my $line = <GMAP> )
      {
	chomp($line);
	if ( ! defined $line || length($line) == 0 ) { next; }
	if ( $line =~ m/^\#/ ) { next; }

	$line =~ s/^\s+//;
	$line =~ s/\s+$//;

	my $user = $line;
	$user =~ s/^.*\s+(\S+)$/$1/;
	my $dn = $line;
	$dn =~ s/\s+$user$//;
	$dn =~ s/^\"//;
	$dn =~ s/\"$//;

	$h{$dn} = $user;
      }

    close(GMAP);

    return \%h;
  }


#
# hashify_ganglia_conf
#
# Return a hash reference representing the contents of a gmond.conf/gmetad.conf
#
sub hashify_ganglia_conf($)
			 {
    my $f = shift;

    if ( ! defined $f || ! -f $f ) { return undef; }

    open(GMAP, "< $f");

    my %h;

    while ( my $line = <GMAP> )
      {
	chomp($line);
	$line =~ s/^\s+//;
	$line =~ s/\s+$//;
	if ( ! defined $line || length($line) == 0 ) { next; }
	if ( $line =~ m/^\#/ ) { next; }

	my $key = undef;
	my $val = undef;

	my @tokens = split /\s+/, $line;

	if ( scalar(@tokens) > 0 ) { $key = $tokens[0]; splice @tokens, 0, 1; }
	if ( scalar(@tokens) > 0 ) { $val = join " ", @tokens; }

	if ( defined $key ) { $h{$key} = $val; }
      }

    close(GMAP);

    return \%h;
  }


#
# obtain_remote_file
#
# Get remote file to localhost; return local filename.
#
sub obtain_remote_file($$$)
		       {
    my ($tests, $opts, $rfile)  = @_;

    if ( ! defined $tests ) { return undef; }
    if ( ! defined $rfile ) { return undef; }

    # Prefer gsiftp (it is faster)
    my $use_gsiftp = undef;

    # determine if gsiftp was reliable or not
    if ( $tests->{'STATUS'}{'gsiftp_service'}     eq SUCCESS ||
	  $tests->{'STATUS'}{'gsiftp_local_out'}   eq SUCCESS ||
	  $tests->{'STATUS'}{'gsiftp_local_in'}    eq SUCCESS ||
	  $tests->{'STATUS'}{'gsiftp_local_diffs'} eq SUCCESS )
      {
	$use_gsiftp = 1;
      }
    
    my $host = undef;
    my $port = 2811;

    my $lfile = undef;

#    my $l = new File::Temp(
#			      TEMPLATE => 'remoteXXXXXX',
#			      DIR      => $opts->{'TMPDIR'}, 
#			      UNLINK   => 0,
#			      );

#    my $ltmp = $l->filename;
    use File::Temp qw(tempfile unlink0 );
    use vars qw($l $ltmp);
    
    my $template = 'remoteXXXXXX';
    my $dir      = '/tmp';
    ($l, $ltmp) = tempfile($template, DIR => $dir);


    if ( defined $use_gsiftp )
      {
	if ( exists $opts->{'GSIFTP_HOST'} )
	  {
	        $host = $opts->{'GSIFTP_HOST'};
	      }
	elsif ( exists $opts->{'GATEKEEPER_HOST'} )
	  {
	        $host = $opts->{'GATEKEEPER_HOST'};
	      }

	if ( exists $opts->{'GSIFTP_PORT'} )
	  {
	        $port = $opts->{'GSIFTP_PORT'};
	      }

	if ( defined $host )
	  {
	        my $remote = $host;
		if ( defined $port ) { $remote .= ":$port" }

		    my $cmd = "globus-url-copy gsiftp://$remote/$rfile file:$ltmp";
		    if ( runcmd2($cmd, $opts) == 0 )
		      {
			$lfile = $ltmp
		      }
	      }
      }

    if ( ! defined $lfile )
      {
	my @cmds;
	push @cmds, "cat $rfile";
	my $a = run_remote_cmds(\@cmds, $opts);

	if ( scalar(@$a) != 0 )
	  {
	        if ( $a->[0]{'ExitStatus'} == 0 && 
		      $a->[0]{'RemoteExitStatus'} == 0 )
		  {
		    my @f = split "\n", $a->[0]{'Stdout'};

		    if ( scalar(@f) > 0 )
		      {
			    for ( my $i = 0 ; $i < scalar(@f) ; $i++ )
			      {
				print $l "$f[$i]\n";
			      }
			  }

		    $lfile = $ltmp
		  }
	      }
      }

    return $lfile;
  }


#
# hashify_remote_inet_conf
#
# Return a hash reference representing the remote site's 
# Internet Services Daemon configuration file.
#
# Prefer xinetd to inetd.
#
# Fills: $opts->{LOCAL_FILES}{INETD_CONF}
#        $opts->{REMOTE_FILES}{INETD_CONF}
#        $opts->{FILE_HASH}{INETD_CONF}
#
sub hashify_remote_inet_conf($$)
  {
    my ($tests, $opts) = @_;

    if ( ! defined $tests ) { return undef; }
    if ( ! defined $opts  ) { return undef; }

    $opts->{'REMOTE_FILES'}{'INETD_CONF'} = '/etc/xinetd.conf';
    $opts->{'LOCAL_FILES'}{'INETD_CONF'} = 
      obtain_remote_file($tests, $opts, 
			    $opts->{'REMOTE_FILES'}{'INETD_CONF'});

    if ( ! defined $opts->{'LOCAL_FILES'}{'INETD_CONF'} )
      {
	$opts->{'REMOTE_FILES'}{'INETD_CONF'} = '/etc/inetd.conf';
	$opts->{'LOCAL_FILES'}{'INETD_CONF'} = 
	      obtain_remote_file($tests, $opts, 
				        $opts->{'REMOTE_FILES'}{'INETD_CONF'});
      }
    else
      {
	# Handle xinetd.conf's 'includedir' option
	# If found, get all the files in remote includedir
	# Ignore filenames containing '.' and ending in '~'
	#   as per xinetd.conf(5)
	my $fh = new IO::File;

	while ( 1 )
	  {
	        if ( $fh->open("< $opts->{LOCAL_FILES}{INETD_CONF}") )
		  {
		    my $includedir = undef;

		    while ( my $line = $fh->getline )
		      {
			    chomp($line);
			        if ( $line =~ m/\s*includedir\s+/ )
				  {
				    $includedir = $line;
				    $includedir =~ s/^\s+//;
				    $includedir =~ s/^\S+\s+(\S+)\s*$/$1/;
				    last;
				  }
			  }

		    $fh->close;
		    if ( ! defined $includedir ) { last; }

		    my @cmds;
		    push @cmds, "for i in \\`ls -l /etc/xinetd.d | grep -v drwx | awk \'{print \\\$9}\' | grep -v -e \'~\$\' | grep -v -e \'\\\.\'\\`; do cat $includedir/\\\$i ; done";
		    my $a = run_remote_cmds(\@cmds, $opts);

		    if ( scalar(@$a) != 0 )
		      {
			    if ( $a->[0]{'ExitStatus'} == 0 && 
				  $a->[0]{'RemoteExitStatus'} == 0 )
			      {
				my @replac = split "\n", $a->[0]{'Stdout'};

    use File::Temp qw(tempfile unlink0 );
    use vars qw($fh $fnew);
    
    my $template = 'remoteXXXXXX';
    my $dir      = '/tmp';
    ($fh, $fnew) = tempfile($template, DIR => $dir);


#				my $fnew = new File::Temp(
#							    TEMPLATE => 'remoteXXXXXX',
#							    DIR      => $opts->{'TMPDIR'}, 
#							    UNLINK   => 0,
#							    );

				if ( $fh->open("< $opts->{LOCAL_FILES}{INETD_CONF}") )
				  {
				        while ( my $line = $fh->getline )
					  {
					    chomp($line);
					    if ( $line =~ m/\s*includedir\s+$includedir/ )
					      {
						    for ( my $i = 0 ; $i < scalar(@replac); 
							    $i++ )
						      {
							print $fnew "$replac[$i]\n";
						      }
						  }
					    else
					      {
						    print $fnew "$line\n";
						    }
					  }

					    $fh->close;
					    unlink $opts->{'LOCAL_FILES'}{'INETD_CONF'};
					    $opts->{'LOCAL_FILES'}{'INETD_CONF'} = 
					      $fnew->filename;
				      }
			      }
			  }
		  }
	      }
    }

    if ( ! defined $opts->{'LOCAL_FILES'}{'INETD_CONF'} )
      {
	return undef;
      }

    return hashify_inet_conf($opts->{'REMOTE_FILES'}{'INETD_CONF'}, 
			          $opts->{'LOCAL_FILES'}{'INETD_CONF'});
  }


#
# hashify_internet_services
#
# Return a hash reference representing internet services contained
# in the given /etc/services-like file.
#
sub hashify_internet_services($)
			      {
    my $f = shift;

    if ( ! defined $f ) { return undef; }
    if ( ! -f $f )      { return undef; }

    my %h;

    my $fh = new IO::File;

    if ( $fh->open("< $f") )
      {
	while ( my $line = $fh->getline )
	  {
	        chomp($line);
		    $line =~ s/^\s+//;
		    $line =~ s/\s+$//;

		if    ( ! defined $line || length($line) == 0 ) { next;}
		elsif ( $line =~ m/^\#/ ) { next; }

		    my $svc = $line;
		    $svc =~ s/^(\S+)\s+\d+\/\S+.*/$1/;
		    my $port = $line;
		    $port =~ s/^\S+\s+(\d+)\/\S+.*/$1/;
		    my $protocol = $line;
		    $protocol =~ s/^\S+\s+\d+\/(\S+).*/$1/;

		    $h{$svc}{$protocol}{'PORT'}    = $port;

		    my $aliases = $line;
		    $aliases =~ s/^\S+\s+\d+\/\S+\s*//;
		    if ( length($aliases) > 0 )
		      {
			$aliases =~ s/\#.*$//;
			$h{$svc}{$protocol}{'ALIASES'} = $aliases;
		      }
	      }
      }

    return \%h;
  }


#
# hashify_remote_internet_services
#
# Return a hash reference representing the remote site's /etc/services.
#
sub hashify_remote_internet_services($$)
  {
    my ($tests, $opts) = @_;

    $opts->{'LOCAL_FILES'}{'SERVICES'} = 
      obtain_remote_file($tests, $opts, '/etc/services');

    return hashify_internet_services($opts->{'LOCAL_FILES'}{'SERVICES'});
  }


#
# hashify_gatekeeper_conf
#
# Return a hash reference representing a globus gatekeeper conf file.
#
sub hashify_gatekeeper_conf($)
			    {
    my $f = shift;

    if ( ! defined $f ) { return undef; }
    if ( ! -f $f )      { return undef; }

    my %h;

    my $fh = new IO::File;

    if ( $fh->open("< $f") )
      {
	while ( my $line = $fh->getline )
	  {
	        chomp($line);
		    $line =~ s/^\s+//;
		    $line =~ s/\s+$//;

		if    ( ! defined $line || length($line) == 0 ) { next;}
		elsif ( $line =~ m/^\#/ ) { next; }

		    my $key = $line;
		    $key =~ s/^(\S+)\s+.*$/$1/;
		    my $val = undef;
		    if ( $line =~ /^(\S+)\s+\S+.*$/ )
		      {
			$val = $line;
			$val =~ s/^\S+\s+(.*)$/$1/;
		      }

		    $h{$key} = $val;
	      }
      }

    return \%h;
  }


#
# hashify_jobmanager
#
# Return a hash reference representing a jobmanager.
#
sub hashify_jobmanager($)
		       {
    my $f = shift;

    if ( ! defined $f ) { return undef; }
    if ( ! -f $f )      { return undef; }

    my %h;

    my $fh = new IO::File;

    if ( $fh->open("< $f") )
      {
	while ( my $line = $fh->getline )
	  {
	        chomp($line);
		    $line =~ s/^\s+//;
		    $line =~ s/\s+$//;

		if    ( ! defined $line || length($line) == 0 ) { next;}
		elsif ( $line =~ m/^\#/ ) { next; }

		    # Everything after the 4th argument is an argument
		    # supplied to the globus-job-manager - this is what we care about
		    my @tokens = split /\s+/, $line;

		    my $num_args = 4;
		if ( scalar(@tokens) < $num_args ) { $num_args = scalar(@tokens); }

		    splice @tokens, 0, $num_args;

		    # globus-job-manager options begin with a dash
		    # and the options may have args.

		    # XXX - FIXME - the following is easily fooled, but should 
		    # XXX - FIXME - get the job done at the level we care.
		    my $opt = undef;
		    foreach my $token (@tokens)
		      {
			if ( $token =~ /^\-/ )
			  {
			    if ( defined $opt ) { $h{$opt} = undef; }
			        $opt = $token;
			  }
			else
			  {
			    if ( defined $opt ) { $h{$opt} = $token; }
			        $opt = undef;
			    }
		      }

		if ( defined $opt ) { $h{$opt} = undef; }
	      }
      }

    return \%h;
  }


#
# get_globus_sh_tools_vars_var
#
# Return the value of the requested variable from globus-sh-tools-vars.sh.
#
sub get_globus_sh_tools_vars_var($$)
  {
    my ($opts, $var) = @_;

    if ( ! defined $opts )  { return undef; }
    if ( ! defined $var )   { return undef; }

    my $cmd = "/bin/sh -c '. $opts->{LOCAL_FILES}{GLOBUS_SH_TOOLS_VARS} > /dev/null 2>&1 ; echo \$$var'";
    my $d = `$cmd`;

    if ( defined $d && length($d) > 0 ) { chomp($d); }
    if ( defined $d && length($d) == 0 ) { $d = undef; }

    return $d;
  }


#
# get_grid3_info_var
#
# Return the value of the requested variable from grid3-info.conf.
#
sub get_grid3_info_var($$$)
		       {
    my ($tests, $opts, $var) = @_;

    if ( ! defined $tests ) { return undef; }
    if ( ! defined $opts )  { return undef; }
    if ( ! defined $var )   { return undef; }

    my $cmd = "/bin/sh -c '. $opts->{LOCAL_FILES}{GRID3_INFO_CONF} > /dev/null 2>&1 ; echo \$$var'";
    my $d = `$cmd`;

    if ( defined $d && length($d) > 0 ) { chomp($d); }
    if ( defined $d && length($d) == 0 ) { $d = undef; }

    return $d;
  }


#
# get_grid3_setup_var
#
# Return the value of the requested variable from $GRID3/setup.sh.
#
sub get_grid3_setup_var($$$)
			{
    my ($tests, $opts, $var) = @_;

    if ( ! defined $tests ) { return undef; }
    if ( ! defined $opts )  { return undef; }
    if ( ! defined $var )   { return undef; }

    my $cmd = "/bin/sh -c '. $opts->{LOCAL_FILES}{GRID3_SETUP_SH} > /dev/null 2>&1 ; echo \$$var'";
    my $d = `$cmd`;

    if ( defined $d && length($d) > 0 ) { chomp($d); }
    if ( defined $d && length($d) == 0 ) { $d = undef; }

    return $d;
  }


#
# get_grid_info_var
#
# Return the value of the requested variable from grid-info.conf.
#
sub get_grid_info_var($$$)
		      {
    my ($tests, $opts, $var) = @_;

    if ( ! defined $tests ) { return undef; }
    if ( ! defined $opts )  { return undef; }
    if ( ! defined $var )   { return undef; }

    my $cmd = "/bin/sh -c '. $opts->{LOCAL_FILES}{GRID_INFO_CONF} > /dev/null 2>&1 ; echo \$$var'";
    my $d = `$cmd`;

    if ( defined $d && length($d) > 0 ) { chomp($d); }
    if ( defined $d && length($d) == 0 ) { $d = undef; }

    return $d;
  }


#
# check_prereqs
#
# Return 0 if prereq tests all passed, non-zero if not.
#
sub check_prereqs($$)
  {
    my ($tests, $opts)  = @_;
    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }
    
    if ( defined $tests->{'PREREQS'}{$t} )
      {
	my @prereqs;

	if ( $tests->{'PREREQS'}{$t} =~ m/,/ )
	  {
	        @prereqs = split ',', $tests->{'PREREQS'}{$t};
	      }
	else
	  {
	        push @prereqs, $tests->{'PREREQS'}{$t};
		}

	for ( my $i = 0; $i < scalar(@prereqs); $i++ )
	  {
	        if ( ! defined $tests->{'STATUS'}{$prereqs[$i]} ||
		      $tests->{'STATUS'}{$prereqs[$i]} eq FAILURE ||
		      $tests->{'STATUS'}{$prereqs[$i]} eq UNTESTED )
		  {
		    return 1;
		  }
	      }
      }

    return 0;
  }


#
# pre_check
#
# Generic pre-test routine.  Just print out "Checking...".
#
sub pre_check($$)
  {
    my ($tests, $opts)  = @_;
    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }
    
    if ( defined $tests->{'DESCRIP'}{$t} )
      {
	print 'Checking ' . $tests->{'DESCRIP'}{$t} . ': ';
      }
    else
      {
	print "Running test \"$t\": ";
    }
  }


#
# post_check
#
# Generic post-test routine.  Just print out the result.
#
sub post_check($$)
  {
    my ($tests, $opts)  = @_;
    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }
    
    if ( defined $tests->{'RESULTS'}{$t} )
      {
	$tests->{'RESULTS'}{$t} =~ s/\n$//;
	print $tests->{'RESULTS'}{$t} . "\n";
      }
    elsif ( defined $tests->{'STATUS'}{$t} )
      {
	$tests->{'STATUS'}{$t} =~ s/\n$//;
	print $tests->{'STATUS'}{$t} . "\n";

	if ( $tests->{'STATUS'}{$t} eq SUCCESS )
	  {
	        if ( defined $tests->{'ACTONPASS'}{$t} )
		  {
		    &$tests->{'ACTONPASS'}{$t}($tests, $opts);
		  }
	      }
	elsif ( $tests->{'STATUS'}{$t} eq FAILURE )
	  {
	        if ( defined $tests->{'ACTONFAIL'}{$t} )
		  {
		    &{$tests->{'ACTONFAIL'}{$t}}($tests, $opts);
		  }
	      }
      }
    else
      {
	print UNKNOWN . "\n";
    }
  }


#
# print_msgs_and_exit
#
# Print any messages corresponding to the test and exit.
#
sub print_msgs_and_exit($$)
  {
    my ($tests, $opts)  = @_;

    if ( defined $tests->{'MESGS'}{$opts->{'TESTNAME'}} )
      {
	print $tests->{'MESGS'}{$opts->{'TESTNAME'}} . "\n";
      }

    exit;
  }


#
# check_path
#
# Make sure all needed programs are in the user's PATH.
#
sub check_path($$)
  {
    my ($tests, $opts)  = @_;
    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }

    $tests->{'MESGS'}{$t}   = undef;
    $tests->{'STATUS'}{$t} = UNTESTED;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    my @exes = (
		'grid-proxy-info',
		'globusrun',
		'globus-job-run',
		'globus-url-copy',
		);

    my @not_found;

    foreach my $exe (@exes)
      {
	if ( system("/bin/sh -c \"type $exe > /dev/null 2>&1\"") != 0 )
	  {
	        push @not_found, $exe;
	      }
      }

    if ( scalar(@not_found) == 0 )
      {
	$tests->{'MESGS'}{$t}   = 'All prerequisites needed for testing were found.';
	$tests->{'STATUS'}{$t} = SUCCESS;
      }
    else
      {
	$tests->{'MESGS'}{$t} = 'Error: Can\'t find prerequisites: ' . 
	      join ',',@not_found;
	$tests->{'STATUS'}{$t} = FAILURE;
    }
  }

#
# check_proxy
#
# Check for a valid user proxy.
#
sub check_proxy($$)
  {
    my ($tests, $opts)  = @_;
    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }
    my $v = undef;
    if ( exists $opts->{'VERBOSE'} )
      {
	$v = $opts->{'VERBOSE'}
      }

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    if ( runcmd("grid-proxy-info -exists", $v) == 0 )
      {
	$tests->{'STATUS'}{$t} = SUCCESS;
      }
    else
      {
	$tests->{'MESGS'}{$t}   = 'Error: No proxy exists for ' . 
	      $opts->{'USER'} . '@' . $opts->{'LOCALHOST'};
	$tests->{'STATUS'}{$t} = FAILURE;
    }
  }


#
# check_remote_host_reachable
#
# Check if we can see the remote host.
#
sub check_remote_host_reachable
  {
    my ($tests, $opts)  = @_;

    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }

    my $host = undef;
    if ( exists $opts->{'GATEKEEPER_HOST'} )
      {
	$host = $opts->{'GATEKEEPER_HOST'};
      }

    my $port = undef;
    if ( exists $opts->{'GATEKEEPER_PORT'} )
      {
	$port = $opts->{'GATEKEEPER_PORT'};
      }

    my $timeout = 10;
    if ( exists $opts->{'TIMEOUT'} )
      {
	$timeout = $opts->{'TIMEOUT'};
      }

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    if ( ! defined $host )
      {
	$tests->{'MESGS'}{$t} = 'Host not supplied';
	return;
      }

    if ( ! defined $port )
      {
	$tests->{'MESGS'}{$t} = 'Port not supplied';
	return;
      }


    # Try a few ports
    my @ports = (22, $port, 23);

    # Try to connect via tcp to $host:$port
    for ( my $i = 0; $i < scalar(@ports); $i++ )
      {
	my $sock = IO::Socket::INET->new(Proto    => "tcp",
					  PeerAddr => "$host:$ports[$i]",
					  Timeout  => $timeout,
					  );
	if ( defined $sock )
	  {
	        $sock->autoflush(1);
		    close($sock);
		    $tests->{'STATUS'}{$t} = SUCCESS;
		    return;
	      }
      }

    $tests->{'STATUS'}{$t} = FAILURE;
    $tests->{'MESGS'}{$opts->{'TESTNAME'}} = NO . "\n  Can't connect to $host";
}


#
# check_gatekeeper_service
#
# Check to see if remote gatekeeper is running.
#
sub check_gatekeeper_service($$)
  {
    my ($tests, $opts)  = @_;

    my $host = undef;
    if ( exists $opts->{'GATEKEEPER_HOST'} )
      {
	$host = $opts->{'GATEKEEPER_HOST'};
      }

    my $port = 2119;
    if ( exists $opts->{'GATEKEEPER_PORT'} )
      {
	$port = $opts->{'GATEKEEPER_PORT'};
      }

    my $timeout = 10;
    if ( exists $opts->{'TIMEOUT'} )
      {
	$timeout = $opts->{'TIMEOUT'};
      }

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    if ( ! defined $host )
      {
	$tests->{'MESGS'}{$opts->{'TESTNAME'}} = 'Host not supplied';
	return;
      }

    my $sock = new IO::Socket::INET (Timeout  => $timeout,
				          PeerAddr => "$host:$port",
				          Proto    => "tcp",
				          );
 
    if ( ! defined $sock || ! defined $sock->connected )
      {
	$tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;
	$tests->{'MESGS'}{$opts->{'TESTNAME'}} = NO . 
	      "\n  Can't connect to $host:$port/tcp";
	return;
      }

    $sock->autoflush(1);
    close($sock);

    $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;
    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = YES . "; port $port";
  }


#
# check_auth
#
# Check if the user can authenticate at remote site.
#
sub check_auth($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    my $v = undef;
    if ( exists $opts->{'VERBOSE'} )
      {
	$v = $opts->{'VERBOSE'}
      }

    my $host = undef;
    if ( exists $opts->{'GATEKEEPER_HOST'} )
      {
	$host = $opts->{'GATEKEEPER_HOST'};
      }

    my $port = 2119;
    if ( exists $opts->{'GATEKEEPER_PORT'} )
      {
	$port = $opts->{'GATEKEEPER_PORT'};
      }

    if ( ! defined $host )
      {
	$tests->{'MESGS'}{$opts->{'TESTNAME'}} = 'Host not supplied';
	return;
      }

    my $dest = $host;
    if ( defined $port ) { $dest .= ":$port" }

    if ( runcmd("globusrun -a -r $dest", $v) == 0 )
      {
	$tests->{'STATUS'}{$opts->{'TESTNAME'}} = SUCCESS;
      }
    else
      {
	$tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;
    }
  }


#
# check_hello_world
#
# Check to see if we can run a trivial job via the fork manager.
#
sub check_hello_world($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    my $str = 'Hello, World';

    my @cmds;
    push @cmds, "echo \'$str\'";

    my $a = run_remote_cmds(\@cmds, $opts);

    if ( scalar(@$a) != 0 )
      {
	if ( $a->[0]{'ExitStatus'} == 0 && $a->[0]{'RemoteExitStatus'} == 0 )
	  {
	        $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;
	      }
      }
  }


#
# check_uptime
#
# Check uptime of remote gatekeeper.
#
sub check_uptime($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    my @cmds;
    push @cmds, "uptime";

    my $a = run_remote_cmds(\@cmds, $opts);

    if ( scalar(@$a) != 0 )
      {
	if ( $a->[0]{'ExitStatus'} == 0 && $a->[0]{'RemoteExitStatus'} == 0 )
	  {
	        $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = SUCCESS . "\n  " . 
		  $a->[0]{'Stdout'};
		    $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;
	      }
      }
  }


#
# check_hostcert
#
# Obtain remote hostcert.pem and check validity.
#
sub check_hostcert($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    my $globus_location = $tests->{'RESULTS'}{'globus_location'};
    my @cmds;
    push @cmds, "openssl=\\\"openssl\\\"; if test -x $globus_location/bin/openssl; then openssl=\\\"$globus_location/bin/openssl\\\"; if test -z \\\$LD_LIBRARY_PATH; then LD_LIBRARY_PATH=\\\"$globus_location/lib\\\"; else LD_LIBRARY_PATH=\\\"$globus_location/lib:\\\$LD_LIBRARY_PATH\\\"; fi; export LD_LIBRARY_PATH; fi; \\\$openssl x509 -in /etc/grid-security/hostcert.pem -noout -text | grep -e \'Not\\ \\+After\'";

    my $a = run_remote_cmds(\@cmds, $opts);

    if ( scalar(@$a) != 0 )
      {
	if ( $a->[0]{'ExitStatus'} == 0 && $a->[0]{'RemoteExitStatus'} == 0 )
	  {
	        (my $r = $a->[0]{'Stdout'}) =~ s/^\s*Not\s+After\s+\:\s+//;

		    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = $r;
		    $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;
	      }
      }
  }


#
# check_services
#
# Get a copy of the remote /etc/services.
#
sub check_services($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    if ( ! exists $opts->{'FILE_HASH'}{'SERVICES'} )
      {
	if ( ! exists $opts->{'LOCAL_FILES'}{'SERVICES'} )
	  {
	        $opts->{'FILE_HASH'}{'SERVICES'} = 
		  hashify_remote_internet_services($tests, $opts);
	      }
	else
	  {
	        $opts->{'FILE_HASH'}{'SERVICES'} = 
		  hashify_internet_services($opts->{'LOCAL_FILES'}{'SERVICES'});
		}
      }

    if ( ! exists  $opts->{'FILE_HASH'}{'SERVICES'} || 
	 ! defined $opts->{'FILE_HASH'}{'SERVICES'} ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = SUCCESS;
  }


#
# check_inetd
#
# Analyze the config of the remote INET super-server.
#
sub check_inetd($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    # Find the service config for this service name

    if ( ! exists $opts->{'FILE_HASH'}{'INETD_CONF'} )
      {
	if ( ! exists $opts->{'LOCAL_FILES'}{'INETD_CONF'} )
	  {
	        $opts->{'FILE_HASH'}{'INETD_CONF'} = 
		  hashify_remote_inet_conf($tests, $opts);
	      }
	else
	  {
	        $opts->{'FILE_HASH'}{'INETD_CONF'} = 
		  hashify_inet_conf($opts->{'REMOTE_FILES'}{'INETD_CONF'}, 
				      $opts->{'LOCAL_FILES'}{'INETD_CONF'});
		}
      }

    if ( ! exists  $opts->{'FILE_HASH'}{'INETD_CONF'} ||
	 ! defined $opts->{'FILE_HASH'}{'INETD_CONF'} ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = SUCCESS;
  }


#
# check_gsiftp_service
#
# Check to see if remote gsiftp server is running.
#
sub check_gsiftp_service($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    my $host = undef;
    if ( exists $opts->{'GSIFTP_HOST'} )
      {
	$host = $opts->{'GSIFTP_HOST'};
      }
    elsif ( exists $opts->{'GATEKEEPER_HOST'} )
      {
	$host = $opts->{'GATEKEEPER_HOST'};
      }

    if ( ! defined $host )
      {
	$tests->{'MESGS'}{$opts->{'TESTNAME'}} = 'Host not supplied';
	return;
      }

    # If gsiftp port is supplied, use it.
    #
    # Otherwise, 
    #
    # Figure out the service name corresponding to gridftp from the inetd.conf
    # Assume the 'correct' gsiftp has the same GLOBUS_LOCATION
    my $port = undef;
    if ( exists $opts->{'GSIFTP_PORT'} )
      {
	$port = $opts->{'GSIFTP_PORT'};
      }
    else
      {
	my $globus_location = $tests->{'RESULTS'}{'globus_location'};

	my $svcname = undef;

	if ( exists  $opts->{'FILE_HASH'}{'INETD_CONF'} &&
	          defined $opts->{'FILE_HASH'}{'INETD_CONF'} )
	  {

	        my $href = $opts->{'FILE_HASH'}{'INETD_CONF'};

		    # Assume tcp
		    foreach my $svc ( keys %$href )
		      {
			if ( ! exists $href->{$svc}{'tcp'}{'server'} ) { next; }

			if ( $href->{$svc}{'tcp'}{'server'} =~ m/in\.ftpd/ )
			  {
			        if ( exists $href->{$svc}{'tcp'}{'env'} &&
				      ($href->{$svc}{'tcp'}{'env'} =~ 
				         m/LD_LIBRARY_PATH=$globus_location/) )
				  {
				    $svcname = $svc;
				  }
				    elsif ( exists $href->{$svc}{'tcp'}{'server_args'} &&
					        ($href->{$svc}{'tcp'}{'server_args'} =~ 
						      m/-G\s+$globus_location/) )
				      {
					$svcname = $svc;
				      }
				    elsif ( exists $href->{$svc}{'tcp'}{'server'} )
				      {
					my $name1 = $href->{$svc}{'tcp'}{'server'};
					my $name2 = "$globus_location/sbin/in.ftpd";

					$name1 =~ s/\/+/\//g;
					$name2 =~ s/\/+/\//g;

					if ( $name1 eq $name2 ) { $svcname = $svc; }
				      }

				if ( defined $svcname ) { last; }
			      }
		      }
	      }

	# If we've found the service name, 
	# map it to the port given in /etc/services
	if ( defined $svcname )
	  {
	        $opts->{'SVCNAMES'}{'gsiftp'} = $svcname;

		    if ( ! exists $opts->{'FILE_HASH'}{'SERVICES'} )
		      {
			if ( ! exists $opts->{'LOCAL_FILES'}{'SERVICES'} )
			  {
			        $opts->{'FILE_HASH'}{'SERVICES'} = 
				  hashify_remote_internet_services($tests, $opts);
			      }
			else
			  {
			        $opts->{'FILE_HASH'}{'SERVICES'} = 
				  hashify_internet_services($opts->{'LOCAL_FILES'}{'SERVICES'});
				}
		      }

		    # XXX FIXME XXX protocol could be an /etc/protocols alias
		    if ( exists $opts->{'FILE_HASH'}{'SERVICES'}{$svcname}{'tcp'}{'PORT'} )
		      {
			$port = $opts->{'FILE_HASH'}{'SERVICES'}{$svcname}{'tcp'}{'PORT'};
		      }
	      }
    }

    # Last resort - guess...
    if ( ! defined $port ) { $port = 2811; }

    my $timeout = 10;
    if ( exists $opts->{'TIMEOUT'} )
      {
	$timeout = $opts->{'TIMEOUT'};
      }

    my $sock = new IO::Socket::INET (Timeout  => $timeout,
				          PeerAddr => "$host:$port",
				          Proto    => "tcp",
				          );

    if ( ! defined $sock || ! defined $sock->connected )
      {
	$tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;
	$tests->{'MESGS'}{$opts->{'TESTNAME'}} = NO . 
	      "\n  Can't connect to $host:$port/tcp";
	return;
      }

    $sock->autoflush(1);
    close($sock);

    $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;
    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = YES . "; port $port";
  }


#
# Check to see if remote MDS service is running.
#
sub check_mds_service($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    my $d = $tests->{'RESULTS'}{'globus_location'};

    $opts->{'LOCAL_FILES'}{'GRID_INFO_CONF'} = 
      obtain_remote_file($tests, $opts, "$d/etc/grid-info.conf");

    if ( ! defined $opts->{'LOCAL_FILES'}{'GRID_INFO_CONF'} ) { return; }

    my $h = get_grid_info_var($tests, $opts, 'GRID_INFO_HOST');
    my $p = get_grid_info_var($tests, $opts, 'GRID_INFO_PORT');

    if ( defined $h && defined $p )
      {

	my $ldap = Net::LDAP->new($h, 
				    port    => $p, 
				    timeout => 30, 
				    );
	if ( ! defined $ldap ) 
	  {
	        $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = 
		  NO . "; no LDAP server at $h:$p";
		    return;
	      }

	my $msg = $ldap->bind();

	if ( $msg->is_error($msg->code()) )
	  {
	        $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = 
		  NO . "; Unable to bind to LDAP server at $h:$p\n" .
		    $msg->error() . "\n"; ;
		    return;
	      }

	$tests->{'STATUS'}{$opts->{'TESTNAME'}} = SUCCESS;
	$tests->{'RESULTS'}{$opts->{'TESTNAME'}} = YES . '; port ' . $p;

	$opts->{'MDS_HOST'} = $h;
	$opts->{'MDS_PORT'} = $h;

	$msg = $ldap->unbind();
      }
  }


#
# check_gsiftp_local_out
#
# Check to see if we can use the local gridftp client to push a file to
# the remote site.
#
sub check_gsiftp_local_out($$)
  {
    my ($tests, $opts)  = @_;
    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }

    my $v = undef;
    if ( exists $opts->{'VERBOSE'} )
      {
	$v = $opts->{'VERBOSE'}
      }

    my $host = undef;
    if ( exists $opts->{'GSIFTP_HOST'} )
      {
	$host = $opts->{'GSIFTP_HOST'};
      }
    elsif ( exists $opts->{'GATEKEEPER_HOST'} )
      {
	$host = $opts->{'GATEKEEPER_HOST'};
      }

    my $port = 2811;
    if ( exists $opts->{'GSIFTP_PORT'} )
      {
	$port = $opts->{'GSIFTP_PORT'};
      }

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    if ( ! defined $host )
      {
	$tests->{'MESGS'}{$t} = 'Host not supplied';
	return;
      }

    my $dest = $host;
    if ( defined $port ) { $dest .= ":$port" }

    my $f = new File::Temp(
			      TEMPLATE => 'gsiftpXXXXXX',
			      DIR      => $opts->{'TMPDIR'}, 
			      SUFFIX   => '.local_out', 
			      UNLINK   => 0,
			      );

    print $f "local: " . $opts->{'LOCALHOST'} . " <-> $host\n";

    my $fname = $f->filename;

    my ($base, $path) = fileparse($fname);

    $opts->{'LOCAL_GSIFTP_OUTFILE'} = $fname;

    my $rfname = "/tmp/$base";

    $opts->{'REMOTE_GSIFTP_OUTFILE'} = $rfname;

    print " runcmd globus-url-copy file:$fname gsiftp://$dest/$rfname ";


    if ( runcmd("globus-url-copy file:$fname gsiftp://$dest/$rfname", $v) == 0 )
      {
	$tests->{'STATUS'}{$t} = SUCCESS;
      }
    else
      {
	$tests->{'STATUS'}{$t} = FAILURE;
    }
  }


#
# check_gsiftp_local_in
#
# Check to see if we can use the local gridftp client to pull a file from
# the remote site.
#
sub check_gsiftp_local_in($$)
  {
    my ($tests, $opts)  = @_;
    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $opts->{'GSIFTP_ONLY'} = 1;

    $opts->{'LOCAL_GSIFTP_INFILE'} = 
      obtain_remote_file($tests, $opts, $opts->{'REMOTE_GSIFTP_OUTFILE'});

    delete $opts->{'GSIFTP_ONLY'};

    if ( defined $opts->{'LOCAL_GSIFTP_INFILE'} )
      {
	$tests->{'STATUS'}{$t} = SUCCESS;
      }
    else
      {
	$tests->{'STATUS'}{$t} = FAILURE;
    }
  }


#
# check_gsiftp_local_diffs
#
# Verify there are no differences in files transferred with local gsiftp client
#
sub check_gsiftp_local_diffs($$)
  {
    my ($tests, $opts)  = @_;
    my $t = undef;
    if ( exists $opts->{'TESTNAME'} )
      {
	$t = $opts->{'TESTNAME'}
      }

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    my $f1 = undef;
    if ( exists $opts->{'LOCAL_GSIFTP_OUTFILE'} )
      {
	$f1 = $opts->{'LOCAL_GSIFTP_OUTFILE'};
      }

    my $f2 = undef;
    if ( exists $opts->{'LOCAL_GSIFTP_INFILE'} )
      {
	$f2 = $opts->{'LOCAL_GSIFTP_INFILE'};
      }

    if ( ! defined $f1 ) { return; }
    if ( ! defined $f2 ) { return; }
    if ( ! -e $f1 ) { return; }
    if ( ! -e $f2 ) { return; }

    if ( compare("$f1", "$f2") == 0 )
      {
	$tests->{'STATUS'}{$t} = SUCCESS;

	# cleanup
	unlink $opts->{'LOCAL_GSIFTP_OUTFILE'};
	unlink $opts->{'LOCAL_GSIFTP_INFILE'};

	my $host = undef;
	if ( exists $opts->{'GATEKEEPER_HOST'} )
	  {
	        $host = $opts->{'GATEKEEPER_HOST'};
	      }

	my $port = 2119;
	if ( exists $opts->{'GATEKEEPER_PORT'} )
	  {
	        $port = $opts->{'GATEKEEPER_PORT'};
	      }

	if ( defined $host )
	  {
	        my $dest = $host;
		if ( defined $port ) { $dest .= ":$port" }

		    runcmd("globus-job-run $dest/jobmanager-fork /bin/sh -c \"rm -f $opts->{REMOTE_GSIFTP_OUTFILE}\"", undef);
	      }

	delete $opts->{'LOCAL_GSIFTP_INFILE'};
	delete $opts->{'LOCAL_GSIFTP_OUTFILE'};
	delete $opts->{'REMOTE_GSIFTP_OUTFILE'};
      }
    else
      {
	$tests->{'STATUS'}{$t} = FAILURE;
    }
  }


#
# check_globus_location
#
# Determine the GLOBUS_LOCATION definition of the remote host.
# We do it by tracking the gatekeeper service.
#
sub check_globus_location($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    # We first try to the simple way: echo GLOBUS_LOCATION in a fork job.
    # If that doesn't work, analyze the remote site's /etc/{services,xinetd}
    my @cmds;
    push @cmds, 'echo \$GLOBUS_LOCATION';
    my $a = run_remote_cmds(\@cmds, $opts);

    if ( scalar(@$a) != 0 )
      {
	if ( $a->[0]{'ExitStatus'} == 0 && $a->[0]{'RemoteExitStatus'} == 0 )
	  {
	        $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = $a->[0]{'Stdout'};
		    $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;
		    return;
	      }
      }

    if ( ! exists  $opts->{'FILE_HASH'}{'SERVICES'} ||
	 ! defined $opts->{'FILE_HASH'}{'SERVICES'} ) { return; }

    my $port = 2119;
    if ( exists $opts->{'GATEKEEPER_PORT'} )
      {
	$port = $opts->{'GATEKEEPER_PORT'};
      }

    if ( ! defined $port ) { return; }

    # get the service name for the port we are using
    my $svcname = get_service_name($opts, $port, 'tcp');
    if ( ! defined $svcname ) { return; }

    # Find the service config for this service name
    my $href = get_inetd_service_name($opts, $svcname, 'tcp');

    if ( ! defined $href ) { return; }

    my $globus_location = undef;

    if ( exists $href->{'env'} && ($href->{'env'} =~ m/LD_LIBRARY_PATH/) )
      {
	$globus_location = $href->{'env'};
	$globus_location =~ s/^.*LD_LIBRARY_PATH=//;
	$globus_location =~ s/(\S+)\s+.*$/$1/;
	my ($base, $path) = fileparse($globus_location);
	$globus_location = $path;
      }
    elsif ( exists $href->{'server'} )
      {
	my ($base,  $path)  = fileparse($href->{'server'});
	my ($base1, $path1) = fileparse($path);
	$globus_location = $path1;
      }

    if ( ! defined $globus_location ) { return; }

    $globus_location =~ s/\/$//;

    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = $globus_location;
    $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;
  }


#
# check_gatekeeper_conf
#
# Find and retrieve the remote gatekeeper configuration file.
#
sub check_gatekeeper_conf($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    my $svcname = undef;

    if ( exists $opts->{'SVCNAMES'}{'globus-gatekeeper'} )
      {
	$svcname = $opts->{'SVCNAMES'}{'globus-gatekeeper'};
      }
    else
      {
	# See if we can figure out the service name
	if ( exists $opts->{'FILE_HASH'}{'INETD_CONF'} )
	  {
	        my $hi = $opts->{'FILE_HASH'}{'INETD_CONF'};

		    if ( exists $opts->{'FILE_HASH'}{'SERVICES'} )
		      {
			my $hs = $opts->{'FILE_HASH'}{'SERVICES'};

			foreach my $svc ( keys %$hi )
			  {
			        if ( exists $hs->{$svc}{'tcp'}{'PORT'} &&
				      ($hs->{$svc}{'tcp'}{'PORT'} == 
				         $opts->{'GATEKEEPER_PORT'}) )
				  {
				    $opts->{'SVCNAMES'}{'globus-gatekeeper'} = $svc;
				    $svcname = $svc;
				    last;
				  }
			      }
		      }

		    if ( ! defined $svcname )
		      {
			my $foo = $tests->{'RESULTS'}{'globus_location'};
			$foo =~ s/\//\\\/\+/g;

			# See if we can match up an inetd service via GLOBUS_LOCATION
			foreach my $svc ( keys %$hi )
			  {
			        if ( exists $hi->{$svc}{'tcp'}{'server_args'} &&
				      $hi->{$svc}{'tcp'}{'server_args'} =~ m/^\-conf\s+$foo\/+etc\// )
				  {
				    $svcname = $svc;
				    $opts->{'SVCNAMES'}{'globus-gatekeeper'} = $svc;
				    last;
				    
				  }
				    elsif ( exists $hi->{$svc}{'tcp'}{'server'} &&
					        $hi->{$svc}{'tcp'}{'server'} =~ m/^$foo\/+sbin\/+globus\-gatekeeper$/ )
				      {
					$svcname = $svc;
					$opts->{'SVCNAMES'}{'globus-gatekeeper'} = $svc;
					last;
				      }
			      }
		      }
	      }
    }

    my $globus_gatekeeper_conf = undef;

    if ( defined $svcname )
      {
	if ( exists $opts->{'FILE_HASH'}{'INETD_CONF'}{$svcname}{'tcp'}{'server_args'} )
	  {
	        $globus_gatekeeper_conf = 
		  $opts->{'FILE_HASH'}{'INETD_CONF'}{$svcname}{'tcp'}{'server_args'};
		    $globus_gatekeeper_conf =~ s/^.*\-conf\s+(\S+).*$/$1/;
	      }
      }

    if ( ! defined $globus_gatekeeper_conf )
      {
	# Educated guess...
	$globus_gatekeeper_conf = $tests->{'RESULTS'}{'globus_location'} . 
	      '/etc/globus-gatekeeper.conf';
      }

    my $f = obtain_remote_file($tests, $opts, $globus_gatekeeper_conf);

    if ( ! defined $f || ! -f $f ) { return; }

    $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'} = hashify_gatekeeper_conf($f);

    unlink $f;

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = SUCCESS;
    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = YES . 
      "\n  $globus_gatekeeper_conf";
  }


#
# check_grid_mapfile_users
#
# Get a list of the remote user accounts used in the remote grid-mapfile.
#
sub check_grid_mapfile_users($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    # Find where the grid-mapfile lives
    my $grid_mapfile = undef;

    if ( exists $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'} )
      {
	if ( exists $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-gridmap'} &&
	          defined $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-gridmap'} )
	  {
	        $grid_mapfile = 
		  $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-gridmap'};
	      }
      }

    # If we can't find it, guess
    if ( ! defined $grid_mapfile )
      {
	$grid_mapfile = '/etc/grid-security/grid-mapfile';
      }

    my $f = obtain_remote_file($tests, $opts, $grid_mapfile);

    if ( ! defined $f || ! -f $f ) { return; }

    $opts->{'LOCAL_FILES'}{'GRID_MAPFILE'} = $f;

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = SUCCESS;

    $opts->{'FILES_HASH'}{'GRID_MAPFILE'} = hashify_grid_mapfile($f);

    my %rev = reverse %{$opts->{'FILES_HASH'}{'GRID_MAPFILE'}};

    if ( scalar(keys %rev) > 0 )
      {
	$tests->{'RESULTS'}{$opts->{'TESTNAME'}} = join(",", 
							sort (keys %rev));
      }
    else
      {
	$tests->{'RESULTS'}{$opts->{'TESTNAME'}} = 'NONE';
    }
  }


#
# check_configured_grid_services
#
# Examine the remote $GLOBUS_LOCATION/etc/grid-services.
#
sub check_configured_grid_services($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    # Find where the grid_services live
    my $grid_services = undef;

    if ( exists $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'} )
      {
	if ( exists $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-grid_services'} &&
	          defined $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-grid_services'} )
	  {
	        my $tmp = 
		  $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-grid_services'};

		    if ( $tmp !~ /^\// )
		      {
			if ( exists $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-home'} &&
			          defined $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-home'} )
			  {
			        $grid_services = 
				  $opts->{'FILE_HASH'}{'GATEKEEPER_CONF'}->{'-home'};
			      }
			else
			  {
			        # use existing GLOBUS_LOCATION
			        $grid_services = $tests->{'RESULTS'}{'globus_location'};
				}

			if ( $grid_services !~ /\/$/ ) { $grid_services .= '/'; }

			$grid_services .= $tmp;    
		      }
		    else
		      {
			$grid_services = $tmp;
			    }
	      }
      }

    # If we can't find it, guess
    if ( ! defined $grid_services )
      {
	$grid_services = $tests->{'RESULTS'}{'globus_location'};
	if ( $grid_services !~ /\/$/ ) { $grid_services .= '/'; }    
	$grid_services .= 'etc/grid-services';
      }

    my @cmds;
    push @cmds, "ls -l $grid_services | grep jobmanager | grep -v ~";

    my $a = run_remote_cmds(\@cmds, $opts);

    if ( scalar(@$a) != 0 )
      {
	if ( $a->[0]{'ExitStatus'} == 0 && $a->[0]{'RemoteExitStatus'} == 0 )
	  {
	        $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;
		    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = NONE;

		    my @results = split "\n", $a->[0]{'Stdout'};

		    my @jms;

		    for ( my $i = 0 ; $i < scalar(@results) ; $i++ )
		      {
			my @tokens = split /\s+/, $results[$i];
			#if ( scalar(@tokens) == 9 )
			if ( scalar(@tokens) > 8 )
			  {
			        push @jms, $tokens[8];
			      }
			#elsif ( scalar(@tokens) == 11 )
			#{
			#    my ($base, $path) = fileparse($tokens[10]);
			#    $jmdef = $base;
			#}
			
		      }

		    if ( scalar(@jms) > 0 )
		      {
			$tests->{'RESULTS'}{$opts->{'TESTNAME'}} = PASS . "\n  " . 
			      join ',', @jms;
		      }

		    #if ( defined $jmdef )
		    #{
		#if ( defined $tests->{'RESULTS'}{$opts->{'TESTNAME'}} )
		#{
		#    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} .= "\n";
		#}
		#$tests->{'RESULTS'}{$opts->{'TESTNAME'}} .= 
		#    "  Default jobmanager is $jmdef";
		#    }

		    # Now get a copy of the jobmanagers for later
		    @cmds = ();
		    foreach my $jm (@jms)
		      {
			push @cmds, "cat $grid_services/$jm";
		      }

		    $a = run_remote_cmds(\@cmds, $opts);

		    for ( my $i = 0; $i < scalar(@$a) ; $i++ )
		      {
    use File::Temp qw(tempfile unlink0 );
    use vars qw($fh $f);
    
    my $template = 'remoteXXXXXX';
    my $dir      = '/tmp';
    ($fh, $f) = tempfile($template, DIR => $dir);

#			my $fh = new File::Temp(
#						TEMPLATE => 'remoteXXXXXX',
#						DIR      => $opts->{'TMPDIR'}, 
#						UNLINK   => 0,
#						);
#
			my @lines = split "\n", $a->[$i]{'Stdout'};

			foreach my $line (@lines)
			  {
			        print $fh "$line\n";
			      }

			close $fh;

			$opts->{'FILE_HASH'}{$jms[$i]} = 
			      hashify_jobmanager($f);

			unlink $f;
		      }
	      }
      }
  }


#
# check_scheduler_types
#
# Check the scheduler types associated with the remote jobmanagers.
#
sub check_scheduler_types($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    # XXX - FIXME - this is icky
    my %h;

    foreach my $key ( sort keys %{$opts->{'FILE_HASH'}} )
      {
	if ( $key =~ /^jobmanager/ )
	  {
	        $h{$key} = undef;
		    if ( exists $opts->{'FILE_HASH'}{$key}{'-type'} && 
			  defined $opts->{'FILE_HASH'}{$key}{'-type'} )
		      {
			$h{$key} = $opts->{'FILE_HASH'}{$key}{'-type'};
		      }
	      }
      }

    my $result = SUCCESS;
    foreach my $key ( keys %h )
      {
	if ( ! defined $h{$key} )
	  {
	        $result = FAILURE;
		    last;
	      }
      }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = $result;
    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = $result . "\n";

    foreach my $key ( sort keys %h )
      {
	my $type = UNKNOWN;
	if ( defined $h{$key} ) { $type = $h{$key}; }

	$tests->{'RESULTS'}{$opts->{'TESTNAME'}} .= 
	      "  $key is of type $type\n";

	$opts->{'JM_TYPES'}{$key} = $type;
      }
  }


#
# check_globus_tools_vars
#
# Examine the remote $GLOBUS_LOCATION/libexec/globus-sh-tools-vars.sh.
#
sub check_globus_tools_vars($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    my $rfile = $tests->{'RESULTS'}{'globus_location'} . 
      '/libexec/globus-sh-tools-vars.sh';

    $opts->{'LOCAL_FILES'}{'GLOBUS_SH_TOOLS_VARS'} = 
      obtain_remote_file($tests, $opts, $rfile);

    if ( defined $opts->{'LOCAL_FILES'}{'GLOBUS_SH_TOOLS_VARS'} )
      {
	$tests->{'STATUS'}{$opts->{'TESTNAME'}} = SUCCESS;
	$tests->{'RESULTS'}{$opts->{'TESTNAME'}} = YES;
      }
    else
      {
	$tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;
	$tests->{'RESULTS'}{$opts->{'TESTNAME'}} = NO;
    }
  }


#
# check_scheduler_paths
#
# Determine the paths to the scheduler commands on the remote site.
#
sub check_scheduler_paths($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = FAILURE;

    foreach my $key ( keys %{$opts->{'JM_TYPES'}} )
      {
	my $sched = $opts->{'JM_TYPES'}{$key};
	if ( ! exists $opts->{'SCHED_PATH'}{$sched} )
	  {
	        $opts->{'SCHED_PATH'}{$sched} = undef;
	      }

	if ( ! defined $sched )  { next; }
	if ( $sched eq UNKNOWN ) { next; }
	if ( $sched eq 'fork' )  { $opts->{'SCHED_PATH'}{$sched} = 'N/A'; next; }

	my $val = undef;
	if ( $sched eq 'condor' )
	  {
	        $val = get_globus_sh_tools_vars_var($opts, 'GLOBUS_SH_CONDOR_Q');
	      }
	elsif ( $sched eq 'pbs' )
	  {
	        $val = get_globus_sh_tools_vars_var($opts, 'GLOBUS_SH_QSTAT');
	      }
	elsif ( $sched eq 'lsf' )
	  {
	        $val = get_globus_sh_tools_vars_var($opts, 'GLOBUS_SH_BJOBS');
	      }
	elsif ( $sched eq 'fbs' )
	  {
	        # Uh oh.  No entry in globus-sh-tools-vars.sh for fbs...
	        $val = 'fbs';
	      }

	if ( defined $val )
	  {
	        my ($base, $path) = fileparse($val);
		    if ( defined $path )
		      {
			$path =~ s/\/+$//;
			$opts->{'SCHED_PATH'}{$sched} = $path;
		      }
	      }
      }

    my $result = SUCCESS;
    foreach my $key ( keys %{$opts->{'JM_TYPES'}} )
      {
	if ( ! exists $opts->{'SCHED_PATH'}{$opts->{'JM_TYPES'}{$key}} || 
	          ! defined $opts->{'SCHED_PATH'}{$opts->{'JM_TYPES'}{$key}} )
	  {
	        $result = FAILURE;
		    last;
	      }
      }

    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = $result;
    $tests->{'RESULTS'}{$opts->{'TESTNAME'}} = $result . "\n";

    foreach my $key ( sort keys %{$opts->{'SCHED_PATH'}} )
      {
	my $path = UNKNOWN;
	if ( defined $opts->{'SCHED_PATH'}{$key} )
	  {
	        $path = $opts->{'SCHED_PATH'}{$key};
	      }

	$tests->{'RESULTS'}{$opts->{'TESTNAME'}} .= 
	      "  Path to $key binaries is $path\n";
      }
  }


#
# check_vdt_version
#
# See if the remote site's middleware deployment is from the VDT
# by looking for (and running, if found) a properly placed vdt-version.info.
#
sub check_vdt_version($$)
  {
    my ($tests, $opts)  = @_;

    if ( check_prereqs($tests, $opts) != 0 ) { return; }

    # This file is optional
    $tests->{'STATUS'}{$opts->{'TESTNAME'}} = SUCCESS;

    my $d = $tests->{'RESULTS'}{'globus_location'};

    my @cmds;
    push @cmds, "if test -x $d/../vdt/bin/vdt-version ; then VDT_LOCATION=$d/.. $d/../vdt/bin/vdt-version; else rpm -qf $d/bin/openssl; fi";

    my $a = run_remote_cmds(\@cmds, $opts);

    if ( scalar(@$a) != 0 )
      {
	if ( $a->[0]{'ExitStatus'} == 0 && $a->[0]{'RemoteExitStatus'} == 0 )
	  {
	        $tests->{'STATUS'}{$opts->{'TESTNAME'}}  = SUCCESS;

		    if ( $a->[0]{'Stdout'} =~ /VDT\s+\S+\s+(\S+).*/ )
		      {
			(my $v = $1) =~ s/\:$//;
			$tests->{'RESULTS'}{$opts->{'TESTNAME'}} = 
			      YES . '; version ' . $v;
		      }
		    elsif ( $a->[0]{'Stdout'} =~ /VDT(\d+\.\d+\.\d+).*/ )
		      {
			my $v = $1;
			$tests->{'RESULTS'}{$opts->{'TESTNAME'}} = 
			      YES . '; version ' . $v;
		      }
	      }
      }
  }




###################################################################
###################################################################
#
# Begin main
#
###################################################################
###################################################################

# Set up some defaults
my $columns = 79;
if ( defined $ENV{'COLUMNS'} ) { $columns = $ENV{'COLUMNS'} - 1; }
if ( exists $ENV{'GLOBUS_LOCATION'} )
  {
    $ENV{'PATH'} = "$ENV{PATH}:$ENV{GLOBUS_LOCATION}/bin";
  }
my $localhost = get_fqdn("localhost");
my $user = getlogin();
if ( !defined($user) )
  {
    $user = getpwuid($<);
  }


# Set up the tests
#
# Tests are performed in a specific order defined in 'ORDER'.
# 'DESCRIP' maps test name to a short description of the test.
# 'SUBS' maps test name to a subroutine.
# 'ENABLED' controls whether or not a test is enabled.
# 'STATUS' records the status of the test (UNTESTED, FAILURE, SUCCESS).
# 'RESULTS' records any test result values.
# 'MESGS' records any messages associated with the test.
# 'PREREQS' describes dependencies among tests.
# 'ACTPRE' maps test name to a subroutine called before a test is performed.
# 'ACTPOST' maps test name to a subroutine called after a test is performed.
# 'ACTONPASS' maps test name to a subroutine called after a passed test.
# 'ACTONFAIL' maps test name to a subroutine called after a failed test.
#
# All the test subroutines have the following prototypes:
#
# void test_subr(reference to tests hash, reference to option hash)
#
my %tests;

$tests{'ORDER'} = [
		      'path', 
		      'proxy', 
		      'remote_host_reachable', 
		      'gatekeeper_service', 
		      'auth',
		      'hello_world',
		      'uptime',
		      'services',
		      'inetd',
		      'globus_location',
		      'hostcert',
		      'gatekeeper_conf',
		      'grid_mapfile_users',
		      'globus_tools_vars',
		      'configured_grid_services',
		      'scheduler_types',
		      'mds_service',
		      'vdt_version',
		      ];

$tests{'DESCRIP'}{'path'}  = 'prerequisites needed for testing';
$tests{'DESCRIP'}{'proxy'} = 'for a valid proxy for ' . $user . '@' . $localhost;
$tests{'DESCRIP'}{'remote_host_reachable'} = 'if remote host is reachable';
$tests{'DESCRIP'}{'gatekeeper_service'} = 'for a running gatekeeper';
$tests{'DESCRIP'}{'auth'}               = 'authentication';
$tests{'DESCRIP'}{'hello_world'}        = '\'Hello, World\' application';
$tests{'DESCRIP'}{'uptime'}             = 'remote host uptime';
$tests{'DESCRIP'}{'services'}           = 'remote Internet network services list';
$tests{'DESCRIP'}{'inetd'}              = 'remote Internet servers database configuration';
$tests{'DESCRIP'}{'globus_location'}    = 'for GLOBUS_LOCATION';
$tests{'DESCRIP'}{'hostcert'}           = 'expiration date of remote host certificate';
$tests{'DESCRIP'}{'gatekeeper_conf'}    = 'for gatekeeper configuration file';
$tests{'DESCRIP'}{'gsiftp_service'}     = 'for a running gsiftp server';
$tests{'DESCRIP'}{'gsiftp_local_in'}    = 'gsiftp (local client, remote -> local)';
$tests{'DESCRIP'}{'gsiftp_local_out'}   = 'gsiftp (local client, local -> remote)';
$tests{'DESCRIP'}{'gsiftp_local_diffs'} = 'that no differences exist between gsiftp\'d files';
$tests{'DESCRIP'}{'grid_mapfile_users'} = 'users in grid-mapfile';
$tests{'DESCRIP'}{'globus_tools_vars'}  = 'for remote globus-sh-tools-vars.sh';
$tests{'DESCRIP'}{'configured_grid_services'} = 'configured grid services';
$tests{'DESCRIP'}{'scheduler_types'}    = 'scheduler types associated with remote jobmanagers';
$tests{'DESCRIP'}{'scheduler_paths'}    = 'for paths to binaries of remote schedulers';
$tests{'DESCRIP'}{'mds_service'}        = 'for a running MDS service';
$tests{'DESCRIP'}{'vdt_version'}        = 'if Globus is deployed from the VDT';

for ( my $i = 0; $i < scalar(@{$tests{'ORDER'}}); $i++ )
  {
    $tests{'SUBS'}{$tests{'ORDER'}[$i]}    = 'check_' . $tests{'ORDER'}[$i];
    if ( ! defined &{$tests{'SUBS'}{$tests{'ORDER'}[$i]}} )
      {
	$tests{'SUBS'}{$tests{'ORDER'}[$i]} = undef;
      }
    $tests{'ENABLED'}{$tests{'ORDER'}[$i]}   = 1;
    $tests{'MESGS'}{$tests{'ORDER'}[$i]}     = undef;
    $tests{'STATUS'}{$tests{'ORDER'}[$i]}    = UNTESTED;
    $tests{'RESULTS'}{$tests{'ORDER'}[$i]}   = undef;
    $tests{'PREREQS'}{$tests{'ORDER'}[$i]}   = undef;
    $tests{'ACTPRE'}{$tests{'ORDER'}[$i]}    = 'pre_check';
    $tests{'ACTPOST'}{$tests{'ORDER'}[$i]}   = 'post_check';
    $tests{'ACTONPASS'}{$tests{'ORDER'}[$i]} = undef;
    $tests{'ACTONFAIL'}{$tests{'ORDER'}[$i]} = undef;
  }

$tests{'ACTONFAIL'}{'path'}  = 'print_msgs_and_exit';
$tests{'ACTONFAIL'}{'proxy'} = 'print_msgs_and_exit';

$tests{'PREREQS'}{'gatekeeper_service'} = 'remote_host_reachable';
$tests{'PREREQS'}{'auth'}               = 'gatekeeper_service';
$tests{'PREREQS'}{'hello_world'}        = 'auth';
$tests{'PREREQS'}{'uptime'}             = 'hello_world';
$tests{'PREREQS'}{'services'}           = 'auth';
$tests{'PREREQS'}{'inetd'}              = 'auth';
$tests{'PREREQS'}{'globus_location'}    = 'auth';
$tests{'PREREQS'}{'hostcert'}           = 'globus_location';
$tests{'PREREQS'}{'gatekeeper_conf'}    = 'globus_location';
$tests{'PREREQS'}{'gsiftp_service'}     = 'globus_location';
$tests{'PREREQS'}{'gsiftp_local_out'}   = 'gsiftp_service';
$tests{'PREREQS'}{'gsiftp_local_in'}    = 'gsiftp_local_out';
$tests{'PREREQS'}{'grid_mapfile_users'} = 'gatekeeper_conf';
$tests{'PREREQS'}{'globus_tools_vars'}  = 'globus_location';
$tests{'PREREQS'}{'configured_grid_services'} = 'gatekeeper_conf';
$tests{'PREREQS'}{'scheduler_types'}    = 'configured_grid_services';
$tests{'PREREQS'}{'scheduler_paths'}    = 'scheduler_types';
$tests{'PREREQS'}{'mds_service'}        = 'globus_location';
$tests{'PREREQS'}{'vdt_version'}        = 'globus_location';


# Handle options
my $help         = undef;
my $host         = 'matrix.physics.ox.ac.uk';
#my $host         = 'localhost';
my $tmpdir       = '/tmp';
my $show_tests   = undef;
my $skip_auth    = undef;
my $skip_mds     = undef;
my $skip_summary = undef;
my $verbose      = undef;

if ( exists $ENV{'TMPDIR'} )
  {
    if ( -d $ENV{'TMPDIR'} )
      {
	$tmpdir = $ENV{'TMPDIR'};
      }
  }

GetOptions(
	      "help"         => \$help,
	      "host=s"       => \$host,
	      "tmpdir=s"     => \$tmpdir,
	      "skip-auth"    => \$skip_auth,
	      "skip-mds"     => \$skip_mds,
	      "skip-summary" => \$skip_summary,
	      "verbose"      => \$verbose,
	      );

if ( defined $help )
  {
    usage();
    exit;
  }

if ( defined $show_tests )
  {
    show_tests();
    exit;
  }

my @sites;
if ( defined $host )
  {
    if ( $host !~ m/,$/ ) { $host .= ","; }

    my @foo = split ",", $host;

    for ( my $i=(scalar(@foo)-1); $i >= 0; $i-- )
      {
	# remove empty elements
	if ( length($foo[$i]) == 0 )
	  {
	        splice(@foo, $i, 1);
		    next;
	      }

	# lowercase the names
	$foo[$i] = lc $foo[$i];

	my $h = $foo[$i];
	$h =~ s/:.*$//;
	my $p = $foo[$i];
	if ( $p =~ m/^.*:/ )
	  {
	        $p =~ s/^.*://;
	      }
	else
	  {
	        $p = 2119;
		}

	my %hash;
	$hash{'GATEKEEPER_HOST'} = $h;
	$hash{'GATEKEEPER_PORT'} = $p;
	if ($hash{'GATEKEEPER_HOST'} eq "localhost")
	  {
	        $hash{'GATEKEEPER_HOST'} = get_fqdn($hash{'GATEKEEPER_HOST'});
	      }

	push @sites, {%hash};
      }
  }

# Here we go.
print get_separator("", "=", $columns) . "\n";
print "Info: Site verification initiated at " . gmtime() . " GMT.\n";
print get_separator("", "=", $columns) . "\n";

# Run the tests for each remote host.
my %opts;
$opts{'VERBOSE'}   = $verbose;
$opts{'TMPDIR'}    = $tmpdir;
$opts{'COLUMNS'}   = $columns;
$opts{'USER'}      = $user;
$opts{'LOCALHOST'} = $localhost;

for ( my $j = 0; $j < scalar(@sites); $j++ )
  {
    if ( exists $sites[$j]->{'GATEKEEPER_HOST'} )
      {
	$opts{'GATEKEEPER_HOST'} = $sites[$j]->{'GATEKEEPER_HOST'};
      }

    if ( exists $sites[$j]->{'GATEKEEPER_PORT'} )
      {
	$opts{'GATEKEEPER_PORT'} = $sites[$j]->{'GATEKEEPER_PORT'};
      }

    # Banner
    my $title = "Begin " . $opts{'GATEKEEPER_HOST'} . " at " . 
      gmtime() . " GMT";
    print get_separator("", "-", $columns) . "\n";
    print get_separator($title, "-", $columns) . "\n";
    print get_separator("", "-", $columns) . "\n";

    for ( my $i = 0; $i < scalar(@{$tests{'ORDER'}}); $i++ )
      {
	my $t = $tests{'ORDER'}[$i];

	$opts{'TESTNAME'} = $t;

	if ( ! defined $tests{'SUBS'}{$t} )
	  {
	        print "Info: No implementation for test \"$t\"; skipping...\n";
		    next;
	      }

	if ( ! defined $tests{'ENABLED'}{$t} )
	  {
	        print "Warn: Test \"$t\" is disabled; skipping... \n";
		    next;
	      }

	# Pre-testing hook
	if ( defined $tests{'ACTPRE'}{$t} )
	  {
	        if ( defined &{$tests{'ACTPRE'}{$t}} )
		  {
		    &{$tests{'ACTPRE'}{$t}}(\%tests, \%opts);
		  }
		    else
		      {
			print "Warn: No pre-testing subroutine \"$tests{ACTPRE}{$t}\" exists (test \"$t\")\n";
			    }
	      }

	# Perform the test
	&{$tests{'SUBS'}{$t}}(\%tests, \%opts);

	# Post-testing hook
	if ( defined $tests{'ACTPOST'}{$t} )
	  {
	        if ( defined &{$tests{'ACTPOST'}{$t}} )
		  {
		    &{$tests{'ACTPOST'}{$t}}(\%tests, \%opts);
		  }
		    else
		      {
			print "Warn: No post-testing subroutine \"$tests{ACTPOST}{$t}\" exists (test \"$t\")\n";
			    }
	      }
      }

    # Banner
    $title = "End " . $opts{'GATEKEEPER_HOST'} . " at " . gmtime() . " GMT";
    print get_separator("", "-", $columns) . "\n";
    print get_separator($title, "-", $columns) . "\n";
    print get_separator("", "-", $columns) . "\n";

    # Cleanup
    delete $opts{'GATEKEEPER_HOST'};
    delete $opts{'GATEKEEPER_PORT'};
    if ( defined $opts{'GSIFTP_HOST'} ) { delete $opts{'GSIFTP_HOST'}; }
    if ( defined $opts{'GSIFTP_PORT'} ) { delete $opts{'GSIFTP_PORT'}; }
    delete $tests{'RESULTS'};
    delete $tests{'STATUS'};
    delete $tests{'MESGS'};
  }

# We are done.
print get_separator("", "=", $columns) . "\n";
print "Info: Site verification completed at " . gmtime() . " GMT.\n";

#
# usage()
#
# Print the usage for this script.
#
sub usage {
    my $scriptname = fileparse("$0");
    print "\n";
    print "Usage: $scriptname [OPTIONS]\n";
    print "\n";
    print "Perfom Grid2003 site verification procedures and display\n";
    print "test results.\n";
    print "\n";
    print "Options:\n";
    print "\n";
    print "  --help          Display this message and exit.\n";
    print "  --host=...      Only query selected host(s).  Multiple hosts\n";
    print "                  can be specified in a comma-separated list.\n";
    print "                  Can query non-official sites in this manner.\n";
#    print "  --vo=...     Filter results by virtual organization (VO).\n";
#    print "               Multiple VOs can be specified in a comma-separated\n";
#    print "               list.  VO name must match the site sponsor\n";
#    print "               advertised by MDS.\n";
    print "  --skip-auth     Skip checks requiring authentication.\n";
    print "  --skip-mds      Skip MDS checks and checks relying on MDS info.\n";
    print "  --skip-summary  Skip generation of summary tables after\n";
    print "                  completion of tests.\n";
    print "  --verbose       Verbose output from authentication checks.\n";
    print "\n";
}


#
# get_fmts()
#
# Return perl formats for a table.
#
sub get_fmts {
    my ($num, $stem, $sites_ref, $keys_ref, $descrip) = @_;

    my $header_fmt = "format HEADER$num = \n";
    $header_fmt .= "Table $num: $descrip\n\n";
    my $data_fmt = "format DATA$num = \n";

    my $betw_field_spacing = "  ";

    my $underscore;
    my @vars;

    for (my $i = 0; $i < scalar(@$keys_ref); $i++) {
      my $longest = 0;
      foreach my $site (@$sites_ref) {
	if (defined($site->{"$stem"})) {
	  my $key = $keys_ref->[$i];
	  $key =~ s/\$/\\\$/g;
	  if (defined($site->{"$stem"}->{"$key"})) {
	    if (length($site->{"$stem"}->{"$key"}) > $longest) { 
	      $longest = length($site->{"$stem"}->{"$key"});
	    }
	  }
	}
      }

      if ($longest < length($keys_ref->[$i])) { 
	    $longest = length($keys_ref->[$i]);
	  }

      my $j = 0;

      while ($j < $longest) {
	if ($j == 0) {
	  $header_fmt .= $keys_ref->[$i];
	} elsif ($j >= length($keys_ref->[$i])) {
	  $header_fmt .= " ";
	}
	if ($j == 0) {
	  $data_fmt .= "@";
	} else {
	  $data_fmt .= "<";
	      }
	    $underscore .= "-";
	    $j++;
      }

      $j = 0;
      while ($j < length($betw_field_spacing)) {
	    $header_fmt .= " ";
	        $data_fmt   .= " ";
	        $underscore .= "-";
	        $j++;
	  }

      my $key = $keys_ref->[$i];
      $key =~ s/\$/\\\$/g;
      push @vars, "\$site->{\"$stem\"}->{\"$key\"}"
    }

    $header_fmt .= "\n" . $underscore . "\n.\n";
    $data_fmt   .= "\n" . join(",",@vars) . "\n.\n";

    return ($header_fmt, $data_fmt);
}

#
# generate_summary_tables()
#
# Print summary info in tabular fashion.
#
sub generate_summary_tables {

    my @sites = @_;

    if (scalar(@sites) > 0) {
      print get_separator("", "=", $columns) . "\n";

      print get_separator("", "-", $columns) . "\n";
      print get_separator("Begin Site Verification Summary Tables", "-", $columns) . "\n";
      print get_separator("", "-", $columns) . "\n\n";

      my @keys;
      my %hash;
      $hash{"Key"} = "Mds";
      $hash{"Mds"} = ["Hostname", "SiteName", "VOsponsor", 
		      "SchemaOK", "MDSVersion"];
      $hash{"Description"} = "MDS-advertised/derived info";
      push @keys, {%hash};

      $hash{"Key"} = "Mds";
      $hash{"Mds"} = ["Hostname", "Schedulers", "PolicyURL"];
      $hash{"Description"} = "MDS-advertised/derived info (cont.)";
      push @keys, {%hash};

      $hash{"Key"} = "Mds";
      $hash{"Mds"} = ["Hostname", "\$GRID3", "\$APP", "\$DATA", "\$TMP", 
		      "\$WNTMP"];
      $hash{"Description"} = "MDS-advertised/derived info (cont.)";
      push @keys, {%hash};

      $hash{"Key"} = "Auth";
      $hash{"Auth"} = ["Hostname", "Authenticate", "gsiftpTo", "gsiftpFrom", 
		        "HelloWorld"];
      $hash{"Description"} = "Checks requiring authentication";
      push @keys, {%hash};

      $hash{"Key"} = "Auth";
      $hash{"Auth"} = ["Hostname", "MultiVOgrid-mapfile"];
      $hash{"Description"} = "Checks requiring authentication (cont.)";
      push @keys, {%hash};

      $hash{"Key"} = "Auth";
      $hash{"Auth"} = ["Hostname", "gmond", "gmetad", "MonALISA"];
      $hash{"Description"} = "Checks on monitoring requiring authentication";
      push @keys, {%hash};

      $hash{"Key"} = "Auth";
      $hash{"Auth"} = ["Hostname", "\$APPexists", "\$APPhasSpace", 
		        "\$APPwritable"];
      $hash{"Description"} = "Checks requiring authentication & MDS (\$APP)";
      push @keys, {%hash};

      $hash{"Key"} = "Auth";
      $hash{"Auth"} = ["Hostname", "\$DATAexists", "\$DATAhasSpace", 
		        "\$DATAwritable"];
      $hash{"Description"} = "Checks requiring authentication & MDS (\$DATA)";
      push @keys, {%hash};

      $hash{"Key"} = "Auth";
      $hash{"Auth"} = ["Hostname", "\$TMPexists", "\$TMPhasSpace", 
		        "\$TMPwritable"];
      $hash{"Description"} = "Checks requiring authentication & MDS (\$TMP)";
      push @keys, {%hash};

      $hash{"Key"} = "Auth";
      $hash{"Auth"} = ["Hostname", "\$WNTMPexists", "\$WNTMPhasSpace", 
		        "\$WNTMPwritable"];
      $hash{"Description"} = "Checks requiring authentication & MDS (\$WNTMP)";
      push @keys, {%hash};

      for (my $i = 0; $i < scalar(@keys); $i++) {
	    my ($fmt_header, $fmt_data) = 
	      get_fmts($i, $keys[$i]->{"Key"}, \@sites, 
		        $keys[$i]->{"$keys[$i]->{\"Key\"}"},
		        $keys[$i]->{"Description"});

	        #print "\n$fmt_header\n";
	        #print "\n$fmt_data\n";

	        eval $fmt_header;
	        eval $fmt_data;

	        $^L = "\n\n";
	        $^ = "HEADER$i";
	        $~ = "DATA$i";

	    for (my $j = 0; $j < scalar(@sites); $j++) {
	      # must be global...
	      $site = $sites[$j];
	      write;
	    }

	        $- = 0;
	  }

      print "\n";
      print get_separator("", "-", $columns) . "\n";
      print get_separator("End Site Verification Summary Tables", "-", $columns) . "\n";
      print get_separator("", "-", $columns) . "\n";
    }
}

########## Utility/Debug routines ##############

#
# print_array_of_hash_refs()
#
# A debugging routine - print the contents of the of an array of 
# hash references.
#
sub print_array_of_hash_refs {

  foreach my $foo (@_) {
    print "********\n";
    foreach my $key (keys(%{$foo})) {
          print "\t$key = $foo->{$key}\n";
	}
  }    
}

#
# runcmd()
#
# Run a command via system().  stdout and stderr are redirected to
# /dev/null unless the verbose flag is defined.
#
sub runcmd {
    my ($cmd, $verbose) = @_;

    if (!defined($verbose)) {
      $cmd .= " > /dev/null";
    }
    $cmd .= " 2>&1";
    $cmd = "/bin/sh -c '$cmd'";

    if (defined($verbose)) {
      print "\n$cmd\n";
    }
    return system($cmd);
}

sub runcmd2 
  {
    my ($cmd, $opts) = @_;

    my $verbose = undef;
    if ( defined $opts && exists $opts->{'VERBOSE'} ) 
      {
	$verbose = $opts->{'VERBOSE'};
      }

    if ( ! defined $verbose ) { $cmd .= " > /dev/null"; }

    $cmd .= " 2>&1";
    $cmd = "/bin/sh -c '$cmd'";

    if ( defined $verbose ) { print "\n$cmd\n"; }

    return system($cmd);
}

#
# runcmd_remote()
#
# Run a command via globus-job-run.  stdout and stderr are redirected to
# /dev/null unless the verbose flag is defined.  Return status of command
# (not of globus-job-run!) is returned.
#
sub runcmd_remote {
    my ($remote, $cmd, $verbose, $dont_redirect) = @_;

    if (!defined($remote)) { return undef; }
    if (!defined($cmd)) { return undef; }

    if (!defined($verbose) && !defined($dont_redirect)) {
      $cmd .= " > /dev/null";
    }
    $cmd .= " 2>&1";
    $cmd = "globus-job-run $remote /bin/sh -c '$cmd; echo \$?'";

    if (defined($verbose)) {
      print "\n$cmd\n";
    }

    my $foo = `$cmd`;
    my @output = split "\n", $foo;

    my $status = undef;
    my $result = undef;

    if (scalar(@output) > 0) {
      $status = $output[scalar(@output)-1];
    } else {
      $status = 0;
    }
    if (scalar(@output) > 1) {
      $result = trim($output[scalar(@output)-2]);
    }

    return ($status, $result);
}

#
# trim()
#
# Remove surrounding whitespace from incoming string.
#
sub trim {
    my $string = shift;

    if (!defined($string)) { return $string; }

    $string =~ s/^\s+//;
    $string =~ s/\s+$//;

    return($string);
}

#
# get_separator()
#
# Return a separator string composed of a centered title surrounded by a 
# single character extended to length "len".
#
sub get_separator {
    my ($title, $char, $len, $just) = @_;

    if (!defined($len)) { $len = 79; }

    my $separator = "";
    my $tlen = length($title);

    if ($tlen >= $len) { return $title; }

    if ($tlen != 0) {
      if (defined($just)) {
	if (lc($just) eq "l") {
	  $title .= " ";
	} elsif (lc($just) eq "r") {
	  $title = " " . $title;
	} else {
	  $title = " " . $title . " ";
	      }
      } else {
	    $title = " " . $title . " ";
	    }
      $tlen = length($title);
    }

    if ($tlen >= $len) { return $title; }

    my $before = ($len - $tlen)/2;
    if (defined($just)) {
      if (lc($just) eq "l") {
	    $before = 0;
	  } elsif (lc($just) eq "r") {
	        $before = $len - $tlen;
	      }
    }

    while (length($separator) < $before) {
      $separator .= $char;
    }

    $separator .= $title;

    while (length($separator) < $len) {
      $separator .= $char;
    }

    return $separator;
}