#!/usr/bin/env perl
##**************************************************************
##
## Copyright (C) 1990-2007, Condor Team, Computer Sciences Department,
## University of Wisconsin-Madison, WI.
## 
## Licensed under the Apache License, Version 2.0 (the "License"); you
## may not use this file except in compliance with the License.  You may
## obtain a copy of the License at
## 
##    http://www.apache.org/licenses/LICENSE-2.0
## 
## Unless required by applicable law or agreed to in writing, software
## distributed under the License is distributed on an "AS IS" BASIS,
## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
## See the License for the specific language governing permissions and
## limitations under the License.
##
##**************************************************************

package MyCondor;
require 5.005;			# don't use 5.x.y string
use strict;
use POSIX qw(mktime);		# standard module
use File::Spec;			# standard module
use File::Basename;		# standard module
use File::Temp qw(tempfile);	# 5.6.1 standard module

use lib dirname($0);
use MyCommon;

# declare prototypes before exporting them
BEGIN {
    use Exporter   ();
    use vars       qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);

    @ISA = qw(Exporter);
    @EXPORT = qw(InitTests RunTests );
    %EXPORT_TAGS = ( );     # eg: TAG => [ qw!name1 name2! ],
    @EXPORT_OK = qw( );
}

# Prototypes
sub check_condor(\@);

# Globals
my $ContactFailure = "";


# *********************************************************
# create_subfile()
# *********************************************************
sub create_subfile ($$$$$) {
    # purpose: create a standard submit file
    # paramtr: $subfn (IN): the filename to use for the submit file
    #          $logfn (IN): the filename that Condor should produce logs in
    #          $outfn (IN): the filename that stdout goes to, or undef
    #          $errfn (IN): the filename that stderr goes to, or undef
    #          $executable (IN): fqpn of the application to run
    #          $arguments (IN): argument to supply, or undef for no args
    #          $universe (IN): either 'vanilla' or 'globus'
    #          $addin (IN): Additional lines as one string to include in sub
    my $Files = shift;
    my $subfn = $Files->{submit};
    my $logfn = $Files->{log};
    my $outfn = $Files->{out};
    my $errfn = $Files->{err};

    my $executable = shift;
    my $arguments = shift;
    my $universe = shift;
    my $addin = shift;

    # create submit file
    if ( open( SUB, ">$subfn" ) ) {
	my $TmpDir = "";
	$TmpDir = $main::tmpdir[0] if ( $#main::tmpdir >= 0 );
	print SUB "#\n# Filename: $subfn\n# generated: ", scalar localtime, "\n#\n";
	# print SUB "initialdir\t=$TmpDir\n" if ( $TmpDir ne "" );
	print SUB "executable\t= $executable\n";
	print SUB "arguments\t=$arguments\n" if ( length($arguments) > 0 );
	print SUB "transfer_executable = false\n";
	print SUB "copy_to_spool\t= false\n";
	print SUB $addin if ( length($addin) > 0 );
	print SUB "universe\t= $universe\n";
	print SUB "notification\t= NEVER\n";
	print SUB "log\t\t= $logfn\n";
	print SUB "output\t\t= $outfn\n" if ( length($outfn) > 0 );
	print SUB "error\t\t= $errfn\n" if ( length($errfn) > 0 );
	print SUB "queue\n";
	close(SUB);
    } else {
	print "FAILURE!";
	print "Unable to create $subfn: $!\n";
	return Result( -1, "can't create $subfn" );
    }

    1;				# success
}

# *********************************************************
# JobAnalyze()
# Analyze a failed job (why did it fail?)
# *********************************************************
sub JobAnalyze ($$$) {
    # paramtr: $JobId (IN): The Job's ID
    # paramtr: $Universe (IN): The Job's universe
    # paramtr: $JobAnalysis (Out): HashRef of our results
    my $JobId = shift;
    my $Universe = shift;
    my $JobAnalysis = shift;

    # Store info on it...
    $JobAnalysis->{Failure} = "";
    $JobAnalysis->{Summary} = "";


    # Vanilla / standard universe..
    if (  ( $Universe eq "vanilla" ) || ( $Universe eq "standard" )  ) {

	print "Performing a more detailed analysis..\n";

	# Try a condor_q -ana...
	my $Cmd = "condor_q -ana $JobId";
	if ( ! open( ANAL, "$Cmd|" ) ) {
	    print "Failure analyzing queue\n";
	    $JobAnalysis->{Summary} = "Job timed out";
	    return -1;
	}
	my @Analysis;
	while ( <ANAL> ) {
	    print;
	    chomp;
	    if ( /^\s+(\d+)\s+(.*)/ ) {
		push( @Analysis, $1 );
	    }
	}
	close( ANAL );
	if ( $#Analysis != 5 ) {
	    print "Failure analyzing queue (I don't understand the summary)\n";
	    $JobAnalysis->{Summary} = "Unknown failure";
	    return -1;
	}

	# Ok... Let's look at the results of condor_q -ana (finally)...
	my $MisMatch = ( $Analysis[0] ) + ( $Analysis[1] );
	my $Busy = ( $Analysis[2] ) + ( $Analysis[4] );
	my $Avail = $Analysis[5];
	my $DoNotPrefer = $Analysis[3];
	my $Result = 0;
	if ( ( 0 != $Busy ) && ( 0 == $Avail ) ) {
	    print "Ok - It appears that the pool is too busy\n";
	    $JobAnalysis->{Summary} = "Pool busy";
	} elsif ( ( 0 == $Busy ) && ( 0 != $Avail ) ) {
	    $JobAnalysis->{Failure} = "Hmmm... This is strange.. $Avail machines are available";
	    $JobAnalysis->{Summary} = "(Unknown; $Avail available)";
	    $Result = -1;
	} elsif ( ( 0 == $Busy ) && ( 0 == $Avail ) && ( 0 == $MisMatch ) ) {
	    $JobAnalysis->{Failure} = "Odd...  Do you have any machines in your pool?";
	    $JobAnalysis->{Summary} = "(No machines in pool)";
	    $Result = -1;
	} elsif ( ( 0 == $Busy ) && ( 0 == $Avail ) && ( 0 != $MisMatch ) ) {
	    $JobAnalysis->{Failure} = "There are no machines that match this job";
	    $JobAnalysis->{Summary} = "(No matches)";
	    $Result = -1;
	} elsif ( 0 != $DoNotPrefer ) {
	    print "Ok - No machines prefer this job\n";
	    $JobAnalysis->{Summary} = "Pool busy / unpreferred";
	    $Result = -1;
	} else {
	    $JobAnalysis->{Failure} = "I don't understand why your job isn't running";
	    $JobAnalysis->{Summary} = "Reason unknown";
	    $Result = -1;
	}

	# Done
	return $Result;

	# Globus Universe
    } elsif ( $Universe eq "globus" ) {

	print "Detailed failure analysis not available for '$Universe' jobs\n";

	# We'll have to do better than this later, but don't know
	# what the right answer is, at least for now....
	$JobAnalysis->{Failure} = "I don't know why your job isn't running";
	$JobAnalysis->{Summary} = "Reason unknown";
	my $Result = 1;

	# Done
	return $Result;

	# Other / unknown..
    } else {

	print "Detailed failure analysis not available for '$Universe' jobs\n";

	# We'll have to do better than this later, but don't know
	# what the right answer is, at least for now....
	$JobAnalysis->{Failure} = "I don't know why your job isn't running";
	$JobAnalysis->{Summary} = "Reason unknown";
	my $Result = 1;

	# Done
	return $Result;
    }


}  # JobAnalyze()


# *********************************************************
# WaitForRemove()
# Wait for the job to be removed...
# *********************************************************
sub WaitForRemove ($) {
    # paramtr: $Files (IN): the filename hash to use
    my $Files = shift;

    my $Aborted = 0;
    for my $Tries ( 1 .. 30 ) {
	last if ( ! open( LOG, $Files->{log} ) );
	while( <LOG> ) {
	    if ( /aborted/i ) {
		print;
		$Aborted++;
	    }
	}
	close( LOG );

	# Sleep _before_ "last if()" to give it a bit to
	# finish writing..
	sleep( 1 );
	last if ( $Aborted );
    }
    if ( 0 == $Aborted ) {
	print "Give up waiting for job to go away...\n";
    }

}

# *********************************************************
# CleanupFiles()
# *********************************************************
sub CleanupFiles ($) {
    # Purpose: create a standard submit file
    # paramtr: $Files (IN): the filename hash to use
    my $Files = shift;

    print "Cleaning up files: ";
    foreach my $Key ( keys %{$Files} )
    {
	if ( -f $Files->{$Key} )
	{
	    print "$Key ";
	    unlink( $Files->{$Key} ) || print STDERR "Unlink($Files->{$Key}) failed $!\n";
	}
    }
    print "\n";
}

# *********************************************************
# submit_condor()
# *********************************************************
sub submit_condor( $$$$ ) {
    my $universe = shift || 'vanilla';
    my $addin = shift;
    my $MaxSeconds = shift;
    my $Dest = shift;

    $ContactFailure = "";

    # Where to put the temp files?
    my $TmpDir = File::Spec->tmpdir();
    $TmpDir = $main::tmpdir[0] if ( $#main::tmpdir >= 0 );
    my ($fh, $fn) = tempfile( "test-XXXXXX", DIR => $TmpDir,
			      SUFFIX => '.sub', UNLINK => 1 );

    unless ( defined $fh ) {
	print "FAILURE!\n";
	print "Unable to create a temporary file. Sorry, that\'s it!\n";
	return Result( -1, "temp file" );
    }

    # create sub file
    my $basefn = substr($fn,0,-4) . "-$$";
    close($fh);
    my %Files = ( submit => $fn,
		  log => "$basefn-log.txt",
		  out => "$basefn-out.txt",
		  err => "$basefn-err.txt" );
    if ( create_subfile( \%Files,
			 # FIXME: Condor 6.4.x < 2 argv bug
			 # '/bin/date', "-u +%S-%M-%H-%d-%m-%Y"
			 "/bin/date", "-u", $universe, $addin ) < 0 )
    {
	CleanupFiles( \%Files );
	return -1;
    }

    # show file
    system("cat $Files{submit}");

    # submit file to condor
    my $cmd = "condor_submit $Files{submit}";
    print "\ntrying to submit $Files{submit}...\n";
    my @submit;
    if ( open( BATCH, "$cmd 2>&1 |" ) ) {
	while ( <BATCH> ) {
	    s/\r*\n$//;
	    push(@submit, $_);
	    print "# $_\n";
	}
	close(BATCH);
    } else {
	my $code = $? >> 8;
	print "FAILURE!\n";
	print "Unable to run $cmd: $code: $!\n";
	print "Please make sure that you can submit simple $universe jobs into your local \n";
	print "Condor system.\n";
	CleanupFiles( \%Files );
	$ContactFailure = "submit failed";
	return Result( -1, "submit failed" );
    }
    unlink( $Files{submit} );

    my $jobid;
    unless ( @submit == 3 && $submit[2] =~ /submitted to cluster (\d+)/ ) {
	print "FAILURE!\n";
	print "I cannot detect the job number. \n";
	print "Your Condor submission returned the following info:\n";
	print "@submit\n";
	CleanupFiles( \%Files );
	if ( my @Error = grep( /doesn\'t support/, @submit ) ) {
	    my $String = join( " ", @Error );
	    $String =~ s/Submitting job\(s\)//g;
	    return Result( -2, $ContactFailure = $String );
	} else {
	    return Result( -1, $ContactFailure = "submit failed: " . join( " ", @submit) );
	}
    } else {
	$jobid = $1;
	print "OK: submitted as job number $jobid\n";
    }

    # Wait for it to finish (or timeout)
    print "\nWaiting for the job to finish (up to $MaxSeconds seconds):\n";
    my $Elapsed = 0;
    my $JobOk = 0;
    $| = 1;
    while( 0 == $JobOk ) {
	sleep( 5 );
	my @lines = grep( /^\s*$jobid/, `condor_q $jobid` );
	$JobOk = 1 if ( ( $? == 0 ) && ( $#lines < 0 )  );
	last if ( ( $Elapsed += 5 ) > $MaxSeconds );
	print ".";
    };
    print "\n";

    # Check the status of the job...
    if ( 0 == $JobOk ) {
	if ( $MaxSeconds > 60 ) {
	    my $MaxMinutes = $MaxSeconds / 60;
	    print "Sorry, $MaxMinutes minutes, and no job status change.\n";
	} else {
	    print "Sorry, $MaxSeconds seconds, and no job status change.\n";
	}

	# Let's see what went wrong...
	my %JobAnalysis;
	my $Status = JobAnalyze( $jobid, $universe, \%JobAnalysis );

	# Done; clean up
	print "Attempting to remove job $jobid...\n";
	system("condor_rm $jobid" );
	WaitForRemove( \%Files );
	sleep( 5 );			# Give it a bit more time to cleanup..
	CleanupFiles( \%Files );

	# If it failed, handle it here...
	if ( $Status < 0 ) {
	    print "FAILURE!\n";
	    print "\t$JobAnalysis{Failure}\n";
	    print "It does not appear as if you can submit $universe jobs to this contact.\n";
	    return Result( $Status,
			   $ContactFailure = $Dest . "job timed out: $JobAnalysis{Summary}" );
	} elsif ( $Status > 0 ) {
	    print "WARNING - \t$JobAnalysis{Failure}\n";
	    print "I don't know if you can submit $universe jobs to this contact.\n";
	    return Result( $Status,
			   $ContactFailure = $Dest . "job timed out: $JobAnalysis{Summary}" );
	}

	# We're done here!
	Result(0, $JobAnalysis{Summary} );
	return 0;
    }
    undef $JobOk;

    # ########################################################
    # Here, it looks like the job completed normally.  Let's
    # check it more closely...
    # ########################################################
    if ( $Elapsed > 60 ) {
	print "OK: job finished in $Elapsed seconds (about time)\n";
    } else {
	print "OK: job finished in $Elapsed seconds\n";
    }

    # Check the error log file (stderr of the process)
    my $ErrFlag = 0;
    if ( -z "$Files{err}" ) {
	print "OK: no messages on stderr\n";
    } else {
	my @ErrMsg;
	if ( open( ERR, $Files{err} ) ) {
	    while( <ERR> ) {
		print;
		chomp;
		push @ErrMsg, $_;
	    }
	    close( ERR );
	}
	my $ErrMsg = join( " / ", @ErrMsg );
	my $FailureMsg = ( ( $#ErrMsg >= 0 ) ? substr( $ErrMsg, 0, 30 ) : "Job Error" );

	# A little analysis
	if ( $ErrMsg =~ /(nis|yp)/i ) {
	    print "This looks like a YP / NIS problem";
	    $FailureMsg = "NIS: " . $FailureMsg;
	    $ErrFlag++;
	} elsif ( $ErrMsg =~ /(invalid date|argument)/ ) {
	    print "If the message above complained about the number of arguments being\n";
	    print "wrong, or something like \"invalid date\", you probably have a version\n";
	    print "of buggy Condor. In that case, you should upgrade!\n";
	    $ErrFlag+=2;
	} else {
	    # Do nothing for now
	    $ErrFlag+=2;
	}
	Result( 1, $ContactFailure = $FailureMsg );
    }
    unlink( $Files{err} );

    # Check the stdout of the process
    if ( -s "$Files{out}" ) {
	$_ = `cat $Files{out}`;
	s/[\r\n]*$//;
	print "OK: message on stdout: $_\n";

	# check differences
	my $diff;
	my @date = parse_date($_);
	if ( @date == 0 ) {
	    print "FAILURE!\n";
	    print "The remote /bin/date -u command returned a string that I cannot parse.\n";
	    $ErrFlag+=3;
	} else {
	    my @mine = gmtime(time);	# save submit local time
	    if ( ($diff = abs(mktime(@mine)-mktime(@date))) < 120 ) {
		print "OK: only $diff s difference, good.\n";
	    } else {
		print "WARNING!\n";
		print "There is a difference between the remote host time and my time of $diff s.\n";
		print "Please note that Globus GSI is very sensitive to mis-timings. You definitely\n";
		print "want to consider installing an NTP service on all machines, the remote\n";
		print "contact and this submit host.\n";
		$ErrFlag++;
	    }
	}
    } else {
	print "WARNING: stdout reported no message, though it should have.\n";
	$ErrFlag+=3;
    }
    unlink( $Files{out} );

    # Finally, the user log
    if ( open( LOG, "<$Files{log}" ) ) {
	local $/="\n...\n";
	while ( <LOG> ) {
	    my $cc = substr($_,0,3);
	    printf( "%s: condor code %s found\n",
		    ( $cc eq '000' || $cc eq '001' || $cc eq '005' || $cc eq '017' ?
		      'OK' : 'WARNING' ), $cc );
	    if ( /^005/ && /\(return value (\d+)\)/ ) {
		if ( $1 eq 0 ) {
		    print "OK: exit code is 0\n";
		} else {
		    print "WARNING! exit code is $1\n";
		    $ErrFlag++;
		}
	    }
	}
	close(LOG);
    } else {
	my $code = $? >> 8;
	print "FAILURE! exit=$code: $!\n";
	print "I am unable to open the log $Files{log} for your job.\n";
	print "This is extremely fishy, and this is it.\n";
	CleanupFiles( \%Files );
	return Result( -1, $ContactFailure = "log file missing" );
    }
    CleanupFiles( \%Files );

    if ( $ErrFlag >= 2 ) {
	print "FAILURE!\n";
	print "There were 1+ problems reported with your job. I don\'t\n";
	print "believe that you can run $universe jobs successfully on this host.\n";
	return Result( -1, "job failed" );
    } elsif ( $ErrFlag > 0 ) {
	print "Warning!\n";
	print "There were some problems with the /bin/date job, but I";
	print "believe that you can run $universe jobs successfully on this host.\n";
	return Result( 1, "Job Warnings" );
    } else {
	print "OK: $universe jobs are runnable\n";
    }

    return 0;

}   # submit_condor()

# *********************************************************
# check_status()
# *********************************************************
sub check_status (\%\@$$) {
    my $status = shift;		# a hashref
    my $flag = shift;		# an arrayref
    my $key = shift;
    my $condition = shift;

    if ( exists $$status{$key} ) {
	$condition =~ s/XYZ/'$$status{$key}'/g;
	my $result = eval $condition;
	if ( $result ) {
	    print "OK: $key matches condition $condition\n";
	    return 1;
	} else {
	    print "WARNING: $key does not match $condition\n";
	    $$flag[0]++;
	    return 0;
	}
    } else {
	print "WARNING: $key does not exist!\n";
	$$flag[1]++;
	return -1;
    }
}

# *********************************************************
# InitTests()
# *********************************************************
sub InitTests() {

    ResultInit( "condor" );
    ResultInit( "condorg" );
}


# *********************************************************
# RunTests()
# *********************************************************
sub RunTests( \@ ) {
    # purpose: check out local Condor and Condor-G
    # paramtr: $contacts (IN): vector of jobmgr contact strings. 
    # warning: $contacts is a vector ref, which will be modified. 
    # returns: true for ok, undef for error
    my $jmlist = shift || return undef; # must have contacts list
    print ">> ", join("\n>> ",@$jmlist),"\n" if $main::DEBUG;
    my $line;
    $|=1;

    # Timeout
    my $CondorMaxTimeout = 5 * 60;
    if (  ( defined $ENV{GRIDPROBE_CONDOR_MAXWAIT} ) &&
	  ( $ENV{GRIDPROBE_CONDOR_MAXWAIT} =~ /^\d/ )  ) {
	$CondorMaxTimeout = $ENV{GRIDPROBE_CONDOR_MAXWAIT};
    }

    # Have we run already?
    if ( TestRan( "condor" ) ) {
	return 0;
    }

    # Skip?
    if ( ! TestEnabled( "condor" ) ) {
	return 0;
    }

    # Set the status name, run status
    SetTestName( "condor" );
    SetTestRan( 1 );

    print "Now let\'s look at your Condor installation.\n";

    #
    # condor binaries
    #
    my @ExeList = qw(condor_submit condor_submit_dag condor_status condor_q condor_version);
    foreach my $exe ( @ExeList ) {
	my $prg = find_exec($exe);
	unless ( defined $prg ) {
	    print "FAILURE!\n";
	    print "Unable to find the location of $exe. Please make extra sure that\n";
	    print "your Condor installation is complete, e.g. comes from the VDT, and\n";
	    print "does include Condor-G. Do not run on the central manager, unless you\n";
	    print "have a personal Condor installation.\n";
	    print "Your path is currently $ENV{PATH}\n\n";
	    return Ret( -1, "Can't find $exe" );
	} else {
	    print "OK: Found $prg\n";
	}
    }

    #
    # condor version
    #
    my @condor_version = `condor_version`;
    my $condor_version;
    $condor_version = $1 if $condor_version[0] =~ /:\s+(\d+\.\d+\.\d+)/;
    unless ( defined $condor_version ) {
	print "FAILURE!\n";
	print "I am unable to detect the version number of your Condor.\n";
	return Ret( -1, "no version" );
    } else {
	print "OK: You are running Condor version $condor_version\n";
	@condor_version = split /\./,$condor_version;
    }

    if ( $condor_version[0] < 6 ||
	 $condor_version[0] == 6 && $condor_version[1] < 4 ) {
	print "WARNING!\n";
	print "I believe strongly that you should upgrade your current version of\n";
	print "Condor to a supported and well-working branch.\n";
    }

    #
    # Check for condor_config
    #
    my $CondorConfig = "";
    my @CondorConfigs = (
			 "/etc/condor/condor_config",
			 "~condor/condor_config",
			 "",
			);
    if ( exists $ENV{CONDOR_CONFIG} ) {
	$CondorConfig = $ENV{CONDOR_CONFIG};
	if ( ! -f $CondorConfig ) {
	    print "CONDOR_CONFIG specified as $ENV{CONDOR_CONFIG}  does not exist!\n";
	    return Ret( -1, "$ENV{CONDOR_CONFIG} not found" );
	}
    } else {
	foreach $CondorConfig ( @CondorConfigs ) {
	    last if ( $CondorConfig eq "" );
	    if ( -f $CondorConfig ) {
		last;
	    }
	}
	if ( $CondorConfig eq "" ) {
	    print "No condor config found!\n";
	    return Ret( -1, "no config" );
	}
    }
    print "Condor config found @ $CondorConfig\n\n";

    print "\nThe essential Condor existence checks were successful. Now,\n";
    print "let\'s check the status of Condor on this machine\n";
    wait_enter;


    #
    # condor_status checks
    #
    my $myself = hostfqdn();
    unless ( defined $myself ) {
	print "FAILURE!\n";
	print "Unable to determine my own hostname. Without it, I cannot do very much\n";
	print "in determining the status. This may be a Perl error, or something else.\n";
	print "You may want to ask your sysadmin to setup up your system correctly.\n";
	return Ret( -1, "Unknown hostsname" );
    }

    print "Trying to obtain local status information from Condor...\n";
    print "myself = '$myself'\n";
    my $Command = "condor_status -l $myself";
    $Command .= " -pool " . $main::CondorHosts[0] if ( $#main::CondorHosts >= 0 );
    print "Trying '$Command' ...\n";
    if ( ! open( STATUS, "$Command 2>&1 |" )  ) {
	print "FAILURE: Error running $Command\n";
	return Ret( -1, "can't run condor_status" );
    }
    my @Status;
    while( <STATUS> ) {
	chomp;
	push @Status, $_;
    }
    close( STATUS );

    # How'd we do?
    my %Status;
    if ( 0 == $? ) {
	my $count = 0;
	foreach ( @Status ) {
	    s/\s*\r*\n$//;
	    next unless ( length );
	    my ($k,$v) = split(/\s=\s/,$_,2);
	    $v = substr($v,1,-1) if ( substr($v,0,1) eq "\"" );
	    $Status{$k} = $v;
	    $count++;
	}
	print "OK: read $count status variables\n" if $count;
    } else {
	my $code = $? >> 8;
	shift(@Status) while ( @Status > 0 && length($Status[0]) <= 1 );

	print "\nFAILURE!\n";
	print "The exit code is $code. Condor has this to say in its defense:\n";
	print "# ", join("\n# ",@Status), "\n";
	print "I cannot use condor_status successfully on this machine. This bodes ill.\n";
	print "Sometimes, the Condor is simply not started on this machine. Please fix\n";
	print "the Condor on this machine, and (re-)start it, if necessary.\n";
	return Ret( -1, "can't run condor_status" );
    }
    $#Status = -1;

    my @flag;
    check_status( %Status, @flag, 'Activity',  q{XYZ eq 'Idle'});
    check_status( %Status, @flag, 'CpuIsBusy', q{XYZ eq 'FALSE'} );
    check_status( %Status, @flag, 'LoadAvg',   q{XYZ <= 0.01} );
    check_status( %Status, @flag, 'CondorLoadAvg', q{XYZ <= 0.01} );
    print "\n";

    if ( $flag[0] > 0 ) {
	print "WARNING!\n";
	print "The machine is busy. The check, if Condor jobs run at all, might fail,\n";
	print "or it might succeed, depending on the configuration. Success is unlikely.\n";
    }

    if ( $flag[1] > 0 ) {
	print "FAILURE!\n";
	print "Your Condor system does not know about several essential variables that I\n";
	print "asked about. Your Condor appears to be uncompatible with this test.\n";
	return Ret( -1, "missing variables" );
    }

    #
    # vanilla universe to self
    #
    if ( TestEnabled( "condorv" ) ) {
	print "\nLet\'s check, if a simple vanilla universe job can be run.\n";
	print "Type \"skip\" without the quotes, if you want to skip this test.\n";
	print "Skipping this test is advised for a VDT client-side Condor.\n";
	if ( wait_enter("[hit enter or type skip]\n") !~ /skip/i ) {
	    my $Timeout = 5 * 60;
	    $Timeout = $CondorMaxTimeout
		if ( $Timeout > $CondorMaxTimeout );
	    my $Status = submit_condor( "vanilla", undef, $Timeout, "" );
	    Ret( $Status, "vanilla failed" ) if ( $Status < 0 );
	} else {
	    print "\nOK: You have chosen to skip the vanilla universe test.\n\n";
	}
    }

    # Done with Condor stuff; switch to Condor-G results...
    UnsetTestName( );

    # Skip?
    if ( ! TestEnabled( "condorg" ) ) {
	return 0;
    }

    # Did the Globus run & pass?  If not, don't bother...
    my $GlobusRes = GetResult( "globus" );
    if ( ! TestRan( "globus" ) ) {
	if ( TestEnabled( "globus" ) ) {
	    $main::Done = 0;
	    return 0;
	}
	print "\nSkipping the Condor-G test because the Globus test was skipped\n\n";
	return 0;
    } elsif ( $GlobusRes->{Status} < 0 ) {
	print "\nSkipping the Condor-G test because the Globus test failed\n\n";
	return 0;
    }

    # Now, setup & do it...
    SetTestName( "condorg" );
    SetTestRan( 1 );

	
    #
    # globus universe to remote scheduler universe
    #
    print "\nLet\'s check, if a simple globus universe scheduler jobs can be run.\n";
    print "Type \"skip\" without the quotes, if you want to skip this test.\n";
    print "Running this test is highly recommended.\n";
    if ( wait_enter("[hit enter or type skip]\n") !~ /skip/i ) {
	my @newlist;
	foreach my $gk ( @$jmlist ) {

	    print "\ntrying contact $gk\n";
	    my $ContactStr = "contact=\"$gk\"";

	    my $Timeout = 45;
	    $Timeout = $CondorMaxTimeout if ( $Timeout > $CondorMaxTimeout );
	    my $Rsl = ( "globusrsl\t=(jobType=single)" .
			"(condor_submit=(universe scheduler))" .
			"\n" );

	    my $Status = submit_condor( "globus",
					"globusscheduler\t= $gk\n" . $Rsl,
					$Timeout,
					"Scheduler " );
	    if ( $Status < -1 ) {
		print "ERROR: Condor-G is broken\n";
		foreach my $gk ( @$jmlist ) {
		    ContactResult( $gk, 1, "skipped", "" );
		}
		return $Status;
	    } elsif ( $Status < 0 ) {
		print "WARNING!\n";
		print "I will ignore contact $gk for now...\n";
		ContactResult( $gk, 2, "FAILED", $ContactFailure );
	    } elsif ( $Status > 0 ) {
		push(@newlist, $gk);
		ContactResult( $gk, 1, "Unknown", $ContactFailure );
	    } else {
		push(@newlist, $gk);
		ContactResult( $gk, 0, "OK", "" );
	    }
	    wait_enter;
	}
	$jmlist = [ @newlist ];
	if ( @$jmlist == 0 ) {
	    print "FAILURE!\n";
	    print "Not a single of your contacts works with CondorG.\n";
	    return Ret( -1, "No Valid Contacts" );
	}
    } else {
	print "\nWARNING!\n";
	print "You chose to forego the Globus tests for CondorG. All bets are off!\n\n";
    }

	
    #
    # globus universe to remote vanilla
    #
    print "\nLet\'s check, if a simple globus universe jobs can be run.\n";
    print "Type \"skip\" without the quotes, if you want to skip this test.\n";
    print "Running this test is highly recommended.\n";
    if ( wait_enter("[hit enter or type skip]\n") !~ /skip/i ) {
	my @newlist;
	foreach my $gk ( @$jmlist ) {

	    print "\ntrying contact $gk\n";
	    my $ContactStr = "contact=\"$gk\"";

	    my $Timeout = ( 300 < $CondorMaxTimeout ) ? 300 : $CondorMaxTimeout;
	    my $Rsl = ( "globusrsl\t=(jobType=single)" .
			"\n" );

	    my $Status = submit_condor( "globus",
					"globusscheduler\t= $gk\n" . $Rsl,
					$Timeout,
					"" );

	    if ( $Status < -1 ) {
		print "ERROR: Condor-G -> scheduler is broken\n";
		foreach my $gk ( @$jmlist ) {
		    ContactResult( $gk, 1, "skipped", "" );
		}
		return $Status;
	    } elsif ( $Status < 0 ) {
		print "WARNING!\n";
		print "I will ignore contact $gk for now...\n";
		ContactResult( $gk, 2, "FAILED", $ContactFailure );
	    } elsif ( $Status > 0 ) {
		push(@newlist, $gk);
		ContactResult( $gk, 1, "Unknown", $ContactFailure );
	    } else {
		push(@newlist, $gk);
		ContactResult( $gk, 0, "OK", "" );
	    }
	    wait_enter;
	}
	$jmlist = [ @newlist ];
	if ( @$jmlist == 0 ) {
	    print "FAILURE!\n";
	    print "Not a single of your contacts works with CondorG.\n";
	    return Ret( -1, "No Valid Contacts" );
	}
    } else {
	print "\nWARNING!\n";
	print "You chose to forego the Globus tests for CondorG. All bets are off!\n\n";
    }

    #
    # check my own DAGMan
    #
    print "\nFIXME: I will check your DAGMan installation another time...\n";

    print "\nYou did come far. Your Condor actually might do the trick.\n";
    wait_enter;

    # return the perl way
    return Ret( 0, "" );

}   # RunTests

1;
