
package Soar;

require 5.0;
use Cwd;
use strict;
use warnings;
use File::Copy;
use File::Basename;
use CondorUtils;
use Time::Local;

use base 'Exporter';

my $DEBUG = 0;
my @imageruns = ();
my @dagruns = ();
my @versions = ();
my %fscache = ();

our @EXPORT = qw(fsconfig FindDagNodes debug DebugOn DebugOff LookupFsConfig FsConfig FindLatestEnv FindLatestRun ReadSetEnv SetDate SetTime MakeNewProject InstallDerivedVersion InstallNewVersion ProcessWhiteList ProcessWhiteListString FindImageRuns ReadRunning DropRunning FindRunning AppendToFile LOCK_SH LOCK_EX LOCK_NB LOCK_UN CutRunFromCron UpdateRunFile VMwareAddFile VMwareRemakeDataDisk sub VMwareCreateRawDisk sub VMwareCreateMTOOLSRC VMwareFetchByPattern UpdateProjectObjects DeriveNewVersion GetJobId ResetDoneRunning AddRunning SetDone GetRunningDone SetNotRunning ResetByPattern GetVersion GetSyncLock ReleaseSyncLock GetResultValue CraftNewIndex GetHtmlLock ReleaseHtmlLock ParseOptions GenerateGeneric MakeProjectTracking DropCleanupDag);
my $softwareversion = "0.8.7";

sub LOCK_SH() {1}
sub LOCK_EX() {2}
sub LOCK_NB() {4}
sub LOCK_UN() {8}

# for automatically generated projects we need a grab bag of
# resources from tools to recreate the index.html to R and Matlab
# wrapper scripts to ( well we'll see what else ).
my $resources = "soar_resources";
my %projecthash; # used to create current index.html

# DMTCP wrapper conversion keys

my %dmtcpkeys =
(
	"executable" => "shim_dmtcp",
	"coresize" => -1,
	"arguments" => "shim_dmtcp.log /dev/null job.out job.err",
	"transfer_input_files" => "dmtcp.tar.gz",
	"environment" => "DMTCP_TMPDIR=./;JALIB_STDERR_PATH=/dev/null",
	"kill_sig" => 2,
);

# html crafting strings for projects
# Also filler files
my $webstart = "index.start";
my $webdiv = "index.div";
my $webend = "index.end";
my $itemstart = "	<tr align=\"center\" valign=\"top\">\n";
my $itemend = "	</tr>\n";
my $itemwhostart = "		<td width=\"60%\" align=\"left\">";
my $itemwhoend = "</td>\n";
my $itemwhatstart = "		<td align=\"left\"><a href=\"./projects.php?project=";
my $itemwhatend = "</a></td>\n";

# read this install's config files
my $fsconfigfile = "/p/stat/soar/control/fsconfig";
my $jobidfile = "JOBIDSRC";
my $initdone = FsConfig($fsconfigfile);
my %updateoptions;
my %genericprojectoptions;
#DebugOn();

BEGIN
{
    $DEBUG = 1;

	@imageruns = ();
	@dagruns = ();
	@versions = ();
	%fscache = ();
}

sub Reset
{
	@imageruns = ();
	@dagruns = ();
	@versions = ();
	%fscache = ();
}

sub GetVersion
{
	print "SOAR Version $softwareversion\n";
}

#####################################################
#
# FsConfig locates the various resources used by SOAR
# accross multiple file systems. We no longer expect
# a predefined location relative to TOPDIR.
#
# loads up hash %fscache
#####################################################

sub FsConfig
{
	my $configfile = shift;
	#debug("FsConfig for file $configfile\n");
	my $line = "";
	my @resources = ();
	open(FIG,"<$configfile") or die "Can not open config file<$configfile>\n";
	while(<FIG>) { 
		chomp();
		$line = $_;
		@resources = split /,/, $line;
		$fscache{$resources[0]} = $resources[1]
	}
	close(FIG);
	#DropConfig();
}

my %FsMainVars = (
	"SOAR" => "1",
	"CONTROL" => "1",
	"SOURCE" => "1",
	"RESULTS" => "1",
	"TARCACHELOC" => "1",
	"CONDORRUNS" => "1",
	"HTMLLOC" => "1",
	"IMAGERUNS" => "1",
);

sub LookupFsConfig
{
	my $request = shift;
	if(exists $fscache{$request}) {
		return($fscache{$request});
	} else {
		if( !(exists $FsMainVars{$request})) {
			# must be looking for a project that is not called out
			return($fscache{"IMAGERUNS"});
		} else {
			return("");
		}
	}
}

sub DropConfig
{
	foreach my $key ( sort keys (%fscache) ) {
		print "FsConfig: $key - $fscache{$key}\n";
	}
}

#####################################################
#
# ReadSetEnv
# Sets DATE, TOPDIR, UNIQUE, RUNLOC, RESLOC into environment
# from settings file
#
#####################################################

sub ReadSetEnv
{
	my $envfile = shift;
	my $line = "";
	debug( "Want environment variables from $envfile\n" );
	open(SAV,"<$envfile") || die "Can not open SAVE file: $!\n";
	while(<SAV>) {
    	chomp;
    	$line = $_;
		debug( "$line\n" );

    	if($line =~ /^DATE:\s*(.*)\s*$/ ) {
        	debug( "Found date <$1>\n");
			$ENV{DATE} = $1;
    	} elsif($line =~ /^TARCACHE:\s*(.*)\s*$/ ) {
        	#debug( "Found tar cache directory <<$1>\n");
			$ENV{TARCACHE} = $1;
    	} elsif($line =~ /^UNIQUE:\s*(.*)\s*$/ ) {
        	debug( "Found unique run directory <<$1>\n");
			$ENV{UNIQUE} = $1;
    	} elsif($line =~ /^DATASETS:\s*(.*)\s*$/ ) {
        	#debug( "Found data set directory <<$1>\n");
			$ENV{DATASETS} = $1;
    	} elsif($line =~ /^TOPDIR:\s*(.*)\s*$/ ) {
        	#debug( "Found top directory <<$1>\n");
			$ENV{TOPDIR} = $1;
    	} elsif($line =~ /^RUNLOC:\s*(.*)\s*$/ ) {
        	debug( "Found run directory <<$1>\n");
			$ENV{RUNLOC} = $1;
    	} elsif($line =~ /^RESLOC:\s*(.*)\s*$/ ) {
        	#debug( "Found result directory <<$1>\n");
			$ENV{RESLOC} = $1;
    	} elsif($line =~ /^SRCLOC:\s*(.*)\s*$/ ) {
        	#debug( "Found source directory <<$1>\n");
			$ENV{SRCLOC} = $1;
    	} elsif($line =~ /^PROJECT:\s*(.*)\s*$/ ) {
        	debug( "Found project <<$1>\n");
			$ENV{PROJECT} = $1;
    	} elsif($line =~ /^VERSION:\s*(.*)\s*$/ ) {
        	debug( "Found version <<$1>\n");
			$ENV{VERSION} = $1;
    	} elsif($line =~ /^KIND:\s*(.*)\s*$/ ) {
        	debug( "Found kind <<$1>\n");
			$ENV{KIND} = $1;
    	} else {
        	die "Found unexpected save <$line> \n";
    	}
	}
	close(SAV);
}


#####################################################
#
# ProcessWhiteList($imagedir, $white)
# If the file exists then check each request
# against an available data set. Return a ":"
# separated list. It is assumed for now that there is 
# one data set request per line
#
#####################################################

sub ProcessWhiteList
{
	
	my $imagedir = shift;
	my $white = shift;
	my @whiteruns;
	my $list = "";

	#DebugOn();
	debug("Setting up whitelist for data here <$imagedir> from this file <$white>\n");
	if(!(-f $white)) {
		print "White list does not exist!<$white>\n";
		exit(1);
	}

	my $line = "";
	open(WHITE,"<$white") || die "Can not open white list<$white>:$!\n";
	while(<WHITE>) {
		chomp();
		$line = $_;
		if($line =~ /^(dataset.*?)-(.*)$/) {
			debug("Testing for data dir<$imagedir/$1/$2>\n");
			if( -d "$imagedir/$1/$2") {
				push @whiteruns, "$1-$2";
			} else {
				debug("skip missing data set from dataset<$1><$imagedir/$1>\n");
			}
		} elsif($line =~ /^\s*$/) {
			print "Skip blank line.\n";
		} elsif($line =~ /^\s*(.*)$/) {
			debug("Look to see if $1 is a dataset\n");
			if( -d "$imagedir/$1") {
				debug("Good data set request\n");
				push @whiteruns, $1;
			} else {
				debug("skip missing data set<$imagedir/$1>\n");
			}
		}
	}
	close(WHITE);
	$list = join ":", @whiteruns;
	return($list);
}

#####################################################
#
# ProcessWhiteListString($imagedir, $whiteliststring)
# Its possible to request particular jobs when you
# change out the code. Process these as a white list
# without requiring a file to parse.
#
#####################################################

sub ProcessWhiteListString
{
	my $imagedir = shift;
	my $whiteliststring = shift;
	my @whiteruns;
	my $list = "";

	#DebugOn();
	debug("Setting up whitelist for data here <$imagedir> from this string <$whiteliststring>\n");

	my @possibleruns = split ",", $whiteliststring;

	foreach my $requested (@possibleruns) {
		if($requested =~ /^(dataset.*?)-(.*)$/) {
			debug("Testing for data dir<$imagedir/$1/$2>\n");
			if( -d "$imagedir/$1/$2") {
				push @whiteruns, "$1-$2";
			} else {
				debug("skip missing data set from dataset<$1><$imagedir/$1>\n");
			}
		} elsif($requested =~ /^\s*(.*)$/) {
			debug("Look to see if $1 is a dataset\n");
			if( -d "$imagedir/$1") {
				debug("Good data set request\n");
				push @whiteruns, $1;
			} else {
				debug("skip missing data set<$imagedir/$1>\n");
			}
		}
	}
	$list = join ":", @whiteruns;
	return($list);
}

#####################################################
#
# FindImageRuns
#
#####################################################

sub FindImageRuns
{
	my $location = shift;
	my $datasets = shift;

	#DebugOn();
	# what if we wish to run one or more specific
	# data sets?????

	debug("Looking for images here: $location\n");

	if( defined $datasets ) {
		my @datasets = split /,/, $datasets;
		debug("Datasets defined\n");
		foreach my $dataset (@datasets) {
			debug("dataset <$dataset>\n");
			if($dataset =~ /^(dataset.*)$/) {
				# handle collections of jobs
				my $subdata = "$location/$1";
				debug("My $subdata <$subdata>\n");
				my $collection = $1;
				opendir DS, $subdata or die "Can not open dataset<$1>\n";
				foreach my $subfile (readdir DS)
				{
    				next if $subfile =~ /^\.\.?$/;
					my $newname = "$collection-$subfile";
					debug("$subfile\n");
					if(-d "$subdata/$subfile") {
						push @imageruns, $newname;
					} else {
						debug("$subfile not a directory\n");
					}
				}
				closedir(DS);
			} else {
				die "Datasets must start with <dataset>. Your name <$dataset>.\n";
			}
		}
	} else {
		debug("Datasets NOT defined\n");
		opendir CH, $location or die "Can not open $location:$!\n";
		foreach my $file (readdir CH)
		{
			debug("file: <$file> \n");
			my $fullpath = "$location/$file";
    		next if $file =~ /^\.\.?$/;
			#print "Consider $fullpath\n";
			if( -d $fullpath ) {
				debug("full path <$fullpath>\n");
				if ($file =~ /^(dataset.*)$/) {
					# handle collections of jobs
					debug("folder is a dataset\n");
					my $subdata = "$location/$1";
					my $collection = $1;
					debug("Open dataset <$subdata>\n");
					opendir DS, $subdata or die "Can not open dataset<$1>\n";
					foreach my $subfile (readdir DS)
					{
    					next if $subfile =~ /^\.\.?$/;
						my $newname = "$collection-$subfile";
						debug("$subfile\n");
						if(-d "$subdata/$subfile") {
							debug("Keep <$subfile> its a dir\n");
							push @imageruns, $newname;
						} else {
							debug("skip plain file <$subfile>\n");
						}
							
					}
					closedir(DS);
				} elsif ($file =~ /^.*$/) {
					debug("$file\n");
					push @imageruns, $file;
				}
			} else {
				debug("FindImageRuns: skip plain file<$file>\n");
				next;
			}
		}
		closedir(CH);
	}
	my $list = join ":", @imageruns;
	return($list);
}

#####################################################
#
# FindDagNodes returns a list of node runtime
# directories.
#
#####################################################

sub FindDagNodes
{
	my $location = shift;
	debug("Looking for dag nodes here: $location\n");

	opendir DH, $location or die "Can not open $location:$!\n";
	foreach my $file (readdir DH)
	{
    	my $line = "";
    	next if $file =~ /^\.\.?$/;
    	#if($file =~ /^(\d+)$/) {
    	if(-d $file) {
		} else {
			next;
		}

		debug("$file\n");
		push @dagruns, $file;
	}
	closedir(DH);
	my $list = join ":", @dagruns;
	return($list);
}	


#####################################################
#
# FindVersions returns a list of versions for
# the requested project.
#
#####################################################

sub FindVersions
{
	my $location = shift;
	debug("Looking for project versions here: $location\n");

	opendir VH, $location or die "Can not open $location:$!\n";
	foreach my $file (readdir VH)
	{
    	my $line = "";
    	next if $file =~ /^\.\.?$/;

		debug("$file\n");
		push @versions, $file;
	}
	closedir(VH);
	my $list = join ":", @versions;
	return($list);
}

# =================================
# Set a time string
# =================================

sub SetTime
{
    my $time = "";
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime();
	# Add date to return to allow future time out of known running jobs
	# now tracked in this version. "0.7.6"
    my $date = "";
    $mon = $mon + 1; 
    $year = $year + 1900;
	$date = "$mon/$mday/$year";
    $time = "$hour:$min:$sec";
    print "Time is <$date - $time>\n";
    return("$date-$time");
}

# =================================
# Set a date string 
# =================================     
        
sub SetDate
{       
    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime();
    my $date = "";
    $mon = $mon + 1; 
    $year = $year + 1900;
    $date = $mon . "_" . $mday . "_" . $year;
    return($date);
}

#
# Update Run File
#

sub UpdateRunFile
{
	my $runstore = shift;
	my $rundir = shift;
	my $kind = shift;
	my $projversion =shift;
	my $currenttime = shift;
	my $runfile = $runstore . "RUNS";
	my $runfileold = $runstore . "RUNS.old";
	my $runfilenew = $runstore . "RUNS.NEW";

	if( -f $runfile ) {
    	open(OLD,"<$runfile") || die "Can not open current runs<$runfile>:$!\n";
    	open(NEW,">$runfilenew") || die "Can not open new run file<$runfilenew>:$!\n";
	
    	# newest always first in list
    	print NEW "$rundir | $kind | $projversion | $currenttime \n";
    	while(<OLD>){
        	print NEW "$_";
    	}    
    	close(NEW);
    	close(OLD);
    	copy("$runfile", "$runfileold");
    	copy("$runfilenew", "$runfile");
	} else {
    	open(OLD,">$runfile") || die "Can not open current runs<$runfile>:$!\n";
	
    	# newest always first in list
    	print OLD "$rundir | $kind | $projversion | $currenttime \n";
    	close(OLD);
	}
}


# =================================
# We currently have two crons.
#	checkprogress.cron
#		periodically runs a report and makes 
#		a profile of the last run for the project
#	continuous.cron
#		starts a repeat reset run of all 
#		current data sets
# =================================     

my $statcron = "checkprogress.cron";
my $continuous = "continuous.cron";

sub AddToContinuousCron
{
}

##############################################################
#
# GetSyncLock
#
# Many project have something to do for the first job but with
# many prejobs potentially running in race conditions we wanted
# a way allow only one to do that work before leaving a tracer
# that the work is now done for the others trying also.
#
# This is done in the run location so we know its permission
# friendly
##############################################################


sub GetSyncLock
{
	my $lockfile = shift;

	debug("About to lock<$lockfile>\n");
	# Lock operations on append to file
	open( SYNCLOCK, ">$lockfile") or die "Failed to open<$lockfile>:$!\n";
	while(! flock(SYNCLOCK, LOCK_EX)) {
		print "Waiting to lock and do preliminary init\n";
		sleep 1;
	}

	debug("Locked $lockfile\n");
}

sub ReleaseSyncLock
{
	my $lockfile = shift;
	#free lock
	flock(SYNCLOCK, LOCK_UN) or die "WARNING: Failed to unlock<$lockfile>:$!\n";
	debug( "UN-Locked $lockfile\n" );
	close(SYNCLOCK);
}

##############################################################
#
# GetHtmlLock
#
# As we work on auto-generating projects we also want to
# embed them into the master index.html file. However we
# can not control conflicts without locking.
#
##############################################################

sub GetHtmlLock
{
	my $lockfile = shift;

	debug("About to lock<$lockfile>\n");
	# Lock operations on append to file
	open( HTMLLOCK, ">$lockfile") or die "Failed to open<$lockfile>:$!\n";
	while(! flock(HTMLLOCK, LOCK_EX)) {
		print "Waiting to lock and do preliminary init\n";
		sleep 1;
	}

	debug("Locked $lockfile\n");
}

sub ReleaseHtmlLock
{
	my $lockfile = shift;
	#free lock
	flock(HTMLLOCK, LOCK_UN) or die "WARNING: Failed to unlock<$lockfile>:$!\n";
	debug( "UN-Locked $lockfile\n" );
	close(HTMLLOCK);
}

sub AppendToFile
{
	# assume these are cron executables and make the new files 755
	my $text = shift;
	my $file = shift;
	my $lockfile = shift;
	my $tmpfile = $file . ".tmp";

	debug("About to lock<$lockfile> to change<$file>\n");
	# Lock operations on append to file
	open( APPLOCK, ">$lockfile") or die "Failed to open<$lockfile>:$!\n";
	while(! flock(APPLOCK, LOCK_EX)) {
		print "Waiting to lock profile and report file\n";
		sleep 1;
	}

	debug("Locked $lockfile\n");

	# copy existing data to new file
	open( TAR, "<$file") or die "Failed to open<$file>:$!\n";
	open( NEWTAR, ">$tmpfile") or die "Failed to open<$tmpfile>:$!\n";
	while(<TAR>) {
		print NEWTAR "$_";
	}
	close(TAR);

	# append new data to file
	print NEWTAR "$text\n";
	close(NEWTAR);
	copy("$file", "$file.old") or die "copy $file to $file.old failed:$!\n";
	copy("$tmpfile", "$file") or die "copy $tmpfile to $file failed:$!\n";
	runcmd("chmod 755 $file",{emit_output=>0});
	runcmd("rm $tmpfile",{emit_output=>0});

	#free lock
	flock(APPLOCK, LOCK_UN) or die "WARNING: Failed to unlock<$lockfile>:$!\n";
	debug( "UN-Locked $lockfile\n" );
	close(APPLOCK);
}

sub CutRunFromCron
{
	# assume these are cron executables and make the new files 755
	my $run = shift;
	my $file = shift;
	my $lockfile = shift;
	my $tmpfile = $file . ".tmp";

	debug( "About to lock<$lockfile> to change<$file> for run<$run>\n" );
	print "About to lock<$lockfile> to change<$file> for run<$run>\n" ;
	# Lock operations on append to file
	open( LOCK, ">$lockfile") or die "Failed to open<$lockfile>:$!\n";
	while(! flock(LOCK, LOCK_EX)) {
		print "Waiting to lock profile and report file\n";
		sleep 2;
	}
	debug( "Locked $lockfile\n" );

	# copy existing data to new file
	open( TAR, "<$file") or die "Failed to open<$file>:$!\n";
	open( NEWTAR, ">$tmpfile") or die "Failed to open<$tmpfile>:$!\n";
	my $line = "";
	while(<TAR>) {
		chomp();
		$line = $_;
		if($line =~ /^.*$run.*$/) {
			# don't copy lines for this run
		} else {
			print NEWTAR "$line\n";
		}
	}
	close(TAR);

	close(NEWTAR);
	copy("$file", "$file.old") or die "copy $file to $file.old failed:$!\n";
	copy("$tmpfile", "$file") or die "copy $tmpfile to $file failed:$!\n";
	runcmd("chmod 755 $file",{emit_output=>0});
	runcmd("rm -f $tmpfile",{emit_output=>0});

	#free lock
	flock(LOCK, LOCK_UN) or die "WARNING: Failed to unlock<$lockfile>:$!\n";
	debug( "UN-Locked $lockfile\n" );
	close(LOCK);
}

        
#####################################################

#
# InstallDerivedVersion($srcloc,$codedir,$version,$preversion,$newproject);
#
#####################################################

sub InstallDerivedVersion
{
	my $srcloc = shift;
	my $codedir = shift;
	my $newversion = shift;
	my $srcversion = shift;
	my $newproject = shift;
	my $homedir = getcwd();

	print "Home in InstallDerivedVersion<$homedir>\n";

	# ensure all the key directories exist for a project

	print "Calling  MakeNewProject ($newproject)\n";

	MakeNewProject($newproject);
	
	my $sourceloc = LookupFsConfig("SOURCE");
	my $newloc = "$sourceloc/$newproject/$newversion";
	my $newcode = "$newloc/code";
	my $newjobs = "$newloc/jobs";
	my $oldjobs = "$srcloc/$srcversion/jobs";
	my $oldcode = "$srcloc/$srcversion/code";

	# Make sure reference version exists
	# and the code to install
	debug("Installing: create project <$newproject> from <$srcloc> vs <$srcversion >\n");

	if( -d $newloc) {
		print "Requested new version <$newversion> already exists. Die\n";
		exit(1);
	}

	if(!( -d $oldjobs)) {
		print "Source for job scripts<$oldjobs> does not exist!!!!!!... Die\n";
		exit(1);
	}

	if(!( -d $oldcode)) {
		print "Source for job scripts<$oldcode> does not exist!!!!!!... Die\n";
		exit(1);
	}

	debug("Create: $newloc $newcode $newjobs\n");
	runcmd("mkdir -p $newloc",{emit_output=>0});
	runcmd("mkdir -p $newcode",{emit_output=>0});
	runcmd("mkdir -p $newjobs",{emit_output=>0});

	# get new code in place
	runcmd("cp $oldcode/* $newcode",{emit_output=>0});
	# place a marker for the .htaccess file
	runcmd("touch $newcode/.htaccess",{emit_output=>0});

	# copy job scripts to new version
	runcmd("cp $oldjobs/* $newjobs",{emit_output=>0});
}

#####################################################
#
# DeriveNewVersion($srcloc,$version,$preversion); 
#
# We have code updates but have been requested to make
# it a new version.
#
#####################################################

sub DeriveNewVersion
{
	my $srcloc = shift;
	my $srcversion = shift;
	my $newversion = shift;
	my $homedir = getcwd();

	print "Home in DeriveNewVersion<$homedir>\n";

	chdir("$srcloc");
	my $newloc = "$srcloc/$newversion";
	my $oldloc = "$srcloc/$srcversion";

	my $derivecmd = "cp -r $oldloc $newloc"; 
	# Make sure reference version exists
	# and the code to install
	debug("Installing: srcloc is $srcloc\n");

	if( -d $newloc) {
		print "Requested new version <$newversion> already exists. Die\n";
		exit(1);
	}

	# Duplicate old version to new location
	runcmd($derivecmd);

	chdir("$homedir");
}

#####################################################
#
# Make New Project ensures all proper directories exist
# and it does not squawk if project exists.
#
#		MakeNewProject($project);
#
#####################################################

sub MakeNewProject
{
	my $newproject = shift;

	debug("MakeNewProject: newproject <$newproject>\n");
	my $tmp = LookupFsConfig("CONDORRUNS");
	if( $tmp eq "" ) {
		die "MakeNewProject failed to lookup CONDORRUNS\n";
	}
	my $projrunloc = $tmp . "/" . $newproject;

	$tmp = LookupFsConfig("RESULTS");
	if( $tmp eq "" ) {
		die "MakeNewProject failed to lookup RESULTS\n";
	}
	my $projresultsloc = $tmp . "/" . $newproject;

	$tmp = LookupFsConfig("TARCACHELOC");
	if( $tmp eq "" ) {
		die "MakeNewProject failed to lookup TARCACHELOC\n";
	}
	my $projtarcache = $tmp . "/" . $newproject;

	$tmp = LookupFsConfig("SOURCE");
	if( $tmp eq "" ) {
		die "MakeNewProject failed to lookup SOURCE\n";
	}
	my $projsources = $tmp . "/" . $newproject;

	$tmp = LookupFsConfig("TRACKING");
	if( $tmp eq "" ) {
		die "MakeNewProject failed to lookup TRACKING\n";
	}
	my $projtracking = $tmp . "/" . $newproject;

	#print "projrunloc <$projrunloc>\n";
	#print "projresultsloc <$projresultsloc>\n";
	#print "projtarcache <$projtarcache>\n";
	#print "projsources <$projsources>\n";
	#print "projtracking <$projtracking>\n";
	runcmd("mkdir -p $projrunloc",{emit_output=>0});
	runcmd("mkdir -p $projresultsloc",{emit_output=>0});
	runcmd("mkdir -p $projtarcache",{emit_output=>0});
	runcmd("mkdir -p $projsources",{emit_output=>0});
	runcmd("mkdir -p $projtracking",{emit_output=>0});

	if((!(-d $projrunloc)) || (!(-d $projresultsloc))) {
		die "Trouble establishing either $projrunloc or $projresultsloc\n";
	}
	if((!(-d $projtarcache)) || (!(-d $projsources))) {
		die "Trouble establishing either $projtarcache or $projsources\n";
	}
}

#####################################################
#
# FindRunning
#
# If we are doing new jobs, we want to exclude currently 
# running jobs so collect all jobs in .running files.
# Find them by reading through the RUN file for run directories.
#
# Take care that jobs are removed with soar_rm(not written)
# so that jobs have a couple things done when they are removed.
# 1. The continuous report and profiling must be stopped
# 2. The .running file must be removed
# 
# Note an automatic pair of entries are created for each run
# such that a profile and a report will(not done yet)
# happen thus allowing a constant updating of the jobs
# currently running.
#####################################################

sub FindRunning
{
	my $location = shift;
	my $project = shift;
	my $version = shift;
	my $resultref = shift;
	my $lastrunfile = "";
	my @collectrunning;

	if(defined $version) {
		print "Looking for running jobs for <$project>/<$version>\n";
	}
	my $runfile = $location . "/" . $project . "/RUNS";
	my $runningfilebase = $location . "/" . $project . "/";
	my $runningfile = $location . "/" . $project . "/";
	open(RUN,"<$runfile") || die "Can not open run file<$runfile> for finding most recent run:$!\n";
	my $latest = "";
	my @runstuff = ();
	while(<RUN>) {
		chomp();
		$latest = $_;
		$latest =~ s/ //g;
		#print "<<<<<<<<< $latest >>>>>>>>>>>>>>\n";
		# current format  run, kind, version, time
		@runstuff = split /\|/, $latest;
		#print "<<<<<<<<<$runstuff[0],$runstuff[1],$runstuff[2],$runstuff[3]>>>>>>>>\n";
		if($runstuff[2] eq $version) {
			#print "$runstuff[0] matches version\n";
			$runningfile = "$runningfilebase$runstuff[0]/$runstuff[0].running";
			if(( -f $runningfile ) &&($runstuff[1] ne "reset")) {
				@collectrunning = ();
				print "Runnning file exists<$runningfile>\n";
				ReadRunning($runningfile,\@collectrunning);
				foreach my $job (@collectrunning) {
					push @{$resultref}, $job;
				}
			}
		}
	}
	close(RUN);
}

#####################################################
#
# Find the latest run of a project and return environment 
# file to use.
#
# This version uses the Project Wide RUNS file
# in the CONDORRUNS/project directory
#
#####################################################

sub FindLatestRun
{
	my $location = shift;
	my $project = shift;
	my $version = shift;
	my $lastrunfile = "";

	if(defined $version) {
		print "Looking for last run for version <$version>\n";
	}
	my $runfile = $location . "/" . $project . "/RUNS";
	open(RUN,"<$runfile") || die "Can not open run file<$runfile> for finding most recent run:$!\n";
	my $latest = "";
	my @runstuff = ();
	while(<RUN>) {
		chomp();
		$latest = $_;
		$latest =~ s/ //g;
		print "<<<<<<<<< $latest >>>>>>>>>>>>>>\n";
		# current format  run, kind, version, time
		@runstuff = split /\|/, $latest;
		print "<<<<<<<<<$runstuff[0],$runstuff[1],$runstuff[2],$runstuff[3]>>>>>>>>\n";
		if(defined $version) {
			if($runstuff[2] eq $version) {
				$latest = $runstuff[0];
				print "Latest for version $version is $latest\n";
				last;
			}
		} else {
			$latest = $runstuff[0];
			last;
		}
	}
	close(RUN);
	$lastrunfile = $location . "/" . $project . "/ENVVARS$latest";
	print "Most recent run: $lastrunfile\n";
	return($lastrunfile);
}

sub DropRunning
{
	my $file = shift;
	my $dataref = shift;
	my @testarray;
	#print "Dropping running file<$file>\n";
	open(RUN,">$file") or die "Can not open for W($file):$!\n";
	while(! flock(RUN, LOCK_EX)) {
		print "Waiting to lock running file\n";
		sleep 1;
	}
	foreach my $run (@{$dataref}) {
		print RUN "$run\n";
	}
	flock(RUN, LOCK_UN) or die "WARNING: Failed to unlock<$file>:$!\n";
	close(RUN);
}

sub ReadRunning
{
	my $file = shift;
	my $dataref = shift;
	print "Reading running file<$file>\n";
	open(RRUN,"<$file") or die "Can not open for R($file):$!\n";
	while(! flock(RRUN, LOCK_EX)) {
		print "Waiting to lock running file\n";
		sleep 1;
	}
	while(<RRUN>) {
		chomp();
		push @{$dataref}, $_;
		debug("$_ found still running\n");
	}
	flock(RRUN, LOCK_UN) or die "WARNING: Failed to unlock<$file>:$!\n";
	close(RRUN);
}

#####################################################
#
# locking code
#
# We are going to have the report writing also drop
# those jobs which we suspect are still running though
# they may have ended badly. But since the control script
# will be checking it when asked to do all the new jobs
# (--kind=new) we will lock(WaitForLock) the running file 
# before we write to it and then unlock it when we are done.
#
# The control script will always call the WaitForLock
# and then FileUnlock to keep it from being changed in 
# the middle.
#
#####################################################

#####################################################
#
#	VM Universe code for VmWare
#
#	Initializing the VM for a job and accessing the
#	results are done by including a suitable second
#	VmWare disk. This disk is created for dynamic
#	allocation and created in a VM which can use fdisk
#	to create a fat32 partition and must have an
#	mkfs which allows a "-t vfat" to complete creation
#	of the disk.
#
#	Once one has this disk it can be used as a template
#	second drive for any VmWare VM. Note keep the core
#   empty disk and copy it when needed as it gets
#	larger with every use.
#
#	We rely on "mtools" and "qemu" for the rest.
#	"qemu-img" is used to convert the disk back and forth
#	between vmdk and raw formats. "mcopy, mdir etc"
#	are used to place and fetch files off the raw
#	disk.
#
#####################################################

sub VMwareAddFile
{
    my $file = shift;
    runcmd("mcopy $file n:");
    runcmd("mdir n:");
}

sub VMwareFetchByPattern
{
    my $pattern = shift;
    runcmd("mcopy n:\*$pattern* .");
    runcmd("mdir n:");
}

sub VMwareRemakeDataDisk
{
    my $data = shift;
    my $raw = shift;

    my $cmd = "qemu-img convert -f raw $raw -O vmdk $data";
    runcmd($cmd);
}

sub VMwareCreateRawDisk
{
    my $empty = shift;
    my $raw = shift;

    my $cmd = "qemu-img convert -f vmdk $empty -O raw $raw";
    runcmd($cmd);
}

sub VMwareCreateMTOOLSRC
{
    my $rawdisk = shift;
    my $top = getcwd();
    my $baseentry = "drive n: file=\"$top/$rawdisk\" offset=32256 mtools_skip_check=1";
    print "Making MTOOLSRC here<$top>\n";
    open(MTRC,">$top/mtoolsrc") or die "Can not open mtoolsrc in <$top>:$!\n";
    print MTRC "$baseentry\n";
    close(MTRC);
    $ENV{MTOOLSRC} = "$top/mtoolsrc";
    runcmd("mdir n:");
}

#####################################################
#
# Code to allow projects to change out their analysis
# objects.
#
#####################################################


sub UpdateProjectObjects
{
	my $newobjloc = shift;
	my $projsrcloc = shift;
	my $projversion = shift;
	my $configfile = shift;
	my $optionargs = shift;
	my $codedir = "$projsrcloc/$projversion/code";
	my $cwd = getcwd();
	my %failreturn;
	my $didupdate = 0;


	#DebugOn();
	debug("Call ParseOptions on <$configfile>\n");
	ParseOptions($configfile,\%updateoptions);
	debug("Back from ParseOptions(update options now:\n");

	if($updateoptions{success} == 0) {
		print "Sorry, Code updates had a parsing issue with objconfig file.\n";
		$failreturn{"success"} = 0;
		return \%failreturn;
	}
	#DebugOff();

	foreach my $key (keys %updateoptions) {
		debug("key: $key value $updateoptions{\"$key\"}\n");
	}

	debug("Moving on in UpdateProjectObjects\n");
	debug("Checking for files in obj location<$newobjloc>\n");
	#runcmd("ls -l $newobjloc");
	debug("Compare against code location<$codedir>\n");
	#runcmd("ls -l $codedir");
	debug("Test for config file\n");
	if( -f $configfile ) {
		debug("Yeah. We have needed config file which follows\n");
		#runcmd("cat $configfile");
	} else {
		debug("Oops. no config file for project object updates\n");
	}

	# can we get at new objects?
	if((-d $newobjloc) && (!(-R $newobjloc))) {
		print "Sorry, Code updates require more open permissions\n";
		$failreturn{"success"} = 0;
		return \%failreturn;
	}

	chdir("$newobjloc");

	my $newM;
	my $oldM;
	my $tmppath;
	my $tmpnew;
	my $newfilesseen = "$codedir/NEWFILES"; # have seen an instance where after
							# a cp of new files there was a remaining COMPILEDONE
							# thus leading to no compile
	my $matlabseen = "no";
	my @movethese;

	opendir DS, "." or die "Can not open dataset<$1>\n";
	foreach my $subfile (readdir DS)
	{
    	next if $subfile =~ /^\.\.?$/;
		debug("examine: $subfile\n");
		if(-d "$subfile") {
			debug("skip directory <$subfile>\n");
		} else {
			debug("considering <$subfile>\n");
			$tmpnew = "$subfile";
			next if (!(-f $tmpnew));
			$newM = -M "$subfile";
			$tmppath = "$codedir/$subfile";
			# Changes for compile and come back require
			# that even if we have moved the code, we come
			# back and check either a Local Universe
			# build or one done with Condor-C. So we will
			# always trigger "$matlabseen". We will only
			# trigger the onthefly build if we see newer
			# files but the other two builds we will revisit
			# if local flag COMPILING is seen.
			# we'll no to not bother with a local onthefly
			# build if the "$movethese" is empty.

			debug("does old version <$tmppath> exist?");
			if($subfile =~ /^[\w\d_]+\.m$/) {
				$matlabseen = "yes";
				debug("Marking matlabseen!!!!!\n");
			}

			if(!(-f $tmppath)) {
				debug("Nope......... New file, add it in\n");
				push @movethese, $tmpnew;
			} else {
				$oldM = -M "$tmppath";
				debug("New <$subfile> $newM old $oldM\n");
				if($newM < $oldM) {
					$didupdate += 1;
					debug("We should update <$subfile>\n");
					push @movethese, $tmpnew;
				}
			}
		}
	}
	closedir(DS);
	my $filestomove = @movethese;
	debug("Filetomove set to <<$filestomove>>\n");

	my $copytoversion = "";

	$copytoversion = $projversion;

	# Now copy replacement files
	debug("Copy files..........\n");
	my $copytocmd = "";
	foreach my $file (@movethese) {
		debug("Copy $file\n");
		runcmd("cp $file $codedir");
	}

	if($filestomove > 0) {
		# indication for local or condor-c builds
		# to ignore last result and initiate another
		# compile.
		runcmd("touch $newfilesseen");
	}

	# back where we started
	chdir("$cwd");

	# do we have to recompile??????
	# Realy 2 cases. One where we did an on the fly
	# last time and the other where we copied
	# the files and started a local or condorc build.
	# in the latter 2 cases, we need to check for
	# the compile being done before moving on
	# or we will always be one off on the compiles

	if(($matlabseen  eq "yes") && ($filestomove > 0) && (${$optionargs}{"build"} eq "onthefly")) {
		my $hashref;
		my $compiletimeout = 10;
		my $compiletimer = 0;
		# remove trigger for returning to check compile
		runcmd("rm -f $newfilesseen");
		debug("about to process matlab re-compile directives\n");
		print "about to process matlab re-compile directives\n";
		chdir("$codedir");
		#runcmd("pwd;ls -lt");

		my $mcccmd = "";
		my $display;
		my $done = 0;
		if(exists $updateoptions{"matlabtarg"}) {
			my $realupdates = @movethese;
			# we have matlab and changed objects
			my @mcctargets = split /,/, $updateoptions{"matlabtarg"};
			debug("There are <$#mcctargets> matlab compile objects\n");
			$ENV{PATH} = "/home/gcc423/bin:$ENV{PATH}";
			#$ENV{DISPLAY} = $display;
			#runcmd("which gcc");
			foreach my $target (@mcctargets) {
				$mcccmd = "mcc -m -R -nodisplay -R -nojvm -nocache $target";
				debug("Compile $target with cmd <$mcccmd>\n");
				$done = 0;
				# allow for the compile to fail, retry.... for 40 minutes every 5;
				while( $done == 0) {
					if($compiletimer > $compiletimeout) {
						print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
						print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
						print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
						print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
						print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
						die "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
					} else {
						$hashref = runcmd($mcccmd,{expect_result=>\&ANY});
						#look at exit value
						my $exitvalue = ${$hashref}{exitcode};
						print "Exit code from <$target> is <$exitvalue>\n";
						if($exitvalue != 0) {
							print "Waiting 5 minutes to retry matlab compiler\n";
							sleep 300; # 5 minutes
							$compiletimer += 5; # add 5 minutes
						} else {
							print "Compile of <$target> is good\n";
							$done = 1; # mark done and move on to others.
							$compiletimer = 0; # reset compile time
						}
					}
				}
			}
		} else {
			die "New matlab code but \"matlabtarg\" not set on objconfig file\n";
		}
	} elsif(($matlabseen  eq "yes") && ((-f "$newfilesseen") || ($filestomove > 0))) {
		# two cases here.... possibly an old build completing
		# and/or new code before using last compile.
		# if we moved files, remove marker for prior compiles
		# being done forcing new compile. In either case
		# start/check last compile/this compile
		#debug("No new matlab files or any changed files seen\n");
		#debug("Therefore no compile\n");
		chdir("$codedir");
		if(${$optionargs}{"build"} eq "local") {
			debug("Checking on Local Universe build Need..../results\n");
			my $compiledone = LocalMatlabBuild("$codedir",$updateoptions{"matlabtarg"});
			if($compiledone == 0) {
				print "Compile has finished. Moving on.........................\n";
			} else {
				print "Will come back and check Matlab compile\n";
DebugOff();
				exit(0);
			}
		} elsif(${$optionargs}{"build"} eq "condorc") {
			debug("Checking on Copndor-C build Need..../results\n");
			my $matlabsources = "matlabsources.tar.gz";
			runcmd("tar -zcvf $matlabsources *",{expect_output=>1});
			my $compiledone = CondorCMatlabBuild("$codedir",$updateoptions{"matlabtarg"},$matlabsources,$optionargs);
			if($compiledone == 0) {
				print "Compile has finished. Moving on.........................\n";
			} else {
				print "Will come back and check Matlab compile\n";
DebugOff();
				exit(0);
			}
		}
	}

	chdir("$cwd");

	DebugOff();
	print "UpdateProjectObjects updated $didupdate objects\n";
	if($didupdate == 0) {
		# we don't want to act on old options for an old update check
		$failreturn{"success"} = 1;
		return \%failreturn;
	}
	return \%updateoptions
}

# Local one are simpler submit file constructs since
# no file transfer is needed.

sub LocalMatlabBuild
{
	my $compilelocation = shift;
	my $matlabtargets = shift;
	my $sourceloc = LookupFsConfig("SOURCE");
	my $submittemplate = $sourceloc . "/soar_resources/localu/mcclocal.template";
	my $buildtool = $sourceloc . "/soar_resources/build/mcccompile.pl";
	my $submitfile = "localu.sub";

	debug("In LocalMatlabBuild for loc <$compilelocation> and target <$matlabtargets>\n");
	# are we a new compile or checking one
	runcmd("date");
	if(-f "COMPILEDONE") {
		runcmd("rm -f COMPILEDONE");
		print "Last compile request worked\n";
		return(0);
	} elsif(-f "RETRYCOMPILE") {
		print "Last attempt failed, trying again.\n";
		runcmd("rm -f RETRYCOMPILE");
	} else {
		print "New compile request\n";
	}

	# get compile tool
	runcmd("cp $buildtool .",{emit_output=>1});

	# create submit file
	my $line = "";
	open(LOC,">$submitfile") or die "Could not open new submit file<$submitfile>:$!\n";
	open(TEM,"<$submittemplate") or die "Opening MCC local template<$submittemplate> Failed:$!\n";
	while(<TEM>) {
		chomp();
		$line = $_;
		if($line =~ /^\s*arguments.*$/) {
			print LOC "arguments = $matlabtargets\n";
		} else {
			print LOC "$line\n";
		}
	}
	close(TEM);
	
	# ok submit and return a 1 saying in progress
	runcmd("condor_submit $submitfile",{emit_output=>1});
	return(1);
}

sub CondorCMatlabBuild
{
	my $compilelocation = shift;
	my $matlabtargets = shift;
	my $matlabsources = shift;
	my $optionargs = shift;
	my $sourceloc = LookupFsConfig("SOURCE");
	my $submittemplate = $sourceloc . "/soar_resources/condorc/mcccondorc.template";
	my $buildtool = $sourceloc . "/soar_resources/build/mcccompile.pl";
	my $submitfile = "condorc.sub";

debug("compilelocation = <$compilelocation>, matlabtargets = <$matlabtargets>, matlabsources = <$matlabsources>\n");
#${$optionargs}{"build"}

	debug("In LocalMatlabBuild for loc <$compilelocation> and target <$matlabtargets>\n");
	# are we a new compile or checking one
	system("date");
	if( -f "NEWFILES") {
		runcmd("rm -f COMPILEDONE RETRYCOMPILE NEWFILES"); # reset to force new compile
	} elsif(-f "COMPILEDONE") {
		runcmd("rm -f COMPILEDONE");
		print "Last compile request worked\n";
		return(0);
	} elsif(-f "RETRYCOMPILE") {
		print "Last attempt failed, trying again.\n";
		runcmd("rm -f RETRYCOMPILE");
	} else {
		print "New compile request\n";
	}

	# get compile tool
	runcmd("cp $buildtool .",{emit_output=>1});

	# create submit file
	my $line = "";
	open(LOC,">$submitfile") or die "Could not open new submit file<$submitfile>:$!\n";
	open(TEM,"<$submittemplate") or die "Opening MCC local template<$submittemplate> Failed:$!\n";
	while(<TEM>) {
		chomp();
		$line = $_;
		if($line =~ /^\s*arguments.*$/) {
			print LOC "arguments = $matlabtargets\n";
		} elsif($line =~ /^\s*transfer_input_files.*$/) {
			print LOC "transfer_input_files = $matlabtargets,$matlabsources\n";
		} elsif($line =~ /^\s*grid_resource.*$/) {
			print LOC "grid_resource = ${$optionargs}{grid_resource}\n";
		} else {
			print LOC "$line\n";
		}
	}
	close(TEM);
	
	# ok submit and return a 1 saying in progress
	runcmd("condor_submit $submitfile",{emit_output=>1});
	return(1);
}

sub LocalRBuild
{
	my $compilelocation = shift;
	my $rtargets = shift;
	my $sourceloc = LookupFsConfig("SOURCE");
	my $submittemplate = $sourceloc . "/soar_resources/localu/Rlocal.template";
	my $buildtool = $sourceloc . "/soar_resources/build/PrepareRLibs.pl";
	my $Rprebuilt = $sourceloc . "/soar_resources/R/built-R-2.10.1.tar.gz";
	my $submitfile = "localu.sub";

	debug("In LocalRBuild for loc <$compilelocation> and target <$rtargets>\n");
	# are we a new compile or checking one
	system("date");
	if(-f "COMPILEDONE") {
		runcmd("rm -f COMPILEDONE");
		print "Last compile request worked\n";
		return(0);
	} elsif(-f "RETRYCOMPILE") {
		print "Last attempt failed, trying again.\n";
		runcmd("rm -f RETRYCOMPILE");
	} else {
		print "New compile request\n";
	}

	# get compile tool
	runcmd("cp $buildtool .",{emit_output=>1});
	runcmd("cp $Rprebuilt .",{emit_output=>1});

	# create submit file
	my $line = "";
	open(LOC,">$submitfile") or die "Could not open new submit file<$submitfile>:$!\n";
	open(TEM,"<$submittemplate") or die "Opening MCC local template<$submittemplate> Failed:$!\n";
	while(<TEM>) {
		chomp();
		$line = $_;
		if($line =~ /^\s*arguments.*$/) {
			print LOC "arguments = $rtargets\n";
		} else {
			print LOC "$line\n";
		}
	}
	close(TEM);
	
	# ok submit and return a 1 saying in progress
	runcmd("condor_submit $submitfile",{emit_output=>1});
	return(1);
}

sub CondorcRBuild
{
	my $compilelocation = shift;
	my $rtargets = shift;
	my $rpackagesources = shift;
	my $optionargs = shift;
	my $version = shift;
	my $sourceloc = LookupFsConfig("SOURCE");
	my $submittemplate = $sourceloc . "/soar_resources/condorc/rcondorc.template";
	my $buildtool = $sourceloc . "/soar_resources/build/PrepareRLibs.pl";
	my $Rprebuilt = $sourceloc . "/soar_resources/R/";
	my $submitfile = "condorc.sub";

	my %versionTarBalls =  
			(
			"R-2.10.1" => "built-R-2.10.1.tar.gz",
			"sl5-R-2.10.1" => "built-sl5-R-2.10.1.tar.gz",
			"R-2.11.1" => "built-R-2.11.1.tar.gz",
			"R-2.12.1" => "built-R-2.12.1.tar.gz",
			"R-2.12.1" => "built-R-2.12.1.tar.gz",
			"win32-R-2.10.1" => "built-win32-R-2.10.1.tar.gz",
			);

	my %versionRunDirs =  
			(
			"R-2.10.1" => "built-R-2.10.1.tar.gz",
			"sl5-R-2.10.1" => "R-2.10.1",
			"R-2.11.1" => "built-R-2.11.1.tar.gz",
			"R-2.12.1" => "built-R-2.12.1.tar.gz",
			"R-2.12.1" => "built-R-2.12.1.tar.gz",
			"win32-R-2.10.1" => "built-win32-R-2.10.1.tar.gz",
			);

	$Rprebuilt = "$Rprebuilt" . "$versionTarBalls{$version}";
	my $versionrunlocation = "$versionRunDirs{$version}";
	my $RprebuiltTarBall = "$versionTarBalls{$version}";
	debug("In CondorcRBuild for loc <$compilelocation> and target <$rtargets> version <$version>\n");
	debug("Tar ball for <$version> is <$Rprebuilt>\n");
	# are we a new compile or checking one
	system("date");
	if(-f "COMPILEDONE") {
		runcmd("rm -f COMPILEDONE");
		print "Last compile request worked\n";
		return(0);
	} elsif(-f "RETRYCOMPILE") {
		print "Last attempt failed, trying again.\n";
		runcmd("rm -f RETRYCOMPILE");
	} else {
		print "New compile request\n";
	}

	# get compile tool
	runcmd("cp $buildtool .",{emit_output=>1});
	runcmd("cp $Rprebuilt .",{emit_output=>1});

	# create submit file
	my $line = "";
	open(LOC,">$submitfile") or die "Could not open new submit file<$submitfile>:$!\n";
	open(TEM,"<$submittemplate") or die "Opening MCC local template<$submittemplate> Failed:$!\n";
	while(<TEM>) {
		chomp();
		$line = $_;
		if($line =~ /^\s*arguments.*$/) {
			print LOC "arguments = $rtargets $RprebuiltTarBall $versionrunlocation\n";
		}elsif($line =~ /^\s*transfer_input_files.*$/) {
			print LOC "transfer_input_files = $rtargets,$rpackagesources,$Rprebuilt\n";
		} elsif($line =~ /^\s*grid_resource.*$/) {
			print LOC "grid_resource = ${$optionargs}{grid_resource}\n";
		} else {
			print LOC "$line\n";
		}
	}
	close(TEM);
	
	# ok submit and return a 1 saying in progress
	runcmd("condor_submit $submitfile",{emit_output=>1});
	return(1);
}

############################################################################
#
#	BASEMODE: Though SOAR has a rich set of command line options
#	it's main purpose is to sweep through requested projects and to
#	submit DAGs for new jobs as the jobs get their input placed
#	as needed for the trigger to occur.
#
#	BaseMode: is used to explore some more basic jobs to be handled by
# 	SOAR using a dedicated location trigger and a definition file.
#
#   There are 4 basic phases for a generic project.
#		Make location
#		Make submit templates
#		populate code and jobs locations
#		tweak glue
############################################################################

my $generictype = ""; # R or Matlab
my $generictarball = "";
my $generictopdir = ""; # rversions have different directory trees as do matlab

sub GenerateGeneric
{
	my $baselocation = shift;
	my $optionargs = shift;
	my $optionfile = $baselocation . "/NEWPROJECT";
	my $mastersrcloc = "";
	my($genericfolder, $toplocation, $suffix) = fileparse($baselocation);

	#DebugOn();
	debug(" In GenerateGeneric for location <$baselocation>\n");
	debug("Test for NEWPROJECT file\n");
	if(-f $optionfile) {
		GenericReadWriteTest($baselocation,"NEWPROJECT","TESTWRITE");
		debug("Commence checking new project definition <$optionfile>\n");
		debug("Call ParseOptions on <$optionfile>\n");
		ParseOptions($optionfile,\%genericprojectoptions);
		debug("Back from ParseOptions(update options now:\n");
	} else {
		die "Improper generic project request: Missing NEWPROJECT file in <$baselocation>\n";
	}
	my $result = ValidateGenericProjectRequest(\%genericprojectoptions);
	if($result == 1) {
		die "Bad Generic Project Config File!\n";
	}
	# Make the project location
	$mastersrcloc = GenerateGenericProjectLocation(\%genericprojectoptions);
	debug("Told it is going here:<$mastersrcloc>\n");
	# Populate Project
	PopulateGenericProject(\%genericprojectoptions,$mastersrcloc,$baselocation,$optionargs);
	# Make the job submit template
	GenerateGenericSubmitTemplates(\%genericprojectoptions,$mastersrcloc);
	# Hide Trigger Directory
	print "Leaving in GenerateGeneric to check GENERIC changes\n";
	runcmd("mv $baselocation $toplocation/$genericprojectoptions{Project}GENERIC",{emit_output=>1});
	#DebugOff();
}

sub ValidateGenericProjectRequest
{
	my $optionhashref = shift;
	my $mode = "good";

	# These MUST be there
	my @mandatory = (
		"AccountingGroup",
		"Department",
		"Project",
		"InputLocation",
		"PhaseOneName",
		"PhaseOneBinary",
		"PhaseOneType",
		"PhaseOneResults",
	);

	# only these entries are allowed in the NEWPROJECT file
	my %allowed = (
		"AccountingGroup" => 1,
		"Department" => 1,
		"Project" => 1,
		"InputLocation" => 1,
		"PhaseOneName" => 1,
		"PhaseOneBinary" => 1,
		"PhaseOneLibrary" => 1,
		"PhaseOneType" => 1,
		"PhaseOneResults" => 1,
		"PhaseOneArg" => 1,
		"PhaseOneExtraFiles" => 1,
		"SharedInputs" => 1,
		"success" => 1,
		"Options" => 1,
		"Version" => 1,
		"Universe" => 1, 	# defaults to vanilla
		"Arch" => 1,			# defaults to "x86_64"
		"Opsys" => 1,		# defaults to "Linix"
		"JobAds" => 1,		# defaults to not if specified its for any type
		"Memory" => 1,		# added to the requirements any type generic
		"Priority" => 1, 	# any type generic
		"Requirements" => 1, # trumps standard project "type" requirements
	);

	ViewOptionsHash($optionhashref);

	# Make sure all mandatory are specified
	for my $hashelement (@mandatory) {
		if(!(exists ${$optionhashref}{$hashelement})) {
			print "Key configuration parameter <$hashelement> missing from config file>\n";
			$mode = "failure";
		} else {
		}
	}

	# make sure every option is legal
	foreach my $key ( sort keys (%{$optionhashref}) ) {
		if(exists $allowed{$key}) {
		} else {
			print "Configuration parameter <$key> not legal>\n";
			$mode = "failure";
		}
	}

	if($mode eq "failure") {
		return(1);
	} else {
		debug("Generic Configuration file good\n");
		debug("Adding defaults\n");
		${$optionhashref}{Universe} = "vanilla";
		${$optionhashref}{Arch} = "\"x86_64\"";
		${$optionhashref}{Opsys} = "\"Linux\"";
		return(0);
	}
	ViewOptionsHash($optionhashref);
}

sub GenerateGenericProjectLocation
{
	my $optionhashref = shift;

	debug("Generate base files for new generic project\n");

	my $sources = LookupFsConfig("SOURCE");
	if( $sources eq "" ) {
		die "GenerateGenericProject failed to lookup SOURCE LOC\n";
	}

	my $newproject = "";
	if( exists ${$optionhashref}{Project} ) {
		$newproject = ${$optionhashref}{Project};
		debug("New generic requested:${$optionhashref}{Project}\n");
	} else {
		die "The existence of the project should have already been verified!\n";
	}

	my $cronupdate = "system(\"./control --project=${$optionhashref}{Project} --kind=new --version=v1\");\n";

	my $htmlloc = LookupFsConfig("HTMLLOC");
	if( $htmlloc eq "" ) {
		die "setting key locations failed to lookup TRACKING\n";
	}

	debug("About to append to continuous.cron<$cronupdate>\n");
	my $cronfile = "$htmlloc/continuous.cron";
	AppendToFile( $cronupdate, $cronfile, "CRON" );

	$sources = "$sources/$newproject";
	runcmd("mkdir -p $sources",{emit_output=>0});
	runcmd("mkdir -p $sources/v1",{emit_output=>0});
	runcmd("mkdir -p $sources/v1/code",{emit_output=>0});
	runcmd("mkdir -p $sources/v1/jobs",{emit_output=>0});
	return("$sources/v1");
}

sub PopulateGenericProject
{
	my $hashref = shift;
	my $projectlocation = shift;
	my $baselocation = shift;
	my $optionargs = shift;
	my $newprojectline = "${$hashref}{Project},${$hashref}{InputLocation}";
my $fsconfigfile = "/p/stat/soar/control/fsconfig";

	my $sources = LookupFsConfig("SOURCE");
	if( $sources eq "" ) {
		die "PopulateGenericProject failed to lookup SOURCE LOC\n";
	}

	my $control = LookupFsConfig("CONTROL");
	if( $sources eq "" ) {
		die "PopulateGenericProject failed to lookup CONTROL LOC\n";
	}

	$fsconfigfile = "$control/fsconfig";

	# we already verified Department, so add index entry for project
	CraftNewIndex(${$hashref}{Department},${$hashref}{Project},"index.html");

	# add projects input location
	AppendToFile( $newprojectline, $fsconfigfile, "FSCONFIG" );

	$sources = "$sources/soar_resources/";

	debug("PopulateGenericProject for project location $projectlocation\n");

	# everyone gets the same glue
	my $gluesrc = "$sources/glue/*";
	my $gluedest = "$projectlocation/jobs";
	runcmd("cp $gluesrc $gluedest",{emit_output=>0});
	# all we know about results to harvest is a comma separated list if patterns
	# so we have to change the postjob scripts to test for these patterns and
	# to copy to result locations things which match.
	FixPostJobResults("$projectlocation/jobs/",${$hashref}{PhaseOneResults});
	FixMakeDagText("$projectlocation/jobs/",${$hashref}{PhaseOneName});
	FixMakeSubmitText("$projectlocation/jobs/",${$hashref}{PhaseOneName},${$hashref}{PhaseOneType},${$hashref}{SharedInputs});

	# common across all project types
	# any extra files????? compiled binaries or whatever
	my $Rdest = "$projectlocation/code";
	if(${$hashref}{PhaseOneExtraFiles}) {
		my @extra = split /,/, ${$hashref}{PhaseOneExtraFiles};
		foreach my $tra (@extra) {
			runcmd("cp $baselocation/$tra $Rdest",{emit_output=>0});
		}
	}

	if(${$hashref}{PhaseOneType} eq "R") {
		PopulateRProject($hashref,$projectlocation,$baselocation,$sources,$optionargs);
	} elsif(${$hashref}{PhaseOneType} eq "Matlab") {
		PopulateMatlabProject($hashref,$projectlocation,$baselocation,$sources,$optionargs);
	} else {
		PopulateProgramProject($hashref,$projectlocation,$baselocation,$sources);
	}
}

sub PopulateRProject
{
	my $hashref = shift;
	my $projectlocation = shift;
	my $baselocation = shift;
	my $sources = shift;
	my $optionargs = shift;

	debug("Populate R\nSource location is <$baselocation> and destination is <$projectlocation> with common elements <$sources>\n");

	my %versionTarBalls =
	(
		"R-2.10.1" => "built-R-2.10.1.tar.gz",
		"sl5-R-2.10.1" => "built-sl5-R-2.10.1.tar.gz",
		"R-2.11.1" => "built-R-2.11.1.tar.gz",
		"R-2.12.1" => "built-R-2.12.1.tar.gz",
		"win32-R-2.10.1" => "built-win32-R-2.10.1.tar.gz",
	);

	my $version = ${$hashref}{Version};    
	my $Rtarball = $versionTarBalls{$version};

	# set up to pass key args to wrappers
	$generictype = "R";
	$generictarball = $Rtarball;
	if($version eq "sl5-R-2.10.1") {
		$generictopdir = "R-2.10.1";
	} else {
		$generictopdir = $version;
	}

	my $Rchecksum = "built-" . "$version" . "_input_checksum.txt";
	my $R_shlib_checksum = "$sources/R/built-R_slib_input_checksum.txt";
	my $Rprebuilt = "$Rtarball";

	# copy R working files and then any R scripts
	my $Rsrcs = "$sources/R/*";
	my $wrapper = "$sources/rungeneric.pl";
	$Rchecksum = "$sources/R/$Rchecksum";
	$Rprebuilt = "$sources/R/runtimes/$Rprebuilt";
	my $Rinstallfrom = "R-2.10.1";
	my $Rdest = "$projectlocation/code";
	runcmd("cp $Rsrcs $Rdest",{emit_output=>1,expect_result=>\&ANY});
	runcmd("cp $Rchecksum $Rdest/input_checksum.txt",{emit_output=>1});
	runcmd("cp $wrapper $Rdest",{emit_output=>1});
	runcmd("cp $R_shlib_checksum $Rdest",{emit_output=>1});
	runcmd("cp $baselocation/*.R $Rdest",{emit_output=>0});
	runcmd("touch $Rdest/.htaccess",{emit_output=>0});

	# we need to customize the r wrapper for the right version
	#
	# NOT anymore. we use rungeneric.pl for Matlab and R and
	# use extra args to make it work for either
	# bt 3/15/11
	#
	#my $template = "$sources/R/runRscript.template";
	#my $line = "";
	#my $newRwrapper = "$Rdest/Rnolib.pl";
	#open(TEMPLATE,"<$template") or die "Can not open template<$template>:$!\n";
	#open(WRAP,">$newRwrapper") or die "Can not open new R wrapper<$newRwrapper>:$!\n";
	#while(<TEMPLATE>) {
		#chomp();
		#$line = $_;
		#if($line =~ /^.*?(XXXXXX).*$/) {
			#print WRAP "my \$Rprebuilt = \"$Rtarball\";\n";
		#} elsif($line =~ /^.*?(YYYYYY).*$/) {
			#print WRAP "my \$Rinstallfrom = \"$version\";\n";
		#} else {
			#print WRAP "$line\n";
		#}
	#}

	# what about libraries?????
	my $home = getcwd();
	#DebugOn();
	if(exists ${$hashref}{PhaseOneLibrary}) {
		debug("********** R libraries to handle!!!!!!********** \n");
		# make marker file and tar archive
		chdir("$baselocation");
		my $buildspot = getcwd();
		my $Rinstallto = "$buildspot/R";
		################################
		if(${$optionargs}{"build"} eq "onthefly") {
			# grap prebuilt R and install it into the GENERIC
			# folder which initiated this action. Then use that
			# R to build packages needed.
			my @libraries = split /,/, ${$hashref}{PhaseOneLibrary};
			runcmd("mkdir -p RR");
			runcmd("mkdir -p RR/library");
			runcmd("cp $Rprebuilt .",{emit_output=>1});
			runcmd("tar -zxvf $Rprebuilt",{emit_output=>1});
			chdir("$Rinstallfrom");
			runcmd("make prefix=$Rinstallto install",{emit_output=>1});
			chdir("$baselocation");
			$ENV{R_HOME_DIR} = "$Rinstallto";
			$ENV{R_HOME} = "$Rinstallto";
			$ENV{LD_LIBRARY_PATH} = "$Rinstallto/lib";
			foreach my $lib (@libraries) {
		    	runcmd("$Rinstallto/bin/R CMD INSTALL --preclean -l ./RR/library $lib",{emit_output=>1});
			}
			print "Making tar archive......\n";
			runcmd("tar -zcvf RLIBS.tar.gz RR",{emit_output=>1});
			runcmd("cp RLIBS.tar.gz $Rdest");
		##############################
		} elsif (${$optionargs}{"build"} eq "localu") {
			# generate a local universe package build so
			# we can detect problems. Come back and check.
			my $compiledone = 1;
			my $target = ${$hashref}{PhaseOneLibrary};
			debug("Compile $target with cmd <LocalRBuild>\n");
			$compiledone = LocalRBuild("$baselocation",$target);
			if($compiledone == 0) {
				print "Compile has finished. Moving on.........................\n";
				runcmd("cp RLIBS.tar.gz $Rdest");
			} else {
				print "Will come back and check R compile\n";
				exit(0);
			}
		} elsif(${$optionargs}{"build"} eq "condorc") {
			my $compiledone = 1;
			my $localRsources = "Rsources.tar.gz";
			runcmd("tar -zcvf $localRsources *.tar.gz",{emit_output=>1});
			my $target = ${$hashref}{PhaseOneLibrary};
			my $reqversion = ${$hashref}{Version};
			debug("Compile $target with cmd <LocalRBuild>\n");
			$compiledone = CondorcRBuild("$baselocation",$target,$localRsources,$optionargs,$reqversion);
			if($compiledone == 0) {
				print "Compile has finished. Moving on.........................\n";
				runcmd("cp RLIBS.tar.gz $Rdest");
			} else {
				print "Will come back and check R compile\n";
				exit(0);
			}
		} else {
			my $buildtype = ${$optionargs}{"build"};
			die "UNKNOWN build option: onthefly, localu or condorc allowed: <$buildtype>\n";
		}
		chdir("$home");
	} else {
		debug("********** no R libraries to handle!!!!!!********** \n");
	}
}

sub PopulateMatlabProject
{
	my $hashref = shift;
	my $projectlocation = shift;
	my $baselocation = shift;
	my $sources = shift;
	my $optionargs = shift;
	my $windowsmatlab = 0;
	my @sharedsources = ("rungeneric.pl");
	my @linuxsources = ("input_checksum.txt");
	my @windowssources = ("matlab.bat", "cygiconv-2.dll", "cygintl-8.dll", "cygwin1.dll", "gzip.exe", "tar.exe", "w32v714.tar.gz");
	debug("Populate Matlab\nSource location is <$baselocation> and destination is <$projectlocation> with common elements <$sources>\n");

	my $currentloc = getcwd();

	# set up to pass key args to wrappers for matlab they are static for now
	$generictype = "Matlab";
	$generictarball = "r2009b.tar.gz";
	$generictopdir = "v711";

	# copy Matlab working m files
	my $Rsrcs = "$sources/Matlab/*";
	my $dmtcp = "$sources/Dmtcp/*";
	my $Rdest = "$projectlocation/code";
	debug("Populate Matlab: setting up source project\n");
	#runcmd("cp $Rsrcs $Rdest",{emit_output=>0});
	if(${$hashref}{Options} =~ /^.*win32.*$/) {
		$windowsmatlab = 1;
		foreach my $file (@windowssources) {
			runcmd("cp $sources/Matlab/$file $Rdest",{emit_output=>0});
		}
	} else {
		foreach my $file (@linuxsources) {
			runcmd("cp $sources/Matlab/$file $Rdest",{emit_output=>0});
		}
		foreach my $file (@sharedsources) {
			runcmd("cp $sources/$file $Rdest",{emit_output=>0});
		}
	}
	if(${$hashref}{Options} =~ /^.*dmtcp.*$/) {
		debug("Wrapping a Matlab job with DMTCP!\n");
		runcmd("cp $dmtcp $Rdest",{emit_output=>0});
	}
	runcmd("cp $baselocation/*.m $Rdest",{emit_output=>0});
	runcmd("touch $Rdest/.htaccess",{emit_output=>0});

	chdir("$baselocation");
	my $compiledone = 1;
	my $target = "${$hashref}{PhaseOneBinary}" . ".m";
	if($windowsmatlab == 0) {
	if(${$optionargs}{"build"} eq "onthefly") {
		# add .m to binary name for entry point
		$target = "${$hashref}{PhaseOneBinary}" . ".m";
		my $mcccmd = "mcc -m -R -nodisplay -R -nojvm -nocache $target";
		debug("Compile $target with cmd <$mcccmd>\n");
		my $compiletimeout = 10;
		my $compiletimer = 0;
		my $done = 0;
		# allow for the compile to fail, retry.... for 40 minutes every 5;
		while( $done == 0) {
			if($compiletimer > $compiletimeout) {
				print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
				print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
				print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
				print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
				print "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
				die "Took too long to Matlab compile $target!!!!!!!!!, DIE!!!!!\n";
			} else {
				my $resref = runcmd($mcccmd,{expect_result=>\&ANY});
				#look at exit value
				my $exitvalue = ${$resref}{exitcode};
				print "Exit code from <$target> is <$exitvalue>\n";
				if($exitvalue != 0) {
					print "Waiting 5 minutes to retry matlab compiler\n";
					sleep 300; # 5 minutes
					$compiletimer += 5; # add 5 minutes
				} else {
					print "Compile of <$target> is good\n";
					$done = 1; # mark done and move on to others.
					$compiletimer = 0; # reset compile time
				}
			}
		}
	} elsif (${$optionargs}{"build"} eq "localu") {
		debug("Compile $target with cmd <LocalMatlabBuild>\n");
		$compiledone = LocalMatlabBuild("$baselocation",$target);
		if($compiledone == 0) {
			print "Compile has finished. Moving on.........................\n";
		} else {
			print "Will come back and check Matlab compile\n";
			exit(0);
		}
	} elsif(${$optionargs}{"build"} eq "condorc") {
		debug("Compile $target with cmd <CondorcMatlabBuild>\n");
		debug("Checking on Copndor-C build Need..../results\n");
		my $matlabsources = "matlabsources.tar.gz";
		runcmd("tar -zcvf $matlabsources *",{expect_output=>1});
		my $compiledone = CondorCMatlabBuild("$baselocation",$target,$matlabsources,$optionargs);
		if($compiledone == 0) {
			print "Compile has finished. Moving on.........................\n";
		} else {
			print "Will come back and check Matlab compile\n";
			exit(0);
		}
	} else {
		my $buildtype = ${$optionargs}{"build"};
		die "UNKNOWN build option: onthefly, localu or condorc allowed: <$buildtype>\n";
	}
	}
	chdir("$currentloc");

	if($windowsmatlab == 1) {
		my $program = "$baselocation/${$hashref}{PhaseOneBinary}";
		my $cmd = "cp $program" . ".exe $Rdest";
		print "Command is: $cmd\n";
		runcmd("$cmd",{emit_output=>0});
	} else {
		runcmd("cp $baselocation/${$hashref}{PhaseOneBinary} $Rdest",{emit_output=>0});
	}

}

sub PopulateProgramProject
{
	my $hashref = shift;
	my $projectlocation = shift;
	my $baselocation = shift;
	my $sources = shift;
	my $dmtcp = "$sources/Dmtcp/*";
	debug("Populate Program\nSource location is <$baselocation> and destination is <$projectlocation> with common elements <$sources>\n");

	my $currentloc = getcwd();

	# copy Binary working m files
	my $Rsrcs = "$sources/Binary/*";
	my $Rdest = "$projectlocation/code";
	runcmd("cp $Rsrcs $Rdest",{emit_output=>0});
	runcmd("touch $Rdest/.htaccess",{emit_output=>0});
	if(${$hashref}{Options} =~ /^.*dmtcp.*$/) {
		debug("Wrapping a Program job with DMTCP!\n");
		runcmd("cp $dmtcp $Rdest",{emit_output=>0});
	}
	runcmd("cp $baselocation/${$hashref}{PhaseOneBinary} $Rdest",{emit_output=>0});
}

sub GenerateGenericSubmitTemplates
{
	my $hashref = shift;
	my $projectlocation = shift;
	debug("GenerateGenericSubmitTemplates for project location $projectlocation\n");
	if(${$hashref}{PhaseOneType} eq "R") {
		GenerateRTemplate($hashref,$projectlocation);
	} elsif(${$hashref}{PhaseOneType} eq "Matlab") {
		GenerateMatlabTemplate($hashref,$projectlocation);
	} else {
		GenerateProgramTemplate($hashref,$projectlocation);
	}
}

sub CommonTemplateAdditions
{
	my $hashref = shift;
	my $TP = shift;
	if(${$hashref}{JobAds}) {
		print "Oh ja, seeing job adds!\n";
		my @ads = split /,/, ${$hashref}{JobAds};
		foreach my $ad (@ads) {
			print $TP "$ad\n";
		}
	}
	if(${$hashref}{Priority}) {
		print "Oh ja, Priority!\n";
		print $TP "Priority = ${$hashref}{Priority}\n";
	}
}

sub RequirementHandling
{
	my $hashref = shift;
	my $TP = shift;
	my $specialstring = shift;

	if((${$hashref}{Memory}) && (${$hashref}{Requirements})) {
		# Replace Requirements passed in AND ignore Memory
		print "Requirements and Memory means Memory ignored\n";
		print $TP "${$hashref}{Requirements}\n";
	} elsif(${$hashref}{Memory}) {
		if($specialstring eq "requirements = ") {
			print $TP "requirements = (Memory > ${$hashref}{Memory}000)\n";
		} else {
			print $TP "$specialstring && (Memory > ${$hashref}{Memory}000)\n";
		}
	} else {
		if($specialstring eq "requirements = ") {
			# do nothing 
		} else {
			print $TP "$specialstring\n";
		}
	}
}

sub GenerateRTemplate
{
	my $hashref = shift;
	my $projectlocation = shift;
	my $TP;
	debug("GenerateRSubmitTemplates for project location $projectlocation\n");
	my $template = "$projectlocation/jobs/${$hashref}{PhaseOneName}.template";
	debug("Template file name is <$template>\n");
	open($TP,">$template") or die "Failed to create<$template>:$!\n";
	print $TP "universe = ${$hashref}{Universe}\n";
	my $phasename = ${$hashref}{PhaseOneName};
	print $TP "executable = ../rungeneric.pl\n";
	print $TP "output = $phasename.out\n";
	print $TP "error = $phasename.err\n";
	print $TP "log = $phasename.log\n";
	print $TP "match_list_length = 5\n";
	print $TP "periodic_hold = (JobStatus == 2) && ((CurrentTime - EnteredCurrentStatus) > (60 * 60 * 24))\n";
	print $TP "should_transfer_files = YES\n";
	print $TP "when_to_transfer_output = ON_EXIT_OR_EVICT\n";

	# route to osg?
	if( ${$hashref}{Options} =~ /^.*osg.*$/ ) {
		print $TP "+WantGlidein = True\n"
	}

	if(exists ${$hashref}{Version}) {
		print $TP "+R = \"${$hashref}{Version}\"\n";
	}

	print $TP "notification = never\n";
	print $TP "\n";

	my $transferin = "transfer_input_files = ";
	$transferin = $transferin . "XXXX,../${$hashref}{PhaseOneBinary},../input_checksum.txt";
	if(${$hashref}{PhaseOneLibrary}) {
		$transferin = $transferin . ",../RLIBS.tar.gz";
	}

	if(${$hashref}{SharedInputs}) {
		if(${$hashref}{SharedInputs} eq "true") {
			$transferin = $transferin . ",../sharedfiles.tar.gz";
		}
	}

	# any extra files????? compiled binaries or whatever
	if(${$hashref}{PhaseOneExtraFiles}) {
		my @extra = split /,/, ${$hashref}{PhaseOneExtraFiles};
		foreach my $tra (@extra) {
			$transferin = $transferin . ",../$tra";
		}
	}

	$transferin = $transferin . "\n";
	print $TP "$transferin";
	print $TP "arguments =  --new --type=$generictype --tarball=$generictarball --installfrom=$generictopdir --cmdtorun=${$hashref}{PhaseOneBinary}\n";
	print $TP "requirements = (TARGET.Name =!= LastMatchName1)\n";
	#RequirementHandling($hashref, $TP, "requirements = (HasR2101 == true)");
	print $TP "+AccountingGroup = \"${$hashref}{AccountingGroup}\"\n";
	print $TP "queue\n";

	close($TP);
} 

##############################################################
#
#	GenerateMatlabTemplate
#
#	A matlab job requires compiling the code so it can run
#	with a MCR. Options allow us to build a DMTCP wrapped
#	project or a windows target project. A windows project
#	can not be wrapped by DMTCP.
#
##############################################################

sub GenerateMatlabTemplate
{
	my $hashref = shift;
	my $projectlocation = shift;
	my $wrapdmtcp = 0;
	my $windowsjob = 0;
	my $TP;
	my @linuxsources = ("r2009b.tar.gz");
	my @windowssources = ("../cygiconv-2.dll", "../cygintl-8.dll", "../cygwin1.dll", "../gzip.exe", "../tar.exe", "../w32v714.tar.gz");

	# set up to pass key args to wrappers for matlab they are static for now
	$generictype = "Matlab";
	$generictarball = "r2009b.tar.gz";
	$generictopdir = "v711";

	# Dmtcp wrap requested?????
	if( ${$hashref}{Options} =~ /^.*dmtcp.*$/ ) {
		$wrapdmtcp = 1;
		debug("Wrap this Matlab job with DMTCP\n");
	} elsif(${$hashref}{Options} =~ /^.*win32.*$/ ) {
		$windowsjob = 1;
	}

	if(($windowsjob == 1) && ($wrapdmtcp == 1)) {
		die "Can not DMTCP wrap a windows matlab job! Die!\n";
	}

	debug("GenerateMatlabSubmitTemplates for project location $projectlocation\n");
	my $template = "$projectlocation/jobs/${$hashref}{PhaseOneName}.template";
	debug("Template file name is <$template>\n");
	open($TP,">$template") or die "Failed to create<$template>:$!\n";
	print $TP "universe = ${$hashref}{Universe}\n";
	my $phasename = ${$hashref}{PhaseOneName};

	if($wrapdmtcp == 1) {
		print $TP "executable = ../$dmtcpkeys{executable}\n";
		print $TP "coresize = $dmtcpkeys{coresize}\n";
		print $TP "output = shim.out\n";
		print $TP "error = shim.err\n";
		print $TP "log = shim.log\n";
	} else {
		if($windowsjob == 1) {
			print $TP "executable = ../matlab.bat\n";
		} else {
			print $TP "executable = ../rungeneric.pl\n";
		}
		print $TP "output = $phasename.out\n";
		print $TP "error = $phasename.err\n";
		print $TP "log = $phasename.log\n";
		print $TP "periodic_hold = (JobStatus == 2) && ((CurrentTime - EnteredCurrentStatus) > (60 * 60 * 24))\n";
	}

	# jobads and priority lines into submit file
	CommonTemplateAdditions($hashref,$TP);

	print $TP "should_transfer_files = YES\n";
	print $TP "when_to_transfer_output = ON_EXIT_OR_EVICT\n";
	print $TP "notification = never\n";
	print $TP "\n";

	# route to osg?
	if( ${$hashref}{Options} =~ /^.*osg.*$/ ) {
		print $TP "+WantGlidein = True\n"
	}

	my $transferin = "transfer_input_files = ";
	if($wrapdmtcp == 1) {
		if($windowsjob == 1) {
			$transferin = $transferin . "XXXX,../matlab.bat,../$dmtcpkeys{transfer_input_files},../r2009b.tar.gz,../${$hashref}{PhaseOneBinary}" . ".exe";
		} else {
			$transferin = $transferin . "XXXX,../rungeneric.pl,../input_checksum.txt,../$dmtcpkeys{transfer_input_files},../${$hashref}{PhaseOneBinary}";
		}
		print $TP "environment = $dmtcpkeys{environment}\n";
		print $TP "kill_sig = $dmtcpkeys{kill_sig}\n";
	} else {
		if($windowsjob == 1) {
			# make sure it runs on windows
			print $TP "requirements = (TARGET.Arch == \"INTEL\") && (TARGET.OpSys == \"winnt61\")\n";
			$transferin = $transferin . "XXXX,../${$hashref}{PhaseOneBinary}" . ".exe";
		} else {
			$transferin = $transferin . "XXXX,../input_checksum.txt,../${$hashref}{PhaseOneBinary}";
		}
	 	if($windowsjob == 1) {
			foreach my $file (@windowssources) {
				$transferin = $transferin . ",$file";
			}
		} else {
			$transferin = $transferin . ",../input_checksum.txt";
		}
	}
	if(${$hashref}{SharedInputs} eq "true") {
		$transferin = $transferin . ",../sharedfiles.tar.gz";
	}

	# any extra files????? compiled binaries or whatever
	if(${$hashref}{PhaseOneExtraFiles}) {
		my @extra = split /,/, ${$hashref}{PhaseOneExtraFiles};
		foreach my $tra (@extra) {
			$transferin = $transferin . ",../$tra";
		}
	}

	$transferin = $transferin . "\n";
	print $TP "$transferin";

	if($wrapdmtcp == 1) {
		if(${$hashref}{PhaseOneArg} eq "\$(JOB)") {
			print $TP "arguments =  $dmtcpkeys{arguments} ./rungeneric.pl --new --type=$generictype --tarballl=$generictarball --installfrom=$generictopdir --cmdtorun=${$hashref}{PhaseOneBinary}\n";
		} else {
			print $TP "arguments =  $dmtcpkeys{arguments} ./rungeneric.pl --new --type=$generictype --tarballl=$generictarball --installfrom=$generictopdir --cmdtorun=${$hashref}{PhaseOneBinary}\n";
		}
	} else {
		if($windowsjob == 1) {
			if(${$hashref}{PhaseOneArg} eq "\$(JOB)") {
				print $TP "arguments =  ${$hashref}{PhaseOneBinary}" . ".exe\n";
			} else {
				print $TP "arguments =  ${$hashref}{PhaseOneBinary}" . ".exe\n";
			}
		} else {
			if(${$hashref}{PhaseOneArg} eq "\$(JOB)") {
			print $TP "arguments =  $dmtcpkeys{arguments} ./rungeneric.pl --new --type=$generictype --tarballl=$generictarball --installfrom=$generictopdir --cmdtorun=${$hashref}{PhaseOneBinary}\n";
			} else {
			print $TP "arguments =  $dmtcpkeys{arguments} ./rungeneric.pl --new --type=$generictype --tarballl=$generictarball --installfrom=$generictopdir --cmdtorun=${$hashref}{PhaseOneBinary}\n";
			}
		}
	}
	#RequirementHandling($hashref, $TP, "requirements = (HasMatlabv711 == true)");
	#print $TP "requirements = (HasMatlabv711 == true)\n";
	print $TP "+AccountingGroup = \"${$hashref}{AccountingGroup}\"\n";
	if(exists ${$hashref}{Version}) {
		print $TP "+Matlab = \"${$hashref}{Version}\"\n";
	} else {
		print $TP "+Matlab = \"r2009b\"\n";
	}

	print $TP "queue\n";

	close($TP);
} 

sub GenerateProgramTemplate
{
	my $hashref = shift;
	my $projectlocation = shift;
	my $wrapdmtcp = 0;
	my $TP;

	# Dmtcp wrap requested?????
	if( ${$hashref}{Options} =~ /^.*dmtcp.*$/ ) {
		$wrapdmtcp = 1;
		debug("Wrap this Program job with DMTCP\n");
	}

	debug("GenerateProgramSubmitTemplates for project location $projectlocation\n");
	my $template = "$projectlocation/jobs/${$hashref}{PhaseOneName}.template";
	debug("Template file name is <$template>\n");
	open($TP,">$template") or die "Failed to create<$template>:$!\n";
	print $TP "universe = ${$hashref}{Universe}\n";
	my $phasename = ${$hashref}{PhaseOneName};

	if($wrapdmtcp == 1) {
		print $TP "executable = ../$dmtcpkeys{executable}\n";
        print $TP "coresize = $dmtcpkeys{coresize}\n";
        print $TP "output = shim.out\n";
        print $TP "error = shim.err\n";
        print $TP "log = shim.log\n";
	} else {
		print $TP "executable = ../runbinary.pl\n";
		print $TP "output = $phasename.out\n";
		print $TP "error = $phasename.err\n";
		print $TP "log = $phasename.log\n";
		print $TP "periodic_hold = (JobStatus == 2) && ((CurrentTime - EnteredCurrentStatus) > (60 * 60 * 24))\n";
	}

	print $TP "should_transfer_files = YES\n";
	print $TP "when_to_transfer_output = ON_EXIT_OR_EVICT\n";
	print $TP "notification = never\n";
	print $TP "\n";

	my $transferin = "transfer_input_files = ";
	if($wrapdmtcp == 1) {
		$transferin = $transferin . "XXXX,../runbinary.pl,../$dmtcpkeys{transfer_input_files},../${$hashref}{PhaseOneBinary}";
		print $TP "environment = $dmtcpkeys{environment}\n";
        print $TP "kill_sig = $dmtcpkeys{kill_sig}\n";
	} else {
		$transferin = $transferin . "XXXX,../${$hashref}{PhaseOneBinary}";
	}

	if(${$hashref}{SharedInputs} eq "true") {
		$transferin = $transferin . ",../sharedfiles.tar.gz";
	}

	# any extra files????? compiled binaries or whatever
	if(${$hashref}{PhaseOneExtraFiles}) {
		my @extra = split /,/, ${$hashref}{PhaseOneExtraFiles};
		foreach my $tra (@extra) {
			$transferin = $transferin . ",../$tra";
		}
	}

	$transferin = $transferin . "\n";
	print $TP "$transferin";
	if($wrapdmtcp == 1) {
		if(${$hashref}{PhaseOneArg} eq "\$(JOB)") {
			print $TP "arguments = $dmtcpkeys{arguments} ./runbinary.pl ${$hashref}{PhaseOneBinary}\n";
		} else {
			print $TP "arguments = $dmtcpkeys{arguments} ./runbinary.pl ${$hashref}{PhaseOneBinary}\n";
		}
	} else {
		if(${$hashref}{PhaseOneArg} eq "\$(JOB)") {
			print $TP "arguments =  ${$hashref}{PhaseOneBinary}\n";
		} else {
			print $TP "arguments =  ${$hashref}{PhaseOneBinary}\n";
		}
	}
	RequirementHandling($hashref, $TP, "requirements = ");
	#print $TP "requirements = (HasMatlabv711 == true)\n";
	print $TP "+AccountingGroup = \"${$hashref}{AccountingGroup}\"\n";
	print $TP "queue\n";

	close($TP);
} 

# GenericReadWriteTest($baselocation,"NEWPROJECT","TESTWRITE");
sub GenericReadWriteTest
{
	my $location = shift;
	my $readtest = shift;
	my $writetest = shift;
	my $readit = "$location/$readtest";
	my $writeit = "$location/$writetest";

	my $line = "";
	open(RR,"<$readit") or die "Generic Project Read Permission Failed<$readit>:$!\n";
	debug("Generic Project Request BEGIN:\n");
	while(<RR>) {
		debug( "$_");
	}
	debug( "Generic Project Request END:\n");
	close(RR);
	open(WW,">$writeit") or die "Generic Project Write Permission Failed<$writeit>:$!\n";
	print WW "Test write\n";
	close(WW);
}

#################################################################################
#
# FixMakeDagText 
#
# This file has a line which feeds a file in the run directory so that
# the correct user log is examined for profiling a set of jobs. Fix it to
# match user log called out in the submit file.
#
#################################################################################

sub FixMakeDagText
{
	my $scriptloc = shift;
	my $resultpatterns = shift;
	my $newfile = "make_job_dag_text.pl";
	my $template = "make_job_dag_text.template";
	my $homedir = getcwd();
	chdir($scriptloc);

	my $line = "";
	open(IN,"<$template") or die "Failed to open postjob template<$template>:$!\n";
	open(OUT,">$newfile") or die "Failed to open new postjob script<$newfile>:$!\n";
	while(<IN>) {
		chomp();
		$line = $_;
		if($line =~ /^(.*)ZZZZZ(.*)$/) {
			print OUT "$1$resultpatterns$2;\n";
		} else {
			print OUT "$line\n";
		}
	}
	close(IN);
	close(OUT);
	system("chmod 755 $newfile");
	chdir($homedir);
}

#################################################################################
#
# FixMakeSubmitText 
#
#
#################################################################################

sub FixMakeSubmitText
{
	my $scriptloc = shift;
	my $processonename = shift;
	my $type = shift;
	my $sharedfiles = shift;
	my $newfile = "make_job_submits.pl";
	my $template = "make_job_submits.template";
	my $homedir = getcwd();
	chdir($scriptloc);

	print "FixMakeSubmitText <$scriptloc>,<$processonename>,<$type>,<$sharedfiles>\n";
	my $line = "";
	open(IN,"<$template") or die "Failed to open postjob template<$template>:$!\n";
	open(OUT,">$newfile") or die "Failed to open new postjob script<$newfile>:$!\n";
	while(<IN>) {
		chomp();
		$line = $_;
		if($line =~ /^(.*)ZZZZZ(.*)$/) {
			print OUT "$1$processonename$2\n";
		} else {
			print OUT "$line\n";
		}
	}
	close(IN);
	close(OUT);
	system("chmod 755 $newfile");
	chdir($homedir);
}

#################################################################################
#
# FixPostJobResults
#
# all we know about results to harvest is a comma separated list if patterns
# so we have to change the postjob scripts to test for these patterns and
# to copy to result locations things which match.
# 
# The template has XXXXX where we want to throw into an array to count
# all the things which match the pattern. The template has YYYYY where
# we want to copy everything which matches into the results
#
#################################################################################

sub FixPostJobResults
{
	my $scriptloc = shift;
	my $resultpatterns = shift;
	my $newfile = "postjob.pl";
	my $template = "postjob.template";
	my @patterns = split /,/, $resultpatterns;
	my $patterncount = @patterns;
	my $line = "";
	my $newline = "";
	my $homedir = getcwd();
	chdir($scriptloc);
	debug("Patterns in FixPostJobResults <$patterncount>\n");
	open(IN,"<$template") or die "Failed to open postjob template<$template>:$!\n";
	open(OUT,">$newfile") or die "Failed to open new postjob script<$newfile>:$!\n";
	while(<IN>) {
		chomp();
		$line = $_;
		if($line =~ /^XXXXX.*$/) {
			$newline = "my \@results = `ls ";
			foreach my $pat (@patterns) {
				$newline = $newline . "*$pat* ";
			}
			$newline = $newline . "`;";
			print OUT "$newline\n";
		} elsif($line =~ /^YYYYY.*$/) {
			foreach my $pat (@patterns) {
				$newline = "system(\"cp *$pat* \$resultsloc/\$job\");\n";
				print OUT "$newline\n";
			}
		} else {
			print OUT "$line\n";
		}
	}
	close(IN);
	close(OUT);
	system("chmod 755 $newfile");
	chdir($homedir);
}

sub ViewOptionsHash
{
	my $hashref = shift;

	foreach my $key (sort keys %{$hashref}) {
		debug( "Key: $key: ${$hashref}{$key}\n");
	}
}

# =================================
# ParseOptions will hash entries desired to tune automatic
# updates things like if a new version is wanted, what limit count
# for first run, run on existing(reset) or new only data sets,
# if matlab, which m files to compile..... etc
#
# A hesh reference must be passed in
# =================================

sub ParseOptions
{
	my $options = shift;
	my $hashref = shift;

	my $jobadsplus = "";
	debug("ParseOptions on file <$options>\n");

	if(!(-f $options)) {
		print "No update configuration file present at <$options>\n";
		${$hashref}{"success"} = 0;
		return(0);
	}

	open(NORUNS,"<$options") || die "Can not open <$options>: $!\n";
	while(<NORUNS>) {
		if($_ =~ /^(\w+)\s*=\s*(.*)\s*/) {
			debug("Option $1 = $2\n");
			next if ( $1 eq "version" );
			${$hashref}{$1} = $2;
		} elsif($_ =~ /^(\+.*)/) {
			if($jobadsplus eq "") {
				$jobadsplus = "$1"
			} else {
				$jobadsplus = $jobadsplus . ",$1"
			}
		} else {
			print "no match: $_";
		}

	}
	close(NORUNS);
	if($jobadsplus ne "") {
		${$hashref}{JobAds} = $jobadsplus;
	}

	${$hashref}{"success"} = 1;

	debug("Leaving ParseOptions - success == ${$hashref}{\"success\"}\n");
}

sub GetJobId
{
	my $jobid = 1;
	my $line = "";

	#DebugOn();
	debug("About to lock<$jobidfile>\n");

	my $tracking = LookupFsConfig("TRACKING");
	if( $tracking eq "" ) {
		die "GetJobId failed to lookup TRACKING\n";
	}

	my $realjobidfile = "$tracking/$jobidfile";

	if(!(-f $realjobidfile)) {
		# if file does not exist create for writing and insert
		# an prepare to write an initial job id into file
		open( JOBIDLOCK, ">$realjobidfile") or die "Failed to open<$realjobidfile>:$!\n";
		# Lock operations on append to file
		while(! flock(JOBIDLOCK, LOCK_EX)) {
			print "Waiting to lock profile and report file\n";
			sleep 1;
		}
		debug("Locked $realjobidfile\n");
	} else {
		# Lock operations on append to file
		open( JOBIDLOCK, "<$realjobidfile") or die "Failed to open<$realjobidfile>:$!\n";
		while(! flock(JOBIDLOCK, LOCK_EX)) {
			print "Waiting to lock profile and report file\n";
			sleep 1;
		}
		debug("Locked $realjobidfile\n");
		# get last issued job id and prepare job id to be re-entered
		# into job id file
		$line = "";
		while(<JOBIDLOCK>) {
			chomp();
			$line = $_;
			$jobid = $line + 1;
		}
		close(JOBIDLOCK);

		# now open the locked file for writing
		open( JOBIDLOCK, ">$realjobidfile") or die "Failed to open<$realjobidfile>:$!\n";
	}

	print JOBIDLOCK "$jobid\n";
	#free lock
	flock(JOBIDLOCK, LOCK_UN) or die "WARNING: Failed to unlock<$realjobidfile>:$!\n";
	debug( "UN-Locked $realjobidfile\n" );
	close(JOBIDLOCK);
	debug("GetJobId returning $jobid\n");
	#DebugOff();

	return($jobid);
}

#####################################################
#
# Tracking running vs done within a project
# This is all per project based and probably should be 
# version based but will not be yet as it has not been an issue
# yet.
#
# Logic and methods follow:
#
# The status program which spins on an interval and looks at done
# and running for a particular set has worked well for 'a' run.
# But when we are looking at multiple dags within the project
# at once the concept of done within the project gets muddled.
# We loose track of exactly which of the project's sets of
# jobs have been completed and startup ones which we have completed.
# It still works well for the reporting of work in the current DAG though.
#
# So we will set up project based DONE and RUNNING files which
# will be adjusted with locking. The determination of which datasets
# to consider in a request for new work will consult these lists.
#
# "control.pl"
#
# The final post-script for each job in the projects DAG
# will call an adjustment routine which with locking will remove it
# from the running file and add it to the done file.
#
# in most projects "postjob.pl" in the project's jobs folder.
#
# As of 0.7.8 we will do this based on the projects version
# and thus allow a large input data set to be run on multiple 
# binaries without duplicating the datasets.
# This has the lock file being moved $sources/project/DONELOCK
# and the DONE and RUNNING files to $sources/project/version.
#
#####################################################

# automatically move old DONE RUN files if they exist

my $oldest_project_done_location = "";
my $oldest_project_running_location = "";

my $old_project_done_location = "";
my $old_project_running_location = "";

my $project_done_location = "";
my $project_running_location = "";
my $project_running_done_lock = "";
my $DONE = "DONE";
my $RUNNING = "RUNNING";
my $DONELOCK = "DONELOCK";

sub SetDoneRunFiles
{
	my $project = shift;
	my $version = shift;

	debug("In SetDoneRunFiles for <$project/$version>\n");

	# Set them all up after getting key info
	# have to set up before locking

	my $src = LookupFsConfig("SOURCE");
	if( $src eq "" ) {
		UnlockDoneRun();
		die "ResetDoneRunning failed to lookup SOURCE\n";
	}
	
	my $run = LookupFsConfig("CONDORRUNS");
	if( $run eq "" ) {
		UnlockDoneRun();
		die "ResetDoneRunning failed to lookup CONDORRUNS\n";
	}
	
	my $tracking = LookupFsConfig("TRACKING");
	if( $tracking eq "" ) {
		die "SetDoneRunFiles failed to lookup TRACKING\n";
	}

	$project_running_done_lock = "$tracking/$project/$DONELOCK";

	LockDoneRun();
	$oldest_project_done_location = "$run/$project/$DONE";
	$oldest_project_running_location = "$run/$project/$RUNNING";

	# we are moving all locks and writable files to tracking directory
	# to keep write permissions localized. We want to preserve which
	# ever location has been used

	$project_done_location = "$tracking/$project/$version/$DONE";
	$project_running_location = "$tracking/$project/$version/$RUNNING";
	debug("Setting <$project_done_location> and <$project_running_location>\n");

	# if oldest project based files exist move to current version
	debug("Old done <$oldest_project_done_location>\n");
	debug("Old running <$oldest_project_running_location>\n");
	#system("ls -l $oldest_project_done_location");
	#system("ls -l $oldest_project_running_location");

	if((-f $oldest_project_running_location) || (-f $oldest_project_done_location)){
		print "Moving project $project RUN/DONE files to project source location.\n";
		runcmd("mv $oldest_project_done_location $project_done_location",{emit_output=>0});
		runcmd("mv $oldest_project_running_location $project_running_location",{emit_output=>0});
	} 

	# now check the more recent location

	$old_project_done_location = "$src/$project/$version/$DONE";
	$old_project_running_location = "$src/$project/$version/$RUNNING";

	# if oldest project based files exist move to current version
	debug("Old done <$old_project_done_location>\n");
	debug("Old running <$old_project_running_location>\n");
	#system("ls -l $old_project_done_location");
	#system("ls -l $old_project_running_location");

	if((-f $old_project_running_location) || (-f $old_project_done_location)){
		print "Moving project $project RUN/DONE files to project source location.\n";
		runcmd("mv $old_project_done_location $project_done_location",{emit_output=>0});
		runcmd("mv $old_project_running_location $project_running_location",{emit_output=>0});
	} 

	#print "Make sure a new project has these files\n";
	if(!(-f $project_done_location)) {
		runcmd("touch $project_done_location",{emit_output=>0});
		print "Making master DONE file\n";
	}

	if(!(-f $project_running_location)) {
		runcmd("touch $project_running_location",{emit_output=>0});
		print "Making master RUNNING file\n";
	}

	UnlockDoneRun();
}

sub ResetByPattern
{
	my $project = shift;
	my $version = shift;
	my $pattern = shift;

	#DebugOn();
	debug("In ResetByPattern for project <$project/$version/$pattern>\n");
	SetDoneRunFiles($project,$version); 	# fill in relative locations and do migration
						# as needed and then lock and filter DONE
						# by the requested pattern

	LockDoneRun();
	#$project_done_location = "$src/$project/$version/$DONE";

	my $newdone = "$project_done_location.new";
	my $olddone = "$project_done_location.old";
	my $line = "";

	open(SRC,"<$project_done_location") or die "Failed to open master DONE file<$project_done_location>:$!\n";
	open(NEW,">$newdone") or die "Can not create filtered DONE file<$newdone>:$!\n";
	print "\n";
	while (<SRC>) {
		chomp();
		$line = $_;
		if($line =~ /^$pattern.*$/) {
			# skip over and leave out
			print ".";
		} else {
			print NEW "$line\n";
		}
	}
	print "\n";
	close(SRC);
	close(NEW);
	UnlockDoneRun();
	runcmd("mv $project_done_location $olddone",{emit_output=>0});
	runcmd("mv $newdone $project_done_location",{emit_output=>0});
	print "Partial reset of version $version of project $project by pattern $pattern is done\n";
	print "Bye\n";
	exit(0);
}

sub ResetDoneRunning
{
	my $project = shift;
	my $version = shift;

	debug("In ResetDoneRunning for project <$project/$version>\n");

	if(!(defined $project)) {
		die "Can not reset done and running lists without a project\n";
	}

	SetDoneRunFiles($project,$version);

	ResetRunning();
	ResetDone();
	#die "Rest running test done...... stopping run\n";
}

sub LockDoneRun
{
	debug("About to lock $project_running_done_lock\n");
	# Lock operations on append to file
	open( DRLOCK, ">$project_running_done_lock") or die "Failed to open<$project_running_done_lock>:$!\n";
	while(! flock(DRLOCK, LOCK_EX)) {
		print "Waiting to lock done and run file\n";
		sleep 1;
	}

	debug("Locked $project_running_done_lock\n");
}

sub UnlockDoneRun
{
	#free lock
	flock(DRLOCK, LOCK_UN) or die "WARNING: Failed to unlock<$project_running_done_lock>:$!\n";
	debug( "UN-Locked $project_running_done_lock\n" );
	close(DRLOCK);
}

sub GetRunningDone
{
	my $project = shift;
	my $version = shift;
	my $doneref = shift;
	my $runningref = shift;

	SetDoneRunFiles($project,$version);
	LockDoneRun();

	# push all done to done array ref
	# and all running to running array ref

	my $file = $project_running_location;
	if(open( RUNNING, "<$file")) {
		while(<RUNNING>) {
			chomp();
			debug("sharing $_ as running\n");
			#print "sharing $_ as running\n" ;
			push @{$runningref}, $_;
		}
		close(RUNNING);
	}

	$file = $project_done_location;
	# copy existing data to new file except done job
	if(open( DONE, "<$file")) {
		while(<DONE>) {
			chomp();
			debug("sharing $_ as done\n");
			#print "sharing $_ as done\n" ;
			push @{$doneref}, $_;
		}
		close(DONE);
	}

	UnlockDoneRun();
}

sub AddRunning
{
	my $project = shift;
	my $version = shift;
	my $arrayref = shift;

	SetDoneRunFiles($project,$version);
	my $file = $project_running_location;
	my $newfile = "$project_running_location.new";

	if(!(-f $file)) {
		ResetDoneRunning($project,$version);
	}

	LockDoneRun();

	# add this list to whatever is current
	# copy existing data to new file
	# if the running file is not there, its the first run of project
	open( TAR, "<$file") or die "Failed to open<$file>:$!\n";
	open( NEWTAR, ">$newfile") or die "Failed to open<$newfile>:$!\n";
	while(<TAR>) {
		print NEWTAR "$_";
	}
	close(TAR);

	# append new data to file
	for my $job (@{$arrayref}) {
		print NEWTAR "$job\n";
	}
	close(NEWTAR);

	copy("$file", "$file.old") or die "copy $file to $file.old failed:$!\n";
	copy("$newfile", "$file") or die "copy $newfile to $file failed:$!\n";
	runcmd("chmod 755 $file",{emit_output=>0});
	runcmd("rm $newfile",{emit_output=>0});


	UnlockDoneRun();
}

sub SetNotRunning
{
	my $project = shift;
	my $version = shift;
	my $unique = shift;

	SetDoneRunFiles($project,$version);
	LockDoneRun();

	# remove this from current running list

	my $file = $project_running_location;
	my $newfile = "$project_running_location.new";
	# copy existing data to new file except done job
	open( TAR, "<$file") or die "Failed to open<$file>:$!\n";
	open( NEWTAR, ">$newfile") or die "Failed to open<$newfile>:$!\n";
	while(<TAR>) {
		chomp();
		if($_ ne $unique) {
			# keep this one in the running file
			print NEWTAR "$_\n";
		} else {
			# remove this one from the running file
		}
	}
	close(TAR);
	close(NEWTAR);

	# Replace file 
	copy("$file", "$file.old") or die "copy $file to $file.old failed:$!\n";
	copy("$newfile", "$file") or die "copy $newfile to $file failed:$!\n";
	runcmd("chmod 755 $file",{emit_output=>0});
	runcmd("rm $newfile",{emit_output=>0});

	UnlockDoneRun();
}

sub SetDone
{
	my $project = shift;
	my $version = shift;
	my $unique = shift;

	SetDoneRunFiles($project,$version);
	LockDoneRun();

	# remove this from current running list

	my $file = $project_running_location;
	my $newfile = "$project_running_location.new";
	# copy existing data to new file except done job
	open( TAR, "<$file") or die "Failed to open<$file>:$!\n";
	open( NEWTAR, ">$newfile") or die "Failed to open<$newfile>:$!\n";
	while(<TAR>) {
		chomp();
		if($_ ne $unique) {
			# keep this one in the running file
			print NEWTAR "$_\n";
		} else {
			# remove this one from the running file
		}
	}
	close(TAR);
	close(NEWTAR);

	# Replace file 
	copy("$file", "$file.old") or die "copy $file to $file.old failed:$!\n";
	copy("$newfile", "$file") or die "copy $newfile to $file failed:$!\n";
	runcmd("chmod 755 $file",{emit_output=>0});
	runcmd("rm $newfile",{emit_output=>0});

	# add this to current done list
	$file = $project_done_location;
	$newfile = "$project_done_location.new";
	# copy existing data to new file except done job
	open( TAR, "<$file") or die "Failed to open<$file>:$!\n";
	open( NEWTAR, ">$newfile") or die "Failed to open<$newfile>:$!\n";
	while(<TAR>) {
		# keep this one in the running file
		print NEWTAR "$_";
	}
	close(TAR);
	print NEWTAR "$unique\n";
	close(NEWTAR);

	# Replace file 
	copy("$file", "$file.old") or die "copy $file to $file.old failed:$!\n";
	copy("$newfile", "$file") or die "copy $newfile to $file failed:$!\n";
	runcmd("chmod 755 $file",{emit_output=>0});
	runcmd("rm $newfile",{emit_output=>0});

	UnlockDoneRun();
}

sub ResetDone
{
	LockDoneRun();
	# if file does not exist create for writing 
	# and lock then unlock then close
	open( DONE, ">$project_done_location") or die "Failed to open<$project_done_location>:$!\n";
	close(DONE);
	UnlockDoneRun();
}

sub ResetRunning
{
	LockDoneRun();
	# if file does not exist create for writing 
	# and lock then unlock then close
	open( RUNNING, ">$project_running_location") or die "Failed to open<$project_running_location>:$!\n";
	close(RUNNING);
	UnlockDoneRun();
}

sub GetResultValue
{
	my $resfile = shift;
	my $line = "";
	open(XX,"<$resfile") or die "Result file<$resfile> open failed:$!\n";
	while(<XX>) {
		chomp();
		$line = $_;
	}
	print "GetResultValue returning <$line>\n";
	return($line);
}

# html crafting strings for projects
# Also filler files
# my $webstart = "index.start";
# my $webdiv = "index.div";
# my $webend = "index.end";
# my %projecthash;

sub CraftNewIndex
{
	my $sponsor = shift;
	my $newproject = shift;
	my $targetfile = shift;
	my $inputfile;
	my $line = "";
	my $htmlloc = LookupFsConfig("HTMLLOC");
	my @parseline;

	#DebugOn();
	print "HTMLLOC = <$htmlloc>\n";
	my $htmllockfile = $htmlloc . "/INDEXLOCK";
	# reserve access
	GetHtmlLock($htmllockfile);

	# move out old file
	runcmd("touch $htmlloc/$targetfile");
	runcmd("mv $htmlloc/$targetfile $htmlloc/targetfile.old");

	# construct new one from table
	$inputfile = $htmlloc . "/projects.webinput";
	
 	my $sourceloc  = LookupFsConfig("SOURCE");
    if( $sourceloc  eq "" ) {
        die "CraftNewIndex failed to lookup SOURCE\n";
    }

	# where are the pieces?
	my $indexstart = "$sourceloc/$resources/index.start";
	my $indexdiv = "$sourceloc/$resources/index.div";
	my $indexend = "$sourceloc/$resources/index.end";

	# open new file
	# create new index file by dropping col end/col start
	# at half way + point(ie next dept starts next col)
	open(NHTML,">$htmlloc/$targetfile") or die "Fail to create<$htmlloc/$targetfile>:$!\n";

	# seed with beginning
	open(BEG,"<$indexstart") or die "Can not get start<$indexstart>:$!\n";
	while(<BEG>) {
		print NHTML "$_";
	}
	close(BEG);

	# eat table of projects
	open(PW,"<$inputfile") or die "Can not read html index inputs<$inputfile>:$!\n";
	while(<PW>) {
		chomp();
		#debug("Examine: $_\n");
		$line = $_;
		@parseline = split /,/, $line;
		if(exists $projecthash{$parseline[0]}) {
		} else {
			$projecthash{$parseline[0]} = ();
		}
		push @{$projecthash{$parseline[0]}} , $parseline[1];
	}
	close(PW);

	# add the new one

	if(exists $projecthash{$sponsor}) {
	} else {
			$projecthash{$sponsor} = ();
	}
	push @{$projecthash{$sponsor}} , $newproject;

	my $newprojectline = "$sponsor,$newproject";
	AppendToFile( $newprojectline, $inputfile, "HTML" );

	# count current projects
	my $projectcount = 0;
	foreach my $project (sort keys %projecthash) {
		#debug("key <$project> has value <$projecthash{$project}>\n");
		my @tmparray =  @{$projecthash{$project}};
		foreach my $p (@tmparray) {
			#print " $p\n";
			$projectcount += 1;
		}
	}
	print "There are <$projectcount> projects\n";
	my $dropcount = 0;
	my $halfcount = ($projectcount / 2);
	print "Half count is <$halfcount>\n";

	# create new index file by dropping col end/col start
	# at half way + point(ie next dept starts next col)
	# open(NHTML,">$htmlloc/$targetfile") or die "Fail to create<$htmlloc/$targetfile>:$!\n";
	my $divdropdone = 0;
	foreach my $project (sort keys %projecthash) {
		#debug("key <$project> has value <$projecthash{$project}>\n");
		my $dropproject = 1;
		if($dropcount > $halfcount) {
			if($divdropdone == 0) {
				$divdropdone = 1;
				print "HALFWAY POINT++++++++\n";
				# seed end
				open(DIV,"<$indexdiv") or die "Can not get start<$indexstart>:$!\n";
				while(<DIV>) {
					print NHTML "$_";
				}
				close(DIV);
			}
		}
# my $itemstart = "	<tr align=\"center\" valign=\"top\">\n";
# my $itemend = "	</tr>\n";
# my $itemwhostart = "		<td width=\"60%\" align=\"left\">";
# my $itemwhoend = "</td>\n";
# my $itemwhatstart = "		<td align=\"left\"><a href=\"./projects.php?project=";
# my $itemwhatend = "</a></td>\n";
		my @tmparray =  @{$projecthash{$project}};
		foreach my $p (@tmparray) {
			print NHTML "$itemstart";
			print NHTML "$itemwhostart";
			if($dropproject == 1) {
				$dropproject = 0;
				print NHTML "$project";
			}
			print NHTML "$itemwhoend";
			print NHTML "$itemwhatstart";
			print NHTML "$p";
			print NHTML "\">$p$itemwhatend";
			print NHTML "$itemend";
			$dropcount += 1;
		}
	}
	# seed end
	open(END,"<$indexend") or die "Can not get start<$indexstart>:$!\n";
	while(<END>) {
		print NHTML "$_";
	}
	close(END);
	close(NHTML);

	# When more then half out, fill second column with rest


	# free up access
	ReleaseHtmlLock($htmllockfile);
	#DebugOff();
}

# all writeable things for a run like locks and running and done
# tracking are moved out of control and sources now.

sub MakeProjectTracking
{
	my $project = shift;
	my $version = shift;

	my $currentpath = "";
	my $tmp = "";

	$tmp = LookupFsConfig("TRACKING");
	if( $tmp eq "" ) {
		die "MakeProjectTracking failed to lookup TRACKING\n";
	}

	$currentpath = $tmp;

	runcmd("mkdir -p $currentpath",{emit_output=>0});

	$currentpath = "$currentpath/$project";

	runcmd("mkdir -p $currentpath",{emit_output=>0});

	$currentpath = "$currentpath/$version";

	runcmd("mkdir -p $currentpath",{emit_output=>0});
}

sub DropCleanupDag
{
	my $cleanupdag = "cleanupdag";
	if(!(-f "$cleanupdag")) {
		print "Dropping cleanupdag\n";
		open(CUD,">$cleanupdag") or die "Failed to open cleanup dag<$cleanupdag>:$!\n";
		print CUD "JOB postreport postreport.cmd\n";
		print CUD "JOB wrapup wrapup.cmd\n";
		print CUD "PARENT postreport CHILD wrapup\n";
		print CUD "JOB pushdata pushdata.cmd\n";
		print CUD "PARENT wrapup CHILD pushdata\n";
		#print CUD "JOB cleanup cleanup.cmd\n"; # don't clean it up in case we want to look!
		close(CUD);
		runcmd("condor_submit_dag $cleanupdag",{emit_output=>1});
	} else {
		print "NOT Dropping cleanupdag - already there\n";
		system("pwd; ls cl*");
	}
}


#####################################################
#
# Debug code
#
#####################################################

sub debug
{
    my $string = shift;
    print( "DEBUG ", timestamp(), ": $string" ) if $DEBUG;
}

sub DebugOn
{
    $DEBUG = 1;
}

sub DebugOff
{
    $DEBUG = 0;
}

sub timestamp {
    return scalar localtime();
}

1;
