#!/usr/bin/env perl

###########################################################################
# This script submits condor-specific NMI runs (either builds or tests) 
# Invoke with "--help" for a detailed usage message 
#
# Overview of the tasks of this script:
# 1. Get the buildtags from the nighlty build-tag on the CSL machine or 
#    from the command line arguments
# 2. For each tag defined
#    - Checkout the nmi_tools/glue/SubmitInfo.pm input file and parse it
#    - Generate the appropriate NMI submit and fetch files
#    - run nmi_submit

######################################################################
# Modules used by this script
######################################################################

use strict;
use Cwd;
use DBI;
use Getopt::Long;
use File::Basename;
use File::Spec;
use IO::File;
use FindBin;
use lib $ENV{'NMI_LIB'} || "/usr/local/nmi/lib";

BEGIN { eval "use NmiConf;" }
my $HAS_NMICONF = $INC{"NmiConf.pm"};

# variables we'll be using for from Getopt::Long.  keep these in the
# same order as the call to GetOptions() to help notice discrepancies
use vars qw/ $opt_build $opt_test
    $opt_help $opt_debug $opt_notify $opt_notify_fail_only $opt_platforms $opt_nmi_glue 
    $opt_workspace $opt_git
    $opt_tag $opt_module $opt_nmiconf $opt_desc $opt_append $opt_importance
    $opt_submit_xtests $opt_configure $opt_src_tag $opt_other_tag
    $opt_nightly $opt_externals $opt_without_tests
    $opt_clear_externals_cache $opt_use_externals_cache
    $opt_clear_externals_cache_weekly $opt_clear_externals_cache_daily
    $opt_externals_only $opt_release_only $opt_version_buildid
    $opt_build_args
	$opt_skip_doc $opt_sha1 $opt_private_build
    $opt_buildid $opt_test_src $opt_test_timeout $opt_test_class
    $opt_input_platform $opt_tests_at_once $opt_coverity_analysis
	$opt_ignore_missing_platforms $opt_test_sources_from_workspace
	$opt_append_config_dir

    /;

sub TRUE { 1; };
sub FALSE { 0; };

# Constants for use with makeFetchFileVCS to specify if fetching from
# source (SRC), externals (EXT) or manual (DOC) repositories.
#sub DOC 	{ 1; };
sub SRC 	{ 0; };

######################################################################
# Configurable settings
######################################################################

# Root of the GIT Source repository
my $GITROOT_SRC = $ENV{'GIT_DIR'} || "/home/condorauto/condor.git";

# Root of the GIT Manual repository
# my $GITROOT_DOC = "/space/git/CONDOR_DOC.git";

# name of the submit_info file
my $submit_info_basename = "SubmitInfo.pm";

# Location of the submit_info file
my $submit_info = "nmi_tools/glue/$submit_info_basename";

# CVS location of condor_nmi_submit
my $cvs_condor_nmi_submit = "nmi_tools/condor_nmi_submit";

# Default location to check for NMI tools (if NMI_BIN isn't set)
my $default_nmi_bin =  "/usr/local/nmi/bin";

# Find where the Condor binaries we should use are located
my $default_condor_bin = "/usr/bin";

# Config file encapsulating dependencies on NMI framework.
my $nmiconf;
if( -f '/usr/local/nmi/etc/nmi.conf' ) {
    $nmiconf = '/usr/local/nmi/etc/nmi.conf';
}

######################################################################
# Global variables (set at runtime, from submit_info, or ARGV)
######################################################################

# Who should get notification emails?
my $notify;

# True if we should only notify on build/test errors
my $notify_fail_only;

# What type of run is this? ("build" or "test")
my $runtype;

# Where the script was initially born
my $init_cwd;

# A temporary workspace to generate scratch files, etc
my $workspace;

# User specified directory to build Condor out of.
# undef for builds out of CVS (or any other VCS, e.g. GIT)
my $src_workspace;

# Hash of tag -> module mappings
my %tags;

# Platforms to use (either from command-line or from submit_info).
# Entries in here can either be things like "x86_rhap_5" or
# "x86_rhap_5#default_minimal_platform"--generally
# "platform_name#platform_key", or a build and test set name.  Use the function
# convertPlatformToKeys() to convert an element into the true platform name,
# and a key which can be used to look up the associated information in the
# submit_info hash table from. If no specific key is specified, the platform
# name is itself the platform key. 
my @platforms;

# Platforms defined on the command-line, overrides values in the
# submit_info file
my %opt_platforms;

# Data structures to hold parsed submit info
# These get reset for each tag we're dealing with
my %platforms;   # this is a hash, not a list, to guarantee uniqueness
my $global_prereqs;
my $global_test_prereqs;
my $global_testargs;

my %platform_prereqs;
my %platform_test_prereqs;

my %platform_x_test_platforms;
my %platform_testargs;

# data for the test only platform definitions for cross testing
my %testplatforms;   # this is a hash, not a list, to guarantee uniqueness
my %testplatform_prereqs;
my %testplatform_test_prereqs;

# Global variable for better error messages when parsing submit_info
my $orig_lineno = 0;

# Test timeout value to use (from command-line)
my $test_timeout = 20;

# NMI-specific configuration values
my %conf_nmi;

# NMI Database parameters
my $database;
my $username;
my $password;
my $mysqlhost;
my $db;
my $RUN_TABLE = "Run";
my $TASK_TABLE = "Task";

# These next 3 paths are initialized at runtime based on the
# environment or the defaults supplied above

# Directory for NMI binaries we should use (e.g. nmi_submit)
my $nmi_bin;

# Full path to the nmi_submit program we should invoke
my $nmi_submit;

# Directory for Condor binaries we schould use (actually, these are
# only used by nmi_submit, but set still have to find them) 
my $condor_bin;

# Whether or not the build should be private.  default: NO
my $private_build;

######################################################################
# Actual work of the script
######################################################################

FindBin::again();
print "Location of invoked $FindBin::RealScript: " .
	"$FindBin::RealBin/$FindBin::RealScript\n";

my $tag;
parseOptions();

# we only use git so leave other crud for now but define $opt_git
$opt_git = 1;

initialize();
getTags();
foreach $tag (sort keys %tags ) {
    if( defined $src_workspace ) {
        if( $tag ne 'workspace' ) {
            die "Internal consistency error: attempting --workspace build, but a non-workspace tag was requested.";
        }
    }

    my $cmdfile;
    print "Working on $tag\n";
    chdir($workspace) || die "Can't chdir($workspace): $!\n";
    if( ! -d $tag ) {
        system("mkdir -p $tag" ) && die "Can't mkdir -p $tag: $!\n";
    }
    chdir( $tag ) || die "Can't chdir($tag): $!\n";

    if( defined $src_workspace ) {
        # this isn't really a tag.  Set it to undef
        # to help catch errors.
        $tag = undef;
    }

	# Uses SubmitInfo.pm to get configuration info.
   	getSubmitInfoNew( $tag, $opt_src_tag );
   	$cmdfile = createSubmitFilesNew( $tag, $opt_src_tag, $opt_other_tag, $opt_importance );
   	submitRun( $cmdfile, $tag );
}

print "All steps completed successfully\n";
myExit( 0 );



######################################################################
# functions
######################################################################

sub initialize
{
    # initialize these first, so we don't create the workspace until 
    # afer we're sure these are valid and working...

    $nmi_bin = $ENV{'NMI_BIN'} || $default_nmi_bin;
    if( ! -d $nmi_bin ) { 
        die "$0: FATAL error\n" .
            "Directory for NMI binaries ($nmi_bin) doesn't exist\n" .
            "Please set your NMI_BIN environment variable correctly\n";
    }
    if( $opt_debug ) {
        $nmi_submit = "/bin/cat";
    } else {
        $nmi_submit = "${nmi_bin}/nmi_submit";
    }

    $condor_bin = $ENV{'CONDOR_BIN'} || $default_condor_bin;
    if( ! -d $condor_bin ) { 
        die "$0: FATAL error\n" .
            "Directory for Condor binaries ($condor_bin) doesn't exist\n" .
            "Please set your CONDOR_BIN environment variable correctly\n";
    }

    # Setup our real system PATH, too, since (for now) NMI tools
    # depend on the caller's PATH to find condor tools like
    # condor_submit_dag
    $ENV{PATH} = "${nmi_bin}:${condor_bin}:" . $ENV{PATH};

    # finally, setup the local workspace for all our temp files
    $init_cwd = &getcwd();
    $workspace = "/tmp/condor_$runtype." . "$$" . "." . time;
    mkdir($workspace) || die "Can't create workspace $workspace: $!\n";
    chdir($workspace) || die "Can't chdir($workspace): $!\n";
}


sub initializeDB
{
    # Initialization of the DB-related paramters.  This is in a
    # seperate function because we don't always need to do all this
    # work (find and parse the NMI config file, die if there are
    # errors, etc).  We only need it if we have to query the DB (at
    # this point, only when we're invoked with "--platforms=all").

    print "Initializing Database Parameters\n";

    unless ($HAS_NMICONF) {
	die("ERROR: NmiConf not available, database inaccessible\n");
    }

    if( ! defined $nmiconf ) {
        die "FATAL: $0: No NMI config file specified.\n";
    } 
    if( ! -r $nmiconf ) {
        die "FATAL: $0: NMI config file $nmiconf is not readable.\n";
    }

    # parse the NMI-specific config file to get useful settings
    %conf_nmi = NmiConf::init($nmiconf);

    ## ---------------------------------------------
    ## Database Connection
    ## ---------------------------------------------
    my $DB_HOST = $conf_nmi{'DB_HOST'};
    my $DB_PORT = $conf_nmi{'DB_PORT'};
    my $DB_NAME = $conf_nmi{'DB_NAME'};
    my $DB_READER_USER = $conf_nmi{'DB_READER_USER'};
    my $DB_READER_PASS = $conf_nmi{'DB_READER_PASS'};

    my $DB_CONNECT_STR = "DBI:mysql:database=$DB_NAME;host=$DB_HOST".
                         ($DB_PORT ? ";port=$DB_PORT" : "");
    $db = DBI->connect($DB_CONNECT_STR, $DB_READER_USER, $DB_READER_PASS);
    unless (defined $db) {
       die("ERROR: Could not connect to database: ${DBI::errstr}\n");
    }
}


sub parseOptions
{
    my $platform;
    my $rc;

    $rc = GetOptions(
# primary switch: build vs. test
           'build'         => \$opt_build,
           'test'          => \$opt_test,
# global options
           'help'          => \$opt_help,
           'debug'         => \$opt_debug,
           'notify=s'      => \$opt_notify,
           'notify-fail-only'      => \$opt_notify_fail_only,
           'platforms=s'   => \$opt_platforms,
           'nmi-glue=s'    => \$opt_nmi_glue,
           'workspace=s'   => \$opt_workspace,
           'git'          => \$opt_git,
           'tag=s'         => \$opt_tag,
           'module=s'      => \$opt_module,
           'nmiconf=s'     => \$opt_nmiconf,
           'desc=s'        => \$opt_desc,
           'append=s'      => \$opt_append,
           'importance=i'  => \$opt_importance,
           'submit-xtests!' => \$opt_submit_xtests,
		   # This next option is poorly used and should be removed/redone.
		   # It is used in build/platform_post to pass stuff to the test
		   # glue. It should really be renamed to testglue-configure-append
		   # or something like that. I need to think about it more.
		   'configure=s' => \$opt_configure,
		   'src-tag=s' => \$opt_src_tag,
		   'other-tag=s' => \$opt_other_tag,
# build-specific options
           'nightly'       => \$opt_nightly,
           'externals=s'   => \$opt_externals,
           'without-tests' => \$opt_without_tests,
           'use_externals_cache' => \$opt_use_externals_cache,
           'use-externals-cache' => \$opt_use_externals_cache,
           'clear_externals_cache' => \$opt_clear_externals_cache,
           'clear-externals-cache' => \$opt_clear_externals_cache,
           'clear_externals_cache_weekly' => \$opt_clear_externals_cache_weekly,
           'clear-externals-cache-weekly' => \$opt_clear_externals_cache_weekly,
           'clear-externals-cache-daily' => \$opt_clear_externals_cache_daily,
           'externals-only' => \$opt_externals_only,
		   'release-only' => \$opt_release_only,
		   'version-buildid=s' => \$opt_version_buildid,
           'add-build-args=s' => \$opt_build_args,
		   'skip-doc' => \$opt_skip_doc,
		   'sha1=s' => \$opt_sha1,
		   'private-build:s' => \$opt_private_build,
		   'coverity-analysis' => \$opt_coverity_analysis,
# test-specific options
           'buildid=i'     => \$opt_buildid,
           'test-src=s'    => \$opt_test_src,
           'test-timeout=i' => \$opt_test_timeout,
           'test-class=s'   => \$opt_test_class,
           'input-platform=s' => \$opt_input_platform,
		   'tests-at-once=i' => \$opt_tests_at_once,
		   'ignore-missing-platforms' => \$opt_ignore_missing_platforms,
		   'test-sources-from-workspace' => \$opt_test_sources_from_workspace,
		   'append-config-dir=s' => \$opt_append_config_dir,
    );

    if( !$rc ) {
        printUsage( 1 );
    }
    if( defined $opt_help ) {
        printUsage( 0 );
    }

    if( defined $opt_build ) {
        if( defined $opt_test ) {
            print "ERROR: You cannot specify both --build and --test\n";
            printUsage( 1 );
        }
        $runtype = "build";
    } elsif( defined $opt_test ) {
        $runtype = "test";
    } else {
        print "ERROR: You must specify either --build or --test\n";
        printUsage( 1 );
    }

	if(! defined( $opt_platforms ) ) {
		$opt_platforms = "all";
	}

	if( defined $opt_platforms ) {
		foreach $platform ( split(/\s*,\s*/, $opt_platforms) ) {
			if( $platform eq "all" ) {
				if($runtype eq "build") {
					my $file = dirname($0) . "/nmi-build-platforms";
					open(IN, '<', $file) or die("Cannot open $file for reading: $!");
					while(<IN>) {
						chomp;
						next if(/^\s*#/);
						next if(/^\s*$/);
						$opt_platforms{$_} = 1;
					}
					close(IN);
					next;
				}
				elsif($runtype eq "test") {
					if( ! defined $opt_buildid ) {
						print "ERROR: --platforms=all only works with --buildid\n";
						printUsage( 1 );
					}
				}
			}

			# Thus, only tests will have 'all' in their platform hash.
			$opt_platforms{$platform} = 1;
		}
	}

    # do some sanity checking for test submit...
    if( $runtype eq "test" ) {
        if( ! defined $opt_platforms ) {
            print "ERROR: You must specify --platforms with --test\n";
            printUsage( 1 );
        }
        if( ! defined $opt_buildid ) {
            print "ERROR: You must specify --buildid with --test\n";
            printUsage( 1 );
        }
        if( defined($opt_externals) ) {
            print "ERROR: --externals is not valid with --test\n";
            printUsage( 1 );
        }
        if( defined($opt_nightly) ) {
            print "ERROR: --nightly is not valid with --test\n";
            printUsage( 1 );
        }
        if( defined($opt_use_externals_cache) ) {
            print "ERROR: --use-externals-cache is not valid with --test\n";
            printUsage( 1 );
        }
        if( defined($opt_clear_externals_cache) ) {
            print "ERROR: --clear-externals-cache is not valid with --test\n";
            printUsage( 1 );
        }
        if( defined($opt_clear_externals_cache_weekly) ) {
            print "ERROR: --clear-externals-cache-weekly is not valid with --test\n";
            printUsage( 1 );
        }
        if( defined($opt_clear_externals_cache_daily) ) {
            print "ERROR: --clear-externals-cache-daily is not valid with --test\n";
            printUsage( 1 );
        }
        if( defined($opt_externals_only) ) {
            print "ERROR: --externals-only is not valid with --test\n";
            printUsage( 1 );
        }
		if( defined($opt_release_only) ) {
			print "ERROR: --release-only is not valid with --test\n";
			printUsage( 1 );
		}
		if( defined($opt_version_buildid) ) {
			print "ERROR: --version-buildid is not valid with --test\n";
			printUsage( 1 );
		}
		if( defined($opt_build_args) ) {
			print "ERROR: --opt-build-args is not valid with --test\n";
			printUsage( 1 );
		}
    } else {
        if( defined $opt_input_platform ) {
            print "ERROR: --input_platform is not valid with --build\n";
            printUsage( 1 );
        }
    }

    if( defined $opt_notify ) {
        $notify = "$opt_notify";
    }

	if( defined $opt_notify_fail_only ) {
		$notify_fail_only = 1;
	}

	if ( defined $opt_tag ) {
		if((defined $opt_src_tag) or (defined $opt_other_tag)) {
			print "ERROR: Either build from same branch in all repositories\n";
			print "or you use a unique branch for source with another for all\n";
			print "the other repositories\n";
			print "Either --tag=AAAAA Or --src-tag=BBBBB --other-tag=CCCCC\n";
	    	printUsage( 1 );
		}
	}

    if ( defined $opt_tag && defined $opt_module ) {
        $tags{"$opt_tag"} = $opt_module;
    } elsif ( defined $opt_src_tag && defined $opt_other_tag ) {
	    print "You requested a split fetch from two branches\n";
        $tags{"$opt_other_tag"} = "split_fetch";
    } elsif ( defined $opt_nightly ) {
        print "You specified --nightly, will munge tag name with --desc for " . 
				"Condor's NMI pages\n";
    } else {
		if( ! defined $opt_workspace ) {
			print( "Using default workspace (the parent directory).\n" );
			$opt_workspace = "..";
		}

		$src_workspace = File::Spec->rel2abs($opt_workspace);
    }

	if ( defined($opt_nightly) && !(defined($opt_tag) || defined($opt_module)))
	{
		print "ERROR: You need to specify --tag and --module with --nightly\n";
		printUsage( 1 );
	}

    if( defined($opt_externals) ) {
        if( defined($opt_nightly) ) {
            print "ERROR: You cannot use --externals with --nightly\n";
            printUsage( 1 );
        }
    }

    if( defined($opt_without_tests) ) {
        if( defined($opt_submit_xtests) && $opt_submit_xtests == 1) {
          print "ERROR: You cannot use --without-tests and --submit-xtests\n";
          printUsage( 1 );
        }
    }

    if( defined $opt_workspace ) {
        if( defined $opt_nightly ) {
            print "ERROR: You cannot use --workspace with --nightly\n";
            printUsage( 1 );
        }
        if(defined $opt_module ) {
            print "ERROR: You cannot use --workspace with --module\n";
            printUsage( 1 );
        }
        if( not -d $opt_workspace ) {
            print "ERROR: Workspace $opt_workspace does not exist\n";
            printUsage( 1 );
        }

        my(@condor_req_dirs) = qw( src );
        if ($runtype eq "build") {
           if( ! defined($opt_skip_doc )) { push @condor_req_dirs, 'docs'; }
           if( ! defined($opt_externals) ) { push @condor_req_dirs, 'externals'; }
        }
    	foreach my $subdir (@condor_req_dirs) {
              if( not -d "$opt_workspace/$subdir" ) {
                print "ERROR: Workspace $opt_workspace does not not appear to be a Condor workspace (missing $opt_workspace/$subdir)\n";
                printUsage( 1 );
            }
        }

        $tags{"workspace"} = undef;
    }

    if( defined($src_workspace) && defined($opt_nmi_glue) && -d "$src_workspace/nmi_tools/glue" ) {
        # If you want to try and fix the race condition, it will
        # involve somehow not copying the $src_workspace/nmi_tools/glue directory
        # in generateBuildFetchFiles().
        print "WARNING: Your workspace ($src_workspace) contains an nmi_tools/glue subdirectory.  You also specified an --nmi-glue directory ($opt_nmi_glue).  At the moment there is a race condition and you could get either directory.  As a workaround, rename %src_workspace/nmi_tools/glue.\n";
    }

    if( defined($opt_nmi_glue) ) {
        # SCP the glue from different location
        if ( not -d "$opt_nmi_glue/$runtype" ) {
            print "Glue directory $opt_nmi_glue/$runtype does not exist\n";
            printUsage( 1 );
        }
    }

    if( defined($opt_test_timeout) ) {
        $test_timeout = $opt_test_timeout;
    }

    if( defined $opt_nmiconf ) {
        if( -f $opt_nmiconf ) {
            $nmiconf = $opt_nmiconf;
        } else {
            print "NMI Config file '$opt_nmiconf' does not exist\n";
            printUsage( 1 );
        }
    }

	if( defined($opt_private_build) ) {
		$private_build = $opt_private_build;
	}

	if( defined( $opt_append_config_dir ) ) {
		if( ! -d $opt_append_config_dir ) {
			print( "No such directory '${opt_append_config_dir}'.\n" );
			myExit( 1 );
		}

		if( ! -f "${opt_append_config_dir}/testconfigappend" ) {
			print( "The append-config-dir must contain a file name 'testconfigappend'.\n" );
			myExit( 1 );
		}
	}
}

# TODO: this should be more clear about the required vs. optional
# arguments, especially for --test...
sub printUsage
{
    my $exit_code = shift;
    print <<END_USAGE;

condor_nmi_submit.pl --build [options] [build-options]
condor_nmi_submit.pl --test  [options] [test-options]

Where [options] can be any of:
--help             <This screen>
--debug            <Print out the generated submit files, don't submit them>
--notify=string    <Comma-seprated list of users to notify with results>
--notify-fail-only  <Only send notification email on build or test failure>
--platforms=string <Comma-sperated list of platforms to run tests on. A
                    platform can be a 'build-and-test-set name', a 'name', or
                    a 'name#key' pair. A build-and-test-set name must match
                    a pre-defined build-and-test-set name in SubmitInfo.pm.
                    A bare platform is its own key and uses the information
                    associated with they key from the %submit_info hash in 
                    the batlab. Bare names that do not exist in the 
                    %submit_info hash will be assigned the 
                    'default_minimal_platform' key. The only valid keys
                    are those that exist in the %submit_info hash table
                    in SubmitInfo.pm.>
--tag=string       <Condor source code tag to be fetched from cvs>
--module=string    <Condor source code module to be fetched from cvs>
--nmi-glue=string  <nmi_glue directory containing customised glue scripts>
--workspace=path   <directory containing Condor source to build> 
--nmiconf=string   <full path to the NMI config file to use if needed
                    (defaults to /nmi/etc/nmi.conf)>
--desc=string      <arbitrary text for a human readable description of the job>
--append=string    <arbitrary text to be appended to the NMI submit file>
--submit-xtests    <should this run submit the cross-platform tests defined
                    in the submit_info file?>
--src-tag          <source only tag to use/ someone's branch>
--other-tag        <tag to use for other repositories. Must be used with
					--src-tag>

Where [build-options] can be any of:
--nightly          <Munges the description of a build to be programmatically
                    understood by the Condor NMI pages. You must still supply
                    --tag and --module.>
--externals=string <Module name to use for fetching externals (only needed 
                    if the tag defined with --tag doesn't define them)>
--without-tests    <Submit a build job such that it will NOT submit
                    test runs as each platform completes>
--use-externals-cache   Use the externals cache stored in /scratch on execute
                        nodes. Do NOT use this option if your build is testing
                        a new external.
--clear-externals-cache Delete the externals cache before building. This can
                        be done along with --use_externals_cache.
--externals-only  Perform a build that only builds the externals. This is
                  useful when testing a new external on all platforms.
--release-only    Perform a build that only executes 'make release',
                  i.e. the public, stripped, static, etc targets are
                  skipped. This can help in performing quick complication
                  on all platforms. Do NOT use this option if you want to
                  submit test runs based on this build.
--version-buildid=string <Build RunId to use in the version string>
--add-build-args=string <Specify extra build config arguments>
--sha1            Place the sha1 into the database as the product_version.
--skip-doc        skips doc requirement
Where [test-options] can be any of:
--buildid=string   <Build RunId to test>
--test-src=string  <Custom source tarball for building tests with>
--test-class=string <Arguments to select what tests to declare>
--test-timeout=int <timeout in minutes for each test task (default is 20)>
--input-platform=string <Fetch binaries from this platform from given buildid>
--tests-at-once=count <run count tests concurrently>
--platforms=all    <will query DB for all platforms in the given buildid>

END_USAGE

    myExit( $exit_code );
}


sub getTags
{
    # Sanity checking 
    if( (scalar keys %tags > 1) && (exists $tags{'workspace'}) ) {
        # parseOptions should have caught this, but just in case.
        die "Internal consistency error: Multiple tags present, one of which is 'workspace'.  This is probably a --workspace build which has no hope of working with --tag or --nightly builds.";
    }
}


sub getSubmitInfoNew
{
    my( $tag, $src_tag ) = @_;
    my $submit_info_path;
	my $ret;
	our $slaved_module = 1; # needed for loading the SubmitInfo.pm module 
	my $p;
	my $i;
	my ($platform_name, $platform_key, $pkeyp);

    if( defined $opt_nmi_glue ) {
        $submit_info_path = "$opt_nmi_glue/$submit_info_basename";
    }
    elsif( defined $src_workspace ) {
        $submit_info_path = "$src_workspace/$submit_info";
        # in this case $tag is worthless, so trying to
        # check it out is hopeless.
    }
    else {
		if(defined $src_tag) {
			my $realtag = "origin\/" . $src_tag;
        	checkoutSubmitInfo( $realtag ) ||
	    		die "$tag does not contain $submit_info!\n";
		} else {
        	checkoutSubmitInfo( $tag ) ||
	    		die "$tag does not contain $submit_info!\n";
		}
        $submit_info_path = $submit_info;
    }

    -f $submit_info_path || die "$submit_info_path doesn't exist!\n";

	loadModule($submit_info_path, "SubmitInfo");

	# Figure out what platforms upon which I need to build.
	# I'm going to get it from the command line only, from the
	# build and test set in SubmitInfo, or a combination of the two.
	# If testing, then dig around in the DB with the runid for the platform
	# to get which platforms the build was for.
    if( %opt_platforms ) {
        if( defined $opt_platforms{all} ) {
            # we already verified the command-line, so we know we've
            # got opt_buildid and we're in test mode... 
            if( ! defined $opt_buildid ) {
                die "IMPOSSIBLE: saw --platforms=all without --buildid\n";
            }
            @platforms = ();
            print "Saw --platforms=all, getting platforms from build runid: "
                . "$opt_buildid\n";
            @platforms = getPlatformsFromRunid( $opt_buildid );
        } else {
			# We're in build mode so pick through the supplied platforms and
			# either add the build and test set specified, or the specific
			# platform mentioned.

			foreach $p (sort keys %opt_platforms) {
				($platform_name, $platform_key, $pkeyp) = 
					convertPlatformToKeys($p);

				if (exists($SubmitInfo::build_and_test_sets{$platform_name})) {
					if ($pkeyp) {
						die "ERROR: A build and test set name with a " .
							"platform key is unimplemented and undefined. " .
							"Don't do that.";
					}
					push @platforms, 
						@{$SubmitInfo::build_and_test_sets{$platform_name}};
				} else {
					# We preserve any platform key that may be already 
					# associated with this platform.
					push @platforms, $p;
				}
			}
		}
    } else {
		# we default to the 'official_ports' key in the build and test set
        @platforms = @{$SubmitInfo::build_and_test_sets{'official_ports'}};
    }

	@platforms = sort(removeDuplicates(@platforms));

	# For any platforms we don't know about, give it a platform key to the
	# default minimal build platform in %submit_info.

	for(my $i = 0; $i < scalar(@platforms); $i++) {
		($platform_name, $platform_key, $pkeyp) = 
			convertPlatformToKeys($platforms[$i]);

		if ($platform_name =~ /^$/) {
			die "ERROR: A platform name must be a defined value!";
		}

		if ($pkeyp) {
			# don't assume the user knows what they are doing with the platform
			# key. If they specified one, it *must* exist in the SubmitInfo
			# database. This doesn't check to see if platform_name is actually
			# meaningful, however.
			if (!exists($SubmitInfo::submit_info{$platform_key})) {
				die "ERROR: While processing platform name: " .
					"'${platform_name}', I found that the platform key: " .
					"'${platform_key}' does not exist!";
			}
		}

		if (!exists($SubmitInfo::submit_info{$platform_key})) {
			print "No platform key for platform: '${platform_name}' " .
				"found. Assigning key: 'default_minimal_platform' " .
				"to it.\n";
			$platforms[$i] = "${platform_name}#default_minimal_platform";
			}
		}

	# We print out the full name of these platforms, even if they want a
	# specified key.
	# print "Found platforms: " . join( ', ', @platforms ) . "\n";

	#
	# Conform @platforms to an external whitelist.  This allows us to
	# easily and temporarily turn off builds for some platform(s),
	# without leaving a long tail in git.
	#
	my %allowedPlatforms;
	my $file = "/home/condorauto/permitted-build-platforms";
	if( ! open( IN, '<', $file ) ) {
		print "WARNING -- unable to open /home/condorauto/permitted-build-platforms, permitting all platforms.\n";
		print "Found platforms: " . join( ', ', @platforms ) . "\n";
		return;
	}
	while( <IN> ) {
		chomp;
	    next if(/^\s*#/);
	    next if(/^\s*$/);
	    $allowedPlatforms{ $_ } = 1;
	}
	close( IN );

	my $i = 0;
	while( $i < scalar( @platforms ) ) {
		my $platform = $platforms[$i];
	    if( ! exists( $allowedPlatforms{ $platform  } ) ) {
			splice( @platforms, $i, 1 );
	    } else { ++$i; }
	}

	print "Found platforms: " . join( ', ', @platforms ) . "\n";
}

sub loadModule
{
	# module_path contains the path up to and including the module name.
	# module_name just includes the module name without a path or suffix.
	my ($module_path, $module_name) =  @_;
	my $mpath;
	
	$mpath = dirname($module_path);

	# Here we make the idempotency check fail so
	# the new module gets loaded and stomps on the old module's
	# definitions--which is what we want. This is vile and ugly but does the
	# thing people implicitly expect.
	if (exists($INC{"$module_name.pm"})) {
		print "Removing perl's module idempotency check to allow reloading " .
				"of $module_name.pm with new definitions.\n";
		delete $INC{"$module_name.pm"};
	}

	# I need the load to happen with runtime information, not compile time, so
	# this is why I'm messing about with @INC here.
	if (scalar(grep(/^\Q$mpath\E$/, @INC)) == 0) {
		# Only push path if not already present.
		push @INC, $mpath;
	}
	print "About to load module: $module_path\n";
	print "You can ignore the warnings about function redefinitions, if any.\n";
	eval "use $module_name;";
	die $@ if $@;
	print "Loaded $module_name.pm!\n";
}

# takes an array, returns an array with duplicates elements removed.
sub removeDuplicates
{
	my @array = @_;
	my %hash;
	my $k;
	my @new;

	foreach $k (@array) {
		$hash{$k} = 1;
	}
	@new = keys %hash;

	return @new;
}

sub createSubmitFilesNew
{
    my ( $tag, $srctag, $othertag, $importance ) = @_;
    my $module;
    my @fetch_files;
    my $cmdfile = "cmdfile";

	print "createSubmitFiles: $tag, $srctag, $othertag\n";

    if( defined $tag ) {
        $cmdfile .= "-$tag";
	$cmdfile =~ s,/,,g;
    } else {
        $cmdfile .= "-workspace";
    }

    if( defined $tag ) {
        $module = $tags{$tag};
    }
    
    # Generate the cmdfile
    open(CMDFILE, ">$cmdfile") || die "Can't open $cmdfile for writing: $!\n";

    writeCommonInfo( *CMDFILE, $tag, $module );

    # define inputs
    if( $runtype eq "build" ) {
        @fetch_files = generateBuildFetchFiles( $tag, $module, $opt_src_tag, $opt_other_tag );
    } else {
        @fetch_files = generateTestFetchFiles( $tag, $module, $opt_src_tag, $opt_other_tag );
    }
    print CMDFILE "inputs = " . join(', ', @fetch_files) . "\n";


	if( $runtype eq "build" ) {
		print CMDFILE "run_type = build\n";
		writeBuildPrereqsNew( *CMDFILE );
		writeBuildGlueNew( *CMDFILE, $tag, $module );
		writeBuildXTestSpecsNew( *CMDFILE );

		# This disk requirement is intended to override the default
		# requirement inserted by condor_submit.  This is a temporary
		# workaround for the problem of interrupted jobs failing to
		# rerun because the slot Disk advertised by a number of platforms
		# is not actually big enough for Condor builds.  The automatic
		# requirements fail because the job's DiskUsage gets updated
		# in the first attempt to run the job and then prevent the job
		# from matching in the second attempt.
		#
		# Then, only run this job on machines that advertise the are for batlab
		print CMDFILE "append_requirements = Disk =!= UNDEFINED && TotalDisk >= 10000000 && DedicatedForBaTLab == true\n";

	} else {

		print CMDFILE "run_type = test\n";
		print CMDFILE "platform_job_timeout = 16h\n";
		# allow tests on changing systems like unmanaged-x86_rhap_5.1"
		print CMDFILE "++unmanaged = true \n";
		print CMDFILE "append_requirements = DedicatedForBaTLab == true\n";

		writeTestPrereqsNew( *CMDFILE );
		writeTestGlueNew( *CMDFILE );
	}

	# Not all builds should be public (e.g. security releases).  Set the flag
	# if defined
	if (defined($private_build)) {
		my $private_username;
		if ($private_build ne "") {
			$private_username = $private_build;
		} else {
			$private_username = "condor-team";
		}

		print "PRIVATE BUILD: $private_username\n";
		print CMDFILE "private_web_users = " . $private_username . "\n";
	} else {
		print "PUBLIC BUILD\n";
	}


	# Sometimes builds and tests get backed up.  If so, we would 
	# like to eventually run all the jobs, but we'd like to see results from
	# newer ones first.  Setting the user job prio to the Qdate should
	# help with this.

	my $job_prio = time;
	if (defined($importance)) { $job_prio = $job_prio + ($importance * 3600); }
	# Lower priority of emulated jobs
	print CMDFILE "++jobprio = $job_prio - (regexp(\"x86_64\", (nmi_target_platform ?: \"x86_64\")) ? 0 : 86400)\n";

	# Glorious hack.
	# It is best if the 'build' and 'test' jobs request the same amount of memory
	# Then we reuse the dynamic slots efficiently on our Docker build machines
	if( $runtype eq "build" ) {
		# Request 1 core.  It gets quantized up to four or eight cores for some machines
		print CMDFILE "request_cpus = 1\n";
		# 9GiB (expect 8 cores, 1+ GiB per core)
		print CMDFILE "request_memory = 9G\n";
		# ARM64 builds on EL9 take a huge amount of memory creating condor_util library
		print CMDFILE "nmi-build:aarch64_AlmaLinux9_request_memory = 20G\n";
		print CMDFILE "request_disk = 8G\n";

		# Select Docker NAT network for more reliable HTCondor startup
		print CMDFILE "+docker_network_type = nat\n";
	} else {
		# Request 1 core.  It gets quantized up to four or eight cores for some machines
		print CMDFILE "request_cpus = 1\n";
		# Match build memory request to prevent starvation.  Otherwise, this could be 4GiB
		print CMDFILE "request_memory = 9G\n";
		# Emulated plaforms need more memory for tests
		print CMDFILE "nmi-build:ppc64le_AlmaLinux8_request_memory = 12G\n";
		print CMDFILE "nmi-build:aarch64_AlmaLinux8_request_memory = 12G\n";
		print CMDFILE "nmi-build:aarch64_AlmaLinux9_request_memory = 12G\n";
		print CMDFILE "nmi-build:ppc64le_Ubuntu20_request_memory = 12G\n";
		print CMDFILE "request_disk = 8G\n";

		# Select Docker NAT network to avoid port exhaustion when running lots of test runs
		print CMDFILE "+docker_network_type = nat\n";
	}

	close CMDFILE;
    return $cmdfile;
}

sub checkoutSubmitInfo
{
    my( $tag ) = @_;
    my @output;
    my $retval;

    print "Attempting to fetch $submit_info from $tag...\n";
    my $cmd = "/bin/false FATAL ERROR, UNKNOWN VCS";
    if (defined($opt_git)) {
# This fails to find $tag on newer version of git.
#	$cmd = "git archive --remote=$GITROOT_SRC $tag $submit_info | tar x";
	$cmd = "git --git-dir=$GITROOT_SRC archive $tag $submit_info | tar x";
    }

    open( OUT, "$cmd 2>&1 |" ) ||
        die "Can't open '$cmd' as a pipe: $!\n";
    @output = <OUT>;
    close( OUT );
    $retval = $? / 256;
    if( $retval != 0 ) {
        print @output;
        print "FAILED (return value $retval)\n";
	print "FAILED (command was '$cmd')\n";
        return 0;
    } 
    if( ! -f $submit_info ) {
        print @output;
        print "FAILED ($submit_info does not exist)\n";
        return 0;
    }
    print "SUCCESS\n";
    return 1;
}


sub clearGlobalSubmitInfo
{
    undef $global_prereqs;
    undef $global_test_prereqs;
    undef $global_testargs;
    undef %platform_prereqs;
    undef %platform_test_prereqs;
    undef %platform_x_test_platforms;
    undef %platform_testargs;
    undef %platforms;
    undef @platforms;
	# cross testing platform defs
    undef %testplatforms;
    undef %testplatform_prereqs;
    undef %testplatform_test_prereqs;
}


sub printSubmitInfo
{
    my $key;

    print "----- Start of data parsed from submit_info -----\n";
    print "Platforms: " . join(', ',sort(keys(%platforms))) . "\n";
    print "Global testargs: $global_testargs\n";
    print "Global prereqs: $global_prereqs\n";
    print "Global test prereqs: $global_test_prereqs\n";

    print "Platform-specific testargs:\n";
    foreach $key (sort keys %platform_testargs ) {
        print "  $key: $platform_testargs{$key}\n";
    }
    print "Platform-specific prereqs:\n";
    foreach $key (sort keys %platform_prereqs ) {
        print "  $key: $platform_prereqs{$key}\n";
    }
    print "Platform-specific testing prereqs:\n";
    foreach $key (sort keys %platform_test_prereqs ) {
        print "  $key: $platform_test_prereqs{$key}\n";
    }
    print "Cross-Platform testing:\n";
    foreach $key (sort keys %platform_x_test_platforms) {
        print "  $key: should also test on $platform_x_test_platforms{$key}\n";
    }
    print "----- End of data parsed from submit_info -----\n";
}

# We convert a platform string like "ia64_rhas_3" or the alternate form for a
# minimal build configuration like: "ia64_rhas_3#default_minimal_platform" into
# the actual platform name, and the platform key for which we should get the
# information from the %submit_info hash table. They will be identical for the
# first case. The key will be "default_minimal_platform" for the second
# case. 
#
# Return the platform name, the platform key, and a boolean for whether or not
# they are different (true if different).

sub convertPlatformToKeys
{
	my $platform = shift @_;
	my ($platform_name, $platform_key, $pkeyp);

	($platform_name, $platform_key) = split(/#/, $platform);
	$platform_name =~ s/^\s+//g;
	$platform_name =~ s/\s+$//g;
	$platform_key =~ s/^\s+//g;
	$platform_key =~ s/\s+$//g;

	if( $platform =~ m!^docker://(.*)! ) {
		$platform_key = basename($1);
		$platform_key =~ s/-\d+$//; # Trim off any tag version
	}

	# assume they are the same unless I can show otherwise.
	$pkeyp = 0;

	if (!defined($platform_key)) {
		# No specific key mentioned, then they are the same.
		$platform_key = $platform;
		$platform_name = $platform;
	} else {
		# there is a specific key...
		if ($platform_name ne $platform_key) {
			# and they are different from each other.
			$pkeyp = 1; 
		}
	}

	return ($platform_name, $platform_key, $pkeyp);
}

sub writeTestPrereqsNew
{
    my $fh = shift;
    my $platform;

	print $fh <<EOF;
###############################################################################
# Test Pre-requisites
###############################################################################

# Global test prerequesites:
#	This is hard coded to none, since each platform now
#	provides their own specific prereqs.
prereqs =

EOF

	# emit the test prereqs, if any, for each platform
    foreach $platform ( @platforms ) {
		my ($platform_name, $platform_key, $pkeyp);

		($platform_name, $platform_key, $pkeyp) = 
			convertPlatformToKeys($platform);

		print $fh "# Test Prereqs for Platform ${platform_name}\n";
		if ($pkeyp) {
			print $fh "# Using the platform key ${platform_key} for " .
				"configuration values.\n";
		}

		if (defined($SubmitInfo::submit_info{$platform_key}{'test'}{'prereqs'})) {
            # print $fh "prereqs_${platform_name} = " .
            print $fh "prereqs_${platform_key} = " .
				join(', ', @{$SubmitInfo::submit_info{$platform_key}{'test'}{'prereqs'}}) .
				"\n";
		} else {
			print $fh "# None.\n";
		}

		print $fh "\n";
	}

}

sub writeBuildPrereqsNew
{
    my $fh = shift;
    my $platform;
	my $bref;

	print $fh <<EOF;
###############################################################################
# Build Pre-requisites
###############################################################################

# Global build prerequesites:
#	This is hard coded to none, since each platform now
#	provides their own specific prereqs.
prereqs =

EOF

	# Now, for each platform, write out the prereqs associated with that
	# platform.

    foreach $platform ( @platforms ) {
		my ($platform_name, $platform_key, $pkeyp);

		($platform_name, $platform_key, $pkeyp) = 
			convertPlatformToKeys($platform);

		print $fh "# Build prereq for: ${platform_name}\n";
		if ($pkeyp) {
			print $fh "# Using the platform key ${platform_key} for " .
				"configuration values.\n";
		}

        if (defined($SubmitInfo::submit_info{$platform_key}{'build'}{'prereqs'})) 
		{
            # print $fh "prereqs_${platform_name} = " .
            print $fh "prereqs_${platform_key} = " .
				join (', ', @{$SubmitInfo::submit_info{$platform_key}{'build'}{'prereqs'}}) .
				"\n";
        } else {
			print $fh "# None.\n";
		}
		print $fh "\n";
    }
}

sub writeBuildXTestSpecsNew
{
    my $fh = shift;
	my $platform;

	print $fh <<EOF;
###############################################################################
# Per Platform Cross Test Specifications
###############################################################################

EOF

    # Drop the info on desired cross testing into environment
    # via entries in the command file.
    if( defined($opt_submit_xtests) && $opt_submit_xtests == 1) {

		foreach $platform (sort @platforms) {
			my ($platform_name, $platform_key, $pkeyp);

			($platform_name, $platform_key, $pkeyp) = 
				convertPlatformToKeys($platform);

			print $fh "# Cross test platforms for ${platform_name}\n";
			if ($pkeyp) {
				print $fh "# Using the platform key ${platform_key} for " .
					"configuration values.\n";
			}

			if (defined($SubmitInfo::submit_info{$platform_key}{'build'}{'xtests'}))
			{
				print $fh "x_test_platforms_${platform_name} = " .
					join(', ', @{$SubmitInfo::submit_info{$platform_key}{'build'}{'xtests'}}) .
					"\n";

			} else {
				print $fh "# None.\n";
			}

			print $fh "\n";
		}
    }
}

sub writeCommonInfo
{
    my ( $fh, $tag, $module ) = @_;

    my $desc;
    my $vers_string;
    my @vers;
	my ($platform, $platform_name, $platform_key, $pkeyp, @platform_names);

    if( defined $opt_src_tag &&  defined $opt_other_tag ) {
		if(defined $opt_desc) {
			# --desc overrides --tag value starting the description
			$desc = $opt_desc;
		} else {
        	($desc, @vers) = parseTag( $tag );
        	$vers_string = join( ',', @vers );
		}
		$tag = "origin\/" . $opt_src_tag;
		print "****** split fetch tag now <$tag> ******\n";
    } elsif( defined $tag ) {
		if (defined($opt_desc) && defined($opt_nightly)) {
			# --desc gets merged with the $desc so it shows up properly
			# on our Condor NMI build and test display pages.
        	($desc, @vers) = parseTag( $tag );
			$desc = "$opt_desc - $desc";
        	$vers_string = join( ',', @vers );
		} elsif (defined $opt_desc) {
			# --desc overrides --tag value starting the description
			$desc = $opt_desc;
		} else {
        	($desc, @vers) = parseTag( $tag );
        	$vers_string = join( ',', @vers );
		}
    } elsif( defined $src_workspace ) {
        $desc = 'workspace';
    	if ( defined $opt_desc ) {
        	$desc = $desc . ": $opt_desc";
    	}
    } else {
        $desc = 'unknown';
    	if ( defined $opt_desc ) {
        	$desc = $desc . ": $opt_desc";
    	}
    }

    #if ( defined $opt_desc ) {
        #$desc = $desc . ": $opt_desc";
    #}

    print $fh "description = $desc\n";
    print $fh "project = condor\n";
	if(defined $opt_sha1) {
        print $fh "project_release = $opt_sha1\n";
    }elsif( defined $vers_string ) {
        print $fh "project_release = $vers_string\n";
    } else {
		# Now that we care about comparing this to
		# to component version to sort displays of
		# native vs pre-binary cross testing lets loose
		# the uninitialized NULL field.
		print $fh "project_release = unknown workspace\n";
	}

    print $fh "component = condor\n";
	if( defined $opt_input_platform ) {
		# brand binaries being tested on this platform so we can find it later
		# its a cross test
		# adjust for api change where all platforms now begin with "nmi:".
		my $adjusted_platform = "nmi:" . "$opt_input_platform";
        print $fh "component_version = $adjusted_platform\n";
    } elsif( $runtype eq "test") {
		# when the optional input is not used its a native test 
        print $fh "component_version = native\n";
	} else {
		# build - so keep marking it as it was
    	if( defined $vers_string ) {
        	print $fh "component_version = $vers_string\n";
    	}
	}
	if( $notify ) {
	    print $fh "notify = $notify\n";
	}
    print $fh "priority = 1\n";
    if ($runtype eq "build") { print $fh "use_remote_post_rval = true\n"; }

	if (defined $notify_fail_only) {
    	print $fh "notify_fail_only = true\n";
	}

	# The following is a short-termhack to work around some condor
	# problems.  This allows our tests to generate core files
	print $fh "+coresize = 100000000\n";

    # print the tag and module in the cmdfile as the user variables
    if( defined $tag ) {
		print "****** print tag and module<$tag><$module> ******\n";
        print $fh "tag = $tag\n";
        print $fh "module = $module\n";
    }
    if ( defined $opt_append ) {
        print $fh "$opt_append\n";
    }

    if ( defined $opt_configure ) {
        print $fh "configure = $opt_configure\n";
    }

    print $fh "++LoadProfile = true\n";

	# collect the names of the platforms, regardless of if they use the
	# default minimal builds or not.
	for $platform (@platforms) {
		($platform_name, $platform_key, $pkeyp) = 
			convertPlatformToKeys($platform);
		push @platform_names, $platform_name;
	}
    print $fh "platforms = " . join(', ', @platform_names) . "\n";

	# Intended so we can add DEBUG_PLATFORMS temporarily.
	my $file = "/home/condorauto/append-to-builds";
	open( IN, '<', $file ) or return;
	while( my $line = <IN> ) {
		print $fh $line;
	}
	close( IN );
}

sub writeBuildGlueNew
{
    my ( $fh, $tag, $module ) = @_;
    my $post_args = '';
    my $common_pre_args = '';
    # tag and module may be null (if $src_workspace is defined)
	my $platform;

	print $fh <<EOF;
###############################################################################
# Build glue specification.
###############################################################################

EOF

#     # define the glue scripts we want
    print $fh "pre_all = nmi_tools/glue/build/pre_all\n";
	my $pre_all_args = "";
	if (defined($opt_skip_doc)) {
		$pre_all_args = $pre_all_args . " --skip-doc";
	}

	if (!defined($opt_workspace)) {
		$pre_all_args = $pre_all_args . " --daily";
	}

    if (defined($opt_version_buildid)) {
      $pre_all_args = $pre_all_args . "--version-buildid=$opt_version_buildid";
    }
    print $fh "pre_all_args = $pre_all_args\n";

    print $fh "remote_declare = nmi_tools/glue/build/remote_declare.pl\n";
    print $fh "remote_pre = nmi_tools/glue/build/remote_pre.pl\n";
    print $fh "remote_task = nmi_tools/glue/build/remote_task.pl\n";
    print $fh "remote_post = nmi_tools/glue/build/remote_post.pl\n";
    print $fh "platform_post = nmi_tools/glue/build/platform_post\n";

	print $fh "post_all = nmi_tools/glue/build/post_all\n";

    if( defined($opt_coverity_analysis)) {
    	print $fh "remote_declare_args = --coverity-analysis\n";
    }

    if( defined($opt_use_externals_cache) ) {
		$common_pre_args .= ' --use-externals-cache';
    }
    if( defined($opt_clear_externals_cache) ) {
		$common_pre_args .= ' --clear-externals-cache';
    }
    if( defined($opt_clear_externals_cache_weekly) ) {
		$common_pre_args .= ' --clear-externals-cache-weekly';
    }
    if( defined($opt_clear_externals_cache_daily) ||
		defined($opt_nightly) ) {

		$common_pre_args .= ' --clear-externals-cache-daily';
    }
	print $fh "\n";

	print $fh <<EOF;
###############################################################################
# Arguments for remote_pre, per platform.
###############################################################################
EOF

	# emit the remote_pre args per platform and mix in the common remote pre
	# args if any.
	for $platform (sort @platforms) {
		my ($platform_name, $platform_key, $pkeyp);
		($platform_name, $platform_key, $pkeyp) = 
			convertPlatformToKeys($platform);

		my $href = $SubmitInfo::submit_info{$platform_key}{'build'}{'configure_args'};
		my $str = "";
		# Add in arguments remote_pre natively understands
		if ($common_pre_args) {
			$str = "$common_pre_args ";
		}

		# everything else will be configure arguments.
		$str .= "-- ";
		if (defined($href)) {
			$str .= join(' ', SubmitInfo::args_to_array($href));
		}
		if (defined($opt_build_args) ) {
			$str .= " " . $opt_build_args;
		}

		print $fh "# Remote Pre Arguments for Platform: ${platform_name}\n";
		if ($pkeyp) {
			print $fh "# Using the platform key ${platform_key} for " .
				"configuration values.\n";
		}
		# print $fh "${platform_name}_remote_pre_args = $str\n";
		print $fh "${platform_key}_remote_pre_args = $str\n";
		print $fh "\n";
	}

	print $fh <<EOF;
###############################################################################
# Arguments for every platform_post invocation
###############################################################################
EOF

    if( defined($opt_without_tests) ) {
        $post_args = '--without-tests';
    } else {
        # If a build is submitted using --submit-xtests, all we have
        # to do is propagate it to platform_post via arguments...
        if( defined($opt_submit_xtests) && $opt_submit_xtests == 1) {
            $post_args = "--submit-xtests";
        } else {
            $post_args = "--nosubmit-xtests";
        }
		# If tests are desired to be run concurrently pass it on
		if( defined $opt_tests_at_once ) {
			$post_args .= " --tests-at-once=$opt_tests_at_once";
		}
        # If a build is submitted using --git pass it on,
        # platform_post checks out the test_glue from some VCS
        if( defined($opt_git) ) {
            $post_args .= " --git";
        }       
        if( defined($src_workspace) ) {
            $post_args .= " --test-args=--workspace='$src_workspace'";
        }
    }
    if( $post_args ) {
        print $fh "platform_post_args = $post_args\n";
    } else {
        print $fh "# None\n";
	}
	print $fh "\n";
}

sub writeTestGlueNew
{
    my ( $fh ) = @_;
    # select scope of testsuite run
    my $args;
	my $platform;

	print $fh <<EOF;
###############################################################################
# Test glue specification.
###############################################################################

EOF

    # define the glue scripts we want for test jobs
    print $fh "pre_all = nmi_tools/glue/test/pre_all\n";
    if( defined $opt_test_src) {
        print CMDFILE "pre_all_args = --src=$opt_test_src\n";
    }
    print $fh "platform_pre = nmi_tools/glue/test/platform_pre\n";
    print $fh "remote_pre_declare = nmi_tools/glue/test/remote_pre_declare.pl\n";
    print $fh "remote_declare = nmi_tools/glue/test/remote_declare.pl\n";
    print $fh "remote_pre = nmi_tools/glue/test/remote_pre.pl\n";
    print $fh "remote_task = nmi_tools/glue/test/remote_task.pl\n";
    print $fh "remote_task_timeout = $test_timeout\n";
    print $fh "remote_post = nmi_tools/glue/test/remote_post.pl\n";
    print $fh "platform_post = nmi_tools/glue/test/platform_post\n";
    print $fh "post_all = nmi_tools/glue/test/post_all\n";
	# turn on testing concurrency
	if( defined $opt_tests_at_once) {
    	print $fh "max_concurrent_subtasks = $opt_tests_at_once\n";
	}
	print $fh "\n";

	print $fh <<EOF;
###############################################################################
# Test Class and Configure Args Specification, per platform.
###############################################################################

EOF
	foreach $platform (sort @platforms) {
		$args = "--test-class "; # clear out the value for each platform.

		my ($platform_name, $platform_key, $pkeyp);
		($platform_name, $platform_key, $pkeyp) = 
			convertPlatformToKeys($platform);

		print $fh "# Testclass/Configure args for platform ${platform_name}\n";
		if ($pkeyp) {
			print $fh "# Using the platform key ${platform_key} for " .
				"configuration values.\n";
		}

		# assemble the test classes from the platform or the command line.

    	if( defined $opt_test_class ) {
			# XXX can only specify a single test class on the command line
			# need to fix GetOpt to handle multiple ones.
			$args .= "$opt_test_class ";

		} else {
			# grab the test classes out of the SubmitInfo file;
			if (defined($SubmitInfo::submit_info{$platform_key}{'test'}{'testclass'}))
			{
				$args .= join(',', @{$SubmitInfo::submit_info{$platform_key}{'test'}{'testclass'}}) . " ";

			} else {
				print $fh "# No test class for ${platform_name}. Default to quick.\n";
   				$args .= "quick ";
			}
		}

		$args .= "-- ";

		# now, add in the configure args for the test suite.

		if (defined($SubmitInfo::submit_info{$platform_key}{'test'}{'configure_args'}))
		{
			$args .= join(' ', 
				SubmitInfo::args_to_array(
					$SubmitInfo::submit_info{$platform_key}{'test'}{'configure_args'}));
		}

		# finally print out the mess which is all passed to remote_declare.
		print $fh "${platform_key}_remote_declare_args = $args\n";
	}
}


sub generateTestFetchFiles
{
    my( $tag, $module ) = @_;
    my @fetch_files;

    @fetch_files = generateSharedFetchFiles( $tag );

    if( defined $src_workspace ) {
        # Workspace build.  Grab out glue from our workspace
        # (We can't assume it will be copied along with the workspace
        # since we don't copy the workspace in a test run.)
        my $glue_file = "test_glue.src";

        # We rather crudely copy all of nmi_glue when we really
        # on need nmi_glue/$runtype.  But we want the nmi_glue path
        # in front of it.
        makeFetchFileSCP( $glue_file, "$src_workspace/nmi_tools", "recursive" );
        push( @fetch_files, $glue_file );

    } elsif( ( ! defined $opt_nmi_glue) ) {
        # $opt_nmi_glue is handled in generateSharedFetchFiles().
        # however, if it's not set, and we're a test job, we need to add a
        # fetch file to checkout the glue from cvs so we have a working
        # copy in our userdir as soon as the fetching is done.  we can't
        # rely on the copy of the glue in the source tarball from the
        # build output, since we have no way to untar it without glue. ;)
        my $glue_file = "test_glue.src";
        makeFetchFileVCS( $glue_file, FALSE, SRC, "nmi_tools/glue/$runtype", $tag );
        push( @fetch_files, $glue_file );
    }
    # else: Not workspace build.  Using --nmi_glue 
    # (handled in generateSharedFetchFiles()).  Nothing to do.

	# if we want different tests then the build we are using had, bring sources
	# along. We could bring less then the entire workspace
    if(( defined $opt_test_sources_from_workspace ) && (defined $src_workspace)) {
		my $test_sources = "source-workspace.src";
		makeFetchFileSCP($test_sources, $src_workspace, "recursive");
		push(@fetch_files, $test_sources);
	}

	# generate the runid input file
	my $runid_file = "input_build_runid.src";
	open(RUNIDFILE, ">$runid_file") || 
	    die "Can't open $runid_file for writing: $!\n";
	print RUNIDFILE "method = nmi\n";
	print RUNIDFILE "input_runids = $opt_buildid\n";
	print RUNIDFILE "untar_results = false\n";     

	# well if we are testing off a previous build missing some results...
	if( defined $opt_ignore_missing_platforms ){
	    print RUNIDFILE "ignore_missing_platforms = true\n";
	} else {
	# allow non-matching test bimaries for cross tests
		if( defined($opt_submit_xtests) && $opt_submit_xtests == 1) {
	        print RUNIDFILE "ignore_missing_platforms = true\n";
	    } else {
	        print RUNIDFILE "ignore_missing_platforms = false\n";
	    }
	}

	if( defined $opt_input_platform ) {
	    my @fetch_platforms;
	    my $platform;
	    foreach $platform (@platforms ) {        
			# new double colon syntax as of 2.5.0
			my $target = "$opt_input_platform" . "::" . "$platform";
	        push( @fetch_platforms, $target );
	    }
	    print RUNIDFILE "platforms = " . join(', ', @fetch_platforms);
	}
	close RUNIDFILE;
	push( @fetch_files, $runid_file );

    if( defined $opt_test_src ) {
        my $test_src_file = "test_sources.src";
        makeFetchFileSCP( $test_src_file, $opt_test_src );
        push( @fetch_files, $test_src_file );
    }

    return @fetch_files;
}


sub generateBuildFetchFiles
{
    my( $tag, $module) = @_;
    my @fetch_files;
	my $srctag = "";
	my $othertag = "";


    if( (defined $tag ) ||(defined $opt_src_tag)) {

		if(defined $opt_src_tag) {
			$srctag = "origin\/" . $opt_src_tag;
			$othertag = "origin\/" . $opt_other_tag;
		} else {
			$srctag = $othertag = $tag;
		}

    	@fetch_files = generateSharedFetchFiles( $srctag );
        my $srcfile = "source-$srctag.src";
		$srcfile =~ s,/,,g;
        makeFetchFileVCS( $srcfile, TRUE, SRC, $module, $srctag );
        push( @fetch_files, $srcfile );

		# This is a bit of a hack, if we are using git then we need to
		# check the externals and manual out from different
		# repositories than the source. Ideally, this would only be
		# visible at the makeFetchFileVCS level.
#		if( defined $opt_git ) {
#	    	my $extfile = "externals-$othertag.src";
#	    	$extfile =~ s,/,,g;
#	    	makeFetchFileVCS( $extfile, TRUE, EXT, $module, $othertag );
#	    	push( @fetch_files, $extfile );
	
#	    	my $docfile = "manual-$othertag.src";
#	    	$docfile =~ s,/,,g;
#	    	makeFetchFileVCS( $docfile, TRUE, DOC, $module, $othertag );
#	    	push( @fetch_files, $docfile );

#	    	my $test_cnffile = "confidential-$othertag.src";
#	    	$test_cnffile =~ s,/,,g;
#	    	makeFetchFileVCS( $test_cnffile, TRUE, TST_CNF, $module, $othertag );
#	    	push( @fetch_files, $test_cnffile );

#	    	my $test_cnffile = "largetests-$othertag.src";
#	    	$test_cnffile =~ s,/,,g;
#	    	makeFetchFileVCS( $test_cnffile, TRUE, TST_LRG, $module, $othertag );
#	    	push( @fetch_files, $test_cnffile );
#		}
    } elsif( defined $src_workspace ) {
        # TODO: There is a race condition if the user also specified
        # --nmi-glue and this directory ($src_workspace) contains
        # an nmi_glue directory.  At the moment parseOptions warns
        # the user, but we don't cope with it.  The solution would
        # likely be: if $opt_nmi_glue is defined and 
        # $src_workspace/nmi_glue exists, somehow pass makeFetchFileSCP
        # a "don't copy this particular subdirectory" option.
        my $srcfile = "source-workspace.src";
    	@fetch_files = generateSharedFetchFiles( $tag );
        makeFetchFileSCP( $srcfile, $src_workspace, "recursive" );
        push( @fetch_files, $srcfile );
    } else {
        die "Internal consistancy error: build lacks a --tag/--nightly or a --workspace.";
    }

#    if( defined $opt_externals ) { 
#        my $ext_file = "externals.src";
#        makeFetchFileVCS( $ext_file, TRUE, EXT, $opt_externals );
#        push( @fetch_files, "$ext_file" );
#    }

    return @fetch_files;
}


sub generateSharedFetchFiles
{
    my( $tag ) = @_;

    my @fetch_files;

    if( defined($opt_nmi_glue) ) {
        my $glue_file = $runtype . "_glue.src";
        makeFetchFileSCP( $glue_file, $opt_nmi_glue, "recursive" );
        push( @fetch_files, "$glue_file" );
    }

	if( defined( $opt_append_config_dir ) ) {
		my $appendFile = "append_config_dir.src";
		makeFetchFileSCP( $appendFile, "${opt_append_config_dir}/testconfigappend" );
		push( @fetch_files, $appendFile );
	}

    return @fetch_files;
}


sub makeFetchFileVCS
{
    # NOTE: this one breaks the convention of $tag first, b/c for
    # things on the trunk (e.g. externals) there is no tag
    # NOTE2: The $is_module parameter is meant to help properly
    # identify the $module parameter as an actual CVS-style module,
    # i.e. a taget that can be checked out that does not directly
    # exist in the repository e.g. something from CVSROOT/modules, or
    # a target that exists, i.e. a file or directory in the
    # repository. $repo is used to target our checkout to either a
    # source repository or externals repository.
    my ( $file, $is_module, $repo, $module, $tag ) = @_;

	print "makeFetchFileVCS: $file, $is_module, $repo, $module, $tag\n";

    open( FILE, ">$file" ) || die "Can't open $file for writing: $!\n";
    if (defined($opt_git)) {
		# git_tree is a commit/tree, could can be a tag or a branch
		# git_path is essentially a filter, only pull git_path or git_path/...
		# ==> git archive --remote=<git_root> <git_tree> <git_path> | tar x
		# perl case clause to allow 3 different test repositories long term
		my $gitroot;
#		if( EXT == $repo) {
#			$gitroot = $GITROOT_EXT;
#		} elsif(DOC == $repo) {
#		if(DOC == $repo) {
#			$gitroot = $GITROOT_DOC;
#		} elsif(TST_CNF == $repo) {
#			$gitroot = $GITROOT_TSTCNF;
#		} elsif(TST_LRG == $repo) {
#			$gitroot = $GITROOT_TSTLRG;
#		} else {
#			$gitroot = $GITROOT_SRC;
#		}
		$gitroot = $GITROOT_SRC;
		print FILE "method = git\n";
		print FILE "git_root = $gitroot\n";
		print FILE "git_path = $module\n" unless $is_module;
		if( $tag ) {
	    	print FILE "git_tree = $tag\n";
		} else {
	    	# the "master" branch is essentially like the CVS
	    	# TRUNK. it should always exist, but it is possible for
	    	# someone to rename it. hopefully we won't do that in the
	    	# NMI copy of the repository
	    	print FILE "git_tree = origin/master\n";
		}
    }

    close FILE;
}


sub makeFetchFileSCP
{
    my ( $file, $path, $is_recursive ) = @_;
    open( FILE, ">$file" ) || die "Can't open $file for writing: $!\n";
    print FILE "method = scp\n";
    print FILE "scp_file = $path\n";
    if( defined $is_recursive ) {
        print FILE "recursive = true\n";
    }
    close FILE;
}


sub parseTag
{
    my $tag = shift;
    my $desc;
    my @vers;

    $tag =~ s/BUILD-//;
    my $desc = $tag;
    $tag =~ s/-branch-.*//;
    $tag =~ s/V//;
    if( $tag =~ /(\d+)(\D*)_(\d+)(\D*)_?(\d+)?(\D*)/ ) {
        $vers[0] = $1;
        $vers[1] = $3;
        if( $5 ) {
            $vers[2] = $5;
        } else {
            $vers[2] = "x";
        }
    }
    return ( $desc, @vers );
}


sub submitRun()
{
    my( $cmdfile, $tag ) = @_;
    my @submit_output;
    print "Submitting condor $runtype from ";
    if( defined $tag ) {
        print "tag: $tag";
    } elsif( defined $src_workspace ) {
        print "workspace: $src_workspace";
    } else {
        print "unknown source";
    }
    print "\n";

	open( SUBMIT, "$nmi_submit $cmdfile|" ) || 
		die "Can't open $nmi_submit as a pipe: $!\n";
	while( <SUBMIT> ) {
		print;
	}
	close( SUBMIT );
	my $status = $? / 256;

    $opt_debug && print "----- Start of nmi_submit input -----\n";


    if ($opt_debug) { 
        print "----- End of nmi_submit input -----\n";
    }
    if ($status == 0) {
        print "nmi_submit successful\n";
    } else {
        die "ERROR: nmi_submit failed!\n";
    }
}


sub run
{
    my ($cmd, $fatal) = @_;
    my $ret;
    my $output = "";

    # if not specified, the command is fatal
    if (!defined($fatal) or ($fatal != 0 and $fatal != 1)) {
        $fatal = 1;
    }

    print "#  $cmd\n";

    # run the command
    system("($cmd)  </dev/null 2>&1");
    $ret = $? / 256;

    # should we die here?
    if ($fatal && $ret != 0) {
        print "\n";
        print "FAILED COMMAND: $cmd\n";
        print "RETURN VALUE:  $ret\n";
        print "\n";
        myExit( 1 );
    }

    # return the commands return value
    return $ret;
}


sub myExit
{
    my $exit_code = shift;
    if ($opt_debug) {
        print "Temporary directory $workspace not removed\n";
    } elsif (defined $workspace && -d $workspace) {
        chdir($init_cwd);
        run("rm -rf $workspace", 0);
    }
    exit $exit_code;
}


sub getPlatformsFromRunid() {
    my( $runid ) = @_;

    # this is the only place we need to talk to the DB, so we'll only
    # incur the cost and complication of initializing our DB settings
    # (i.e. finding and parsing the NMI config file, etc) if we really
    # need to...
    initializeDB();

    my @platforms = ();
    my $platform;

    my $cmd_str = qq/SELECT DISTINCT platform from $TASK_TABLE WHERE
        runid='$runid'/;
    print "$cmd_str\n";

    my $foundDockerPlatforms = 0;
    my $handle = $db->prepare("$cmd_str");
    $handle->execute();
    while ( my $row = $handle->fetchrow_hashref() ) {
        $platform = $row->{'platform'};
        # remove the problem cause by no API and plaform becomming nmi:platform
        $_ = $platform;
        s/nmi://;
        $platform = $_;

        if( $platform =~ /^nmi-zdev:/ ) { $foundDockerPlatforms = 1; next; }

        push(@platforms, $platform) unless $platform =~ /local/;
    }
    $handle->finish();
    $db->disconnect;

    if( $foundDockerPlatforms ) {
        my $output = `${nmi_bin}/nmi_rundir $runid`;
        my( $r, $s ) = split(/ /, $output);
        my( $h, $dir ) = split(":", $s);
        chomp($dir);
        my $list = "${dir}/docker_platforms";

        open( LIST, "<", $list ) or die( "Problem opening '${list}', aborting.\n" );
        while(my $p = <LIST>) {
            chomp($p);
            push(@platforms, "docker://${p}");
        }
        close( LIST );
    }

    return @platforms;
}


######################################################################
# submit_info parsing routines
######################################################################

#----------------------------------------------------------------------
# The main parsing method.  This just parses the input to find the
# data blocks.  For each block, it calls parseBlock() to do the real
# work of parsing, managing, and saving data for the block.  Once it
# reads the whole file, this method also calls cleanPlatformData() on
# each platform to clean and make-sane all the platform-specific data
# (removing duplicate prereqs, sorting, fixing the formatting, etc).
#----------------------------------------------------------------------
sub parseSubmitInfo
{
    my $filename = shift;
    open(FH, "<$filename" ) || die "Can't open '$filename': $!\n";

    my ( $line, $val, $platform );

    while( $line = getLine(*FH) ) {
        if( $line =~ /^\[/ ) {
            # this will consume upto and including the next ']'
            parseBlock(*FH);
            next;
        }
        if( $line =~ /^\]/ ) {
            parseError( "saw ']' while not in a block!" );
        }
        parseError( "Data ('$line') defined outside of a block!" );
    }

    # Now that we're done with the whole file, do some final cleaning
    # of the data
    foreach $platform (sort keys %platforms ) {
        cleanPlatformData($platform);
    }
    delete $platforms{"global"};
}


#----------------------------------------------------------------------
# Helper method to clean up the platform-specific data.  Makes sure
# none of the prereqs are duplicate and ensure we've got a sorted,
# well-formatted list without any funky whitespace, etc.  This method
# just reads out the data for the given platform, stuffs it all into a
# hash table (to guarantee uniqueness), and then uses sort and join to
# format it nicely.  It also handles the special case of the "global"
# platform, and takes the global data out of our hash tables and
# stuffs it into the global scalar values, instead.
#----------------------------------------------------------------------
sub cleanPlatformData
{
    my $platform = shift;

    my( $item, %prereqs, %t_prereqs, %x_test_platforms );

    foreach $item ( split(/[, \t]+/, $platform_prereqs{$platform}) ) {
        $prereqs{$item}=1;
    }
    foreach $item ( split(/[, \t]+/, $platform_test_prereqs{$platform}) ) {
        $t_prereqs{$item}=1;
    }
    foreach $item ( split(/[, \t]+/, $platform_x_test_platforms{$platform}) ) {
        $x_test_platforms{$item}=1;
    }
    if( $platform =~ /global/i ) {
        if( $platform_prereqs{$platform} ) {
            $global_prereqs = join(', ', sort(keys(%prereqs)));
            delete( $platform_prereqs{$platform} );
        }
        if( $platform_test_prereqs{$platform} ) {
            $global_test_prereqs = join(', ', sort(keys(%t_prereqs)));
            delete( $platform_test_prereqs{$platform} );
        }
        if( $platform_testargs{$platform} ) {
            $global_testargs = $platform_testargs{$platform};
            delete( $platform_testargs{$platform} );
        }
    } else {
        if( $platform_prereqs{$platform} ) {
            $platform_prereqs{$platform} = join(', ', sort(keys(%prereqs)));
        }
        if( $platform_test_prereqs{$platform} ) {
            $platform_test_prereqs{$platform} =
                join(', ', sort(keys(%t_prereqs)));
        }
        if( $platform_x_test_platforms{$platform} ) {
            $platform_x_test_platforms{$platform} =
                join(', ', sort(keys(%x_test_platforms)));
        }
    }
}


#----------------------------------------------------------------------
# Method to parse a given platform block.  This is the meat of the
# entire parsing logic.  Assumes it's already read the '[' to start
# the block.  Once it reads a complete block (upto a ']'), it updates
# the global data structures with all the info for the platforms it
# found in that block.
#----------------------------------------------------------------------
sub parseBlock
{
    my $fh = shift;

    my ( $line, $val, $item, $platform );

    # all of the "block_*" local variables hold data as defined in
    # this block...  otherwise, the name should be obvious.

    # these are hashes to guarantee uniqueness
    my %block_platforms;
    my %block_prereqs;
    my %block_test_prereqs;
    my %block_testargs;
    my %block_x_test_platforms;

    # these are hashes to map platform -> prereqs
    my %block_platform_prereqs;
    my %block_platform_test_prereqs;
    my %block_testplatform_prereqs;
    my %block_testplatform_test_prereqs;

	# test platform define local hashes
    my %block_testplatforms;
    my %block_testprereqs;
    my %block_testtest_prereqs;

    # flag to help check for sanity
    my $in_block = 1;

    while( $line = getLine(*$fh) ) {
        if( $line =~ /^\[/ ) {
            parseError( "saw '[' while already in a block" );
        }
        elsif( $line =~ /^\]/ ) {
            $in_block = 0;
            last;
        }
        elsif( $line =~ /^platforms\s*=\s*(.*)/i ) {
            foreach $item ( split(/[, \t]+/, $1)) {
                $block_platforms{$item}=1;
            }
        }
        elsif( $line =~ /^testplatforms\s*=\s*(.*)/i ) {
            foreach $item ( split(/[, \t]+/, $1)) {
                $block_testplatforms{$item}=1;
            }
        }
        elsif( $line =~ /^prereqs\s*=\s*(.*)/i ) {
            foreach $item ( split(/[, \t]+/, $1)) {
                $block_prereqs{$item}=1;
            }
        }
        elsif( $line =~ /^testprereqs\s*=\s*(.*)/i ) {
            foreach $item ( split(/[, \t]+/, $1)) {
				$block_testprereqs{$item} = 1;
            }
        }
        elsif( $line =~ /^prereqs_(\S+)\s*=\s*(.*)/i ) {
            $platform = $1;
            $val = $2;
            if( ! $block_platforms{$platform} ) {
                parseError( "prereqs_$platform defined but "
                            . "$platform not declared in this block" );
            }
            $block_platform_prereqs{$platform} = $val;
        }
        elsif( $line =~ /^testprereqs_(\S+)\s*=\s*(.*)/i ) {
            $platform = $1;
            $val = $2;
            if( ! $block_testplatforms{$platform} ) {
                parseError( "testprereqs_$platform defined but "
                            . "$platform not declared in this block" );
            }
            $block_testplatform_prereqs{$platform} = $val;
        }
        elsif( $line =~ /^test_append_prereqs\s*=\s*(.*)/i ) {
            foreach $item ( split(/[, \t]+/, $1)) {
                $block_test_prereqs{$item} = 1;
            }
        }
        elsif( $line =~ /^testonly_append_prereqs\s*=\s*(.*)/i ) {
            foreach $item ( split(/[, \t]+/, $1)) {
                $block_testtest_prereqs{$item} = 1;
            }
        }
        elsif( $line =~ /^test_append_prereqs_(\S+)\s*=\s*(.*)/i ) {
            $platform = $1;
            $val = $2;
            if( ! $block_platforms{$platform} ) {
                parseError( "test_append_prereqs_$platform defined but "
                            . "$platform not declared in this block" );
            }
            $block_platform_test_prereqs{$platform} = $val;
        }
        elsif( $line =~ /^testonly_append_prereqs_(\S+)\s*=\s*(.*)/i ) {
            $platform = $1;
            $val = $2;
            if( ! $block_testplatforms{$platform} ) {
                parseError( "test_append_prereqs_$platform defined but "
                            . "$platform not declared in this block" );
            }
            $block_testplatform_test_prereqs{$platform} = $val;
        }
        elsif( $line =~ /^testargs(_(\S+))?\s*=\s*(.*)/i ) {
            $platform = $2;
            $val = $3;
            if( ! $platform ) {
                $platform = "block";
            }
            if( $block_testargs{$platform} ) {
                $block_testargs{$platform} .= " ";
            }
            $block_testargs{$platform} .= $val;
        }
        elsif( $line =~ /^x_test_platforms(_(\S+))?\s*=\s*(.*)/i ) {
            $platform = $2;
            $val = $3;
            if( ! $platform ) {
                $platform = "block";
            } elsif( ! $block_platforms{$platform} ) {
                parseError( "x_test_platforms_$platform defined but "
                            . "$platform not declared in this block" );
            }
            if( $block_x_test_platforms{$platform} ) {
                $block_x_test_platforms{$platform} .= " ";
            }
            $block_x_test_platforms{$platform} .= $val;
        }
        else {
            parseError( "unrecognized: '$line'" );
        }
    }

    # now that we broke out of the loop, we're done with this block
    # and we can process what we saw...

    # first, some sanity checks:
    $in_block && parseError( "hit EOM while still in a block" );
    if( (! %block_platforms) && (! %block_testplatforms) ) {
        parseError( "Block defined without any platforms" );
    }

    # append all the (new) platforms we saw to our global list:
    foreach $platform ( keys(%block_platforms) ) {
        $platforms{$platform} = 1;
    }
	 # same for test platforms
    foreach $platform ( keys(%block_testplatforms) ) {
        $testplatforms{$platform} = 1;
    }

    # for each platform we saw, append any platform-specific stuff to
    # the global hash tables
    foreach $platform ( keys(%block_platforms) ) {

        # append all shared prereqs in this block to each platform
        if( %block_prereqs ) { 
            $platform_prereqs{$platform} .= 
                join( ', ', sort(keys(%block_prereqs)) );
            $platform_test_prereqs{$platform} .= 
                join( ', ', sort(keys(%block_prereqs)) );
        }

        # append all shared test prereqs in this block to each platform
        if( %block_test_prereqs ) { 
            if( $platform_test_prereqs{$platform} ) {
                $platform_test_prereqs{$platform} .= ", ";
            }
            $platform_test_prereqs{$platform} .= 
                join( ', ', sort(keys(%block_test_prereqs)) );
        }

        # if this platform also has platform-specific prereqs, append
        # those, too...
        if( $block_platform_prereqs{$platform} ) {
            if( $platform_prereqs{$platform} ) {
                $platform_prereqs{$platform} .= ", ";
            }
            $platform_prereqs{$platform} .= $block_platform_prereqs{$platform};
            if( $platform_test_prereqs{$platform} ) {
                $platform_test_prereqs{$platform} .= ", ";
            }
            $platform_test_prereqs{$platform} .= $block_platform_prereqs{$platform};
        }

        # if this platform also has platform-specific test prereqs,
        # append those, too...
        if( $block_platform_test_prereqs{$platform} ) {
            if( $platform_test_prereqs{$platform} ) {
                $platform_test_prereqs{$platform} .= ", ";
            }
            $platform_test_prereqs{$platform} .=
                $block_platform_test_prereqs{$platform};
        }

        # if this platform defined testargs, over-write those.
        # otherwise if there was a global testarg defined for this
        # block, use that...
        if( $block_testargs{$platform} ) {
            $platform_testargs{$platform} = $block_testargs{$platform};
        } elsif( $block_testargs{"block"} ) {
            $platform_testargs{$platform} = $block_testargs{"block"};
        }

        # if this platform defined x_test_platforms, append those.
        # otherwise, if there was a global x_test_platforms defined
        # for this block, use that...
        my $append = '';
        if( $block_x_test_platforms{$platform} ) {
            $append = $block_x_test_platforms{$platform};
        } elsif( $block_x_test_platforms{"block"} ) {
            $append = $block_x_test_platforms{'block'};
        }
        if( $append ) {
            if( $platform_x_test_platforms{$platform} ) {
                $platform_x_test_platforms{$platform} .= ", ";
            }
            $platform_x_test_platforms{$platform} .= $append;
        }
    }
    foreach $platform ( keys(%block_testplatforms) ) {

        # append all shared prereqs in this block to each platform
        if( %block_testprereqs ) { 
            $testplatform_prereqs{$platform} .= 
                join( ', ', sort(keys(%block_testprereqs)) );
            $testplatform_test_prereqs{$platform} .= 
                join( ', ', sort(keys(%block_testprereqs)) );
        }

        # append all shared test prereqs in this block to each platform
        if( %block_testtest_prereqs ) { 
            if( $testplatform_test_prereqs{$platform} ) {
                $testplatform_test_prereqs{$platform} .= ", ";
            }
            $testplatform_test_prereqs{$platform} .= 
                join( ', ', sort(keys(%block_testtest_prereqs)) );
        }

        # if this platform also has platform-specific prereqs, append
        # those, too...
        if( $block_testplatform_prereqs{$platform} ) {
            if( $testplatform_prereqs{$platform} ) {
                $testplatform_prereqs{$platform} .= ", ";
            }
            $testplatform_prereqs{$platform} .= $block_testplatform_prereqs{$platform};
            if( $testplatform_test_prereqs{$platform} ) {
                $testplatform_test_prereqs{$platform} .= ", ";
            }
            $testplatform_test_prereqs{$platform} .= $block_testplatform_prereqs{$platform};
        }

        # if this platform also has platform-specific test prereqs,
        # append those, too...
        if( $block_testplatform_test_prereqs{$platform} ) {
            if( $testplatform_test_prereqs{$platform} ) {
                $testplatform_test_prereqs{$platform} .= ", ";
            }
            $testplatform_test_prereqs{$platform} .=
                $block_testplatform_test_prereqs{$platform};
        }

    }
}


#----------------------------------------------------------------------
# Helper method that reads another line out of the input file.  Keeps
# track of the original line number in the file we're at, and handles
# comments, continuation, and ignoring/trimming whitespace.
#----------------------------------------------------------------------
sub getLine
{
    my $fh = shift;
    my $line = "";
    my $tmp;

    while( $tmp = <$fh> ) {
        $orig_lineno++;
        chomp( $tmp );
        if( $tmp =~ /(.*)\\$/ ) {
            # continuation, append everything except the '\' to $line
            $line .= $1;
            next;
        } else {
            # note: we want to append to line first, then decide if
            # we're going to throw it out so that we handle comments
            # that have a line continuation...
            $line .= $tmp;
            if( $line =~ /^\s*#.*/ ) {
                # comment
                $line = "";
                next;
            }
            if( $line =~ /^\s*$/ ) {
                # just whitespace
                $line = "";
                next;
            }
            # we're about to return the line.  first, clean it up:
            # - ignore leading whitespace
            # - ignore everything after a '#' (if there is one)
            $line =~ /^\s*([^#]*)\#?(.*)$/;
            return $1;
        }
    }
    # if we're here, <> returned undefined, so we're done.
    return;
}


#----------------------------------------------------------------------
# Helper method to die with a message that includes the line number
#----------------------------------------------------------------------
sub parseError
{
    my $err = shift;
    die "ERROR (line $orig_lineno): $err!\n";
}
