#!/bin/bash

#
# Arguments.
#
SYSTEM=$1; shift
STARTD_NOCLAIM_SHUTDOWN=$1; shift

JOB_NAME=$1
COLLECTOR_HOST=$2
LIFETIME=$3
OWNERS=$4
REQUEST_ID=$5
STARTD_NAME=$6
CPUS=$7
MEM_MB=$8
SCHEDD_NAME=$9


#
# Constants.
#
BIRTH=`date +%s`
CLEAN_UP_TIME=300
BINARIES_FILE=condor-*.tar.gz
CONFIGURATION_FILE=hpc-config.tar.gz


#
# Unpack the binaries.
#

TAR_LOGGING=`tar -z -x -f ${BINARIES_FILE} 2>&1`
if [[ $? != 0 ]]; then
    echo "Failed to unpack binaries from '${BINARIES_FILE}', aborting."
    echo ${TAR_LOGGING}
    exit 3
fi


#
# Make the personal condor.
#

rm condor-*.tar.gz
cd condor-*

MPC_LOGGING=`./bin/make-personal-from-tarball 2>&1`
if [[ $? != 0 ]]; then
    echo "Failed to make personal condor, aborting."
    echo ${MPC_LOGGING}
    exit 4
fi


#
# Configure the personal condor.
#
YOUTH=$((`date +%s` - ${BIRTH}))
REMAINING_LIFETIME=$(((${LIFETIME} - ${YOUTH}) - ${CLEAN_UP_TIME}))

rm local/config.d/00-personal-condor
echo "
use role:execute
use security:recommended_v9_0
use feature:PartitionableSLot

COLLECTOR_HOST = <${COLLECTOR_HOST}>

SHARED_PORT_PORT = 0

STARTD_NAME = ${STARTD_NAME}

# Don't start more jobs than we requested room for.  This isn't enforcing.
NUM_CPUS = ${CPUS}
MEMORY = ${MEM_MB}

# FIXME: use same-AuthenticatedIdentity once that becomes available, instead.
# Allows condor_off (et alia) to work from the submit node.
ALLOW_ADMINISTRATOR = \$(ALLOW_ADMINISTRATOR) condor_pool@*
SEC_DEFAULT_AUTHENTICATION_METHODS = FS IDTOKENS PASSWORD

# Eliminate a bogus, repeated warning in the logs.  This is a bug;
# it should be the default.
SEC_PASSWORD_DIRECTORY = \$(LOCAL_DIR)/passwords.d

# This is a bug; it should be the default.
SEC_TOKEN_SYSTEM_DIRECTORY = \$(LOCAL_DIR)/tokens.d

# Having to set it twice is also a bug.
SEC_TOKEN_DIRECTORY = \$(LOCAL_DIR)/tokens.d

# Don't run benchmarks.
RUNBENCHMARKS = FALSE

# We definitely need CCB.
CCB_ADDRESS = \$(COLLECTOR_HOST)

# Commit suicide after being idle for long enough.
STARTD_NOCLAIM_SHUTDOWN = ${STARTD_NOCLAIM_SHUTDOWN}

# It's a magic constant.  (If the initial update to the collector
# doesn't work right for some reason, at least try to update it a
# second time getting bored and commiting suicide.)
UPDATE_INTERVAL = 137

# Don't run for any longer than was requested.
MASTER.DAEMON_SHUTDOWN_FAST = (CurrentTime - DaemonStartTime) > ${REMAINING_LIFETIME}

# Only start jobs from the specified owner(s).
START = \$(START) && stringListMember( Owner, \"${OWNERS}\" )

# Advertise the standard annex attributes (in the master ad for condor_off).
IsAnnex = TRUE
AnnexName = \"${JOB_NAME}\"
hpc_annex_request_id = \"${REQUEST_ID}\"
STARTD_ATTRS = \$(STARTD_ATTRS) AnnexName IsAnnex hpc_annex_request_id
MASTER_ATTRS = \$(MASTER_ATTRS) AnnexName IsAnnex hpc_annex_request_id

# This pilot runs in a container; the image is expected to have singularity
# installed and configured correctly.
SINGULARITY                 = /usr/bin/singularity

#
# Connect directly to the schedd.
#
STARTD_DIRECT_ATTACH_SCHEDD_NAME = ${SCHEDD_NAME}
STARTD_DIRECT_ATTACH_SCHEDD_POOL = ${COLLECTOR}

# Update slots often enough that we can do more than one job. ;)
# The direct attach interval is the minimum interval; this interval
# is checked every UPDATE_INTERVAL.
STARTD_DIRECT_ATTACH_INTERVAL = 5
UPDATE_INTERVAL = 5

" > local/config.d/00-basic-pilot

mkdir local/tokens.d
mv ../token_file local/tokens.d

mkdir local/passwords.d
mv ../password_file local/passwords.d/POOL


#
# Unpack the configuration on top.
#
TAR_LOGGING=`tar -z -x -f ../${CONFIGURATION_FILE} 2>&1`
if [[ $? != 0 ]]; then
    echo "Failed to unpack binaries from '${CONFIGURATION_FILE}', aborting."
    echo ${TAR_LOGGING}
    exit 5
fi


#
# Start the personal condor.
#
. ./condor.sh
# Using exec means that the master catches the SIGTERM on condor_rm and
# give the daemons a chance to shutdown cleanly (most importantly,
# unregister their ads from the collector).
exec condor_master -f
