Improved the Slurm scripts.

The 'slurm-jobscript' now uses the 'slurm-wrapper' in the given cudadl directory, and the 'slurm-wrapper' now has the dpnodes and workers write to a temporary file, and copies that file to the NFS once completed.

slurm-jobscript.sh

@@ -40,4 +40,4 @@ TEMP_NFS_FILE=$(mktemp -u "${TEMP_NFS_DIR}/loc.XXXXXXXXXX")
 #
 echo "Starting with $NUM_DPNODES dpnodes ($MEM_PER_DPNODE GB memory) and $NUM_WORKERS workers."
 #
-srun ./slurm-wrapper.sh $SAVE_OUTPUT_DIR $TEMP_NFS_FILE $CUDADL_PATH -w $NUM_WORKERS -n $NUM_DPNODES -m $MEM_PER_DPNODE $PROBLEM_FILE 1
+srun ${CUDADL_PATH}/slurm-wrapper.sh $SAVE_OUTPUT_DIR $TEMP_NFS_FILE $CUDADL_PATH -w $NUM_WORKERS -n $NUM_DPNODES -m $MEM_PER_DPNODE $PROBLEM_FILE 1

slurm-wrapper.sh

@@ -71,7 +71,9 @@ if [ "$SLURM_LOCALID" == "0" ] ; then
 		read IP PORT < $FILE_WITH_LOC
 		#
 		OUTPUT_FILE="${OUTPUT_FILE_DIR}/dpnode_${SLURM_NODEID}_${SLURM_LOCALID}.out"
-		${CUDADL_PATH}/dpnode $IP $PORT > "${OUTPUT_FILE}" 2>&1
+		TMP_OUTPUT_FILE="${SLURM_TMPDIR}/dpnode_${SLURM_NODEID}_${SLURM_LOCALID}.out"
+		${CUDADL_PATH}/dpnode $IP $PORT > "${TMP_OUTPUT_FILE}" 2>&1
+		mv "$TMP_OUTPUT_FILE" "$OUTPUT_FILE"
 	fi
 	#
 	if [ ! -z "$CONTROLLER_PID" ] ; then
@@ -97,6 +99,8 @@ else
 		WORKER_ID=$((SLURM_LOCALID-1))
 		#
 		OUTPUT_FILE="${OUTPUT_FILE_DIR}/worker_${SLURM_NODEID}_${WORKER_ID}.out"
-		${CUDADL_PATH}/worker $IP $PORT $WORKER_ID > "${OUTPUT_FILE}" 2>&1
+		TMP_OUTPUT_FILE="${SLURM_TMPDIR}/worker_${SLURM_NODEID}_${WORKER_ID}.out"
+		${CUDADL_PATH}/worker $IP $PORT $WORKER_ID > "${TMP_OUTPUT_FILE}" 2>&1
+		mv "$TMP_OUTPUT_FILE" "$OUTPUT_FILE"
 	fi
 fi