MPI version redirects some output to files.

The MPI version now redirects the stdout and stderr of each worker and dpnode to corresponding files. The controller output is not redirected and will be handled by MPI.

mpi.cc

@@ -4,6 +4,7 @@
 #include <sys/types.h>
 #include <sys/resource.h>
 #include <sys/wait.h>
+#include <fcntl.h>
 #include <errno.h>
 #include <mpi.h>
 
@@ -71,6 +72,17 @@ void desired_resources(const ZZ &order, unsigned short &desired_dpnodes,
     }
 }
 
+void redirect_output(const char* dir, const char* filename_base, int rank_index, int process_index, const char* filename_ext){
+    // redirect stdout and stderr to a file
+    char buf[512];
+    snprintf(buf, sizeof(buf), "%s/%s_%d_%d.%s", dir, filename_base, rank_index, process_index, filename_ext);
+
+    int fd = open(buf, O_RDWR|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR);
+    dup2(fd, 1);
+    dup2(fd, 2);
+    close(fd);
+}
+
 static pid_t fork_and_remember(vector<pid_t> &children)
 {
     pid_t pid = fork();
@@ -86,8 +98,11 @@ int main(int argc, char **argv)
     // Init MPI
     MPI_Init(&argc, &argv);
 
-    if (argc < 3) {
-	printf("You must pass at least two arguments (the number of workers and number of dpnodes to start per node/rank).\n");
+    if (argc < 4) {
+	printf("You must pass at least three arguments:\n");
+	printf(" 1) number of workers to start per node/rank\n");
+	printf(" 2) number of dpnodes to start per node/rank\n");
+	printf(" 3) directory to store output files (stdout+stderr)\n");
 	printf("The remaining arguments will be passed to the controller.\n");
 
 	MPI_Finalize();
@@ -120,10 +135,9 @@ int main(int argc, char **argv)
 	if (fork_and_remember(children) == 0) {
 	    // Child; close the read half of the pipe and all other fds
 	    close_highfds_except(controllerfds[1]);
-
 	    char name[] = "controller";
-	    argv[2] = name;
-	    execv("./controller", argv+2);
+	    argv[3] = name;
+	    execv("./controller", argv+3);
 	    return 1;
 	} else {
 	    // Parent; close the write half of the pipe
@@ -150,9 +164,12 @@ int main(int argc, char **argv)
     const char *boundaddr = boundportaddr + 2;
 
     // The child will spawn its own children to be the workers
-    for (int i=0; i<num_workers; i++) {
+    for (unsigned int i=0; i<num_workers; i++) {
 	if (fork_and_remember(children) == 0) {
 	    close_highfds_except(-1);
+
+	    redirect_output(argv[3], "worker", rank, i, "out");
+
 	    char gpu_index[10];
 	    snprintf(gpu_index, 10, "%d", i);
 	    execl("./worker", "./worker", boundaddr, portstr, gpu_index, NULL);
@@ -161,9 +178,12 @@ int main(int argc, char **argv)
     }
 
     // And dpnodes
-    for (int i=0; i<num_dpnodes; i++) {
+    for (unsigned int i=0; i<num_dpnodes; i++) {
 	if (fork_and_remember(children) == 0) {
 	    close_highfds_except(-1);
+
+	    redirect_output(argv[3], "dpnode", rank, i, "out");
+
 	    execl("./dpnode", "./dpnode", boundaddr, portstr, NULL);
 	    return 1;
 	}