Fix the MPI driver to not conflict with libevent

It turns out you can't use MPI and libevent in the same program,
as MPI uses libevent itself.  :-(

So we change the MPI driver to fork and exec the standalone programs,
which is fine.

Makefile

@@ -26,7 +26,7 @@ CXX ?= g++
 WORDS = 24
 
 CXXFLAGS=-g -Wall -O0
-CPPFLAGS=-DWORDS=$(WORDS) -I$(LIBEVENT)/include -I$(GMP)/include
+CPPFLAGS=-DWORDS=$(WORDS) -I$(LIBEVENT)/include -I$(GMP)/include -UVERBOSE
 NVCCOPTS=-g -arch sm_20 --ptxas-options=-v -O2
 NVCC=nvcc $(NVCCOPTS)
 
@@ -64,8 +64,8 @@ dpnode: dpnode.o evutils.o dpnode_main.o
 worker: worker.o evutils.o cudadl.o worker_main.o
 	g++ -g -Wall $^ -o $@ -L$(LIBEVENT)/lib -Wl,-rpath=$(LIBEVENT)/lib -levent -levent_pthreads -lntl -L$(GMP) -lgmp -lpthread -L$(CUDA)/lib64 -lcudart
 
-mpi: mpi.o controller.o worker.o dpnode.o evutils.o cudadl.o
-	mpiCC -g -Wall $^ -o $@ -L$(LIBEVENT)/lib $(LIBEVENT)/lib/libevent.a $(LIBEVENT)/lib/libevent_pthreads.a -lntl -L$(GMP) -lgmp -lpthread -L$(CUDA)/lib64 -lcudart
+mpi: mpi.o
+	mpiCC -g -Wall $^ -o $@ -L/work/iang/sw/lib -lntl -L$(GMP) -lgmp -lpthread
 
 mpi.o: mpi.cc
 	mpiCC $(CXXFLAGS) $(CPPFLAGS) $^ -c -o $@

controller.cc

@@ -26,7 +26,7 @@ extern "C" {
 
 NTL_CLIENT
 
-#undef VERBOSE
+// #undef VERBOSE
 
 struct SubproblemProgress;
 

controller_main.cc

@@ -13,7 +13,7 @@ void desired_resources(const ZZ &order, unsigned short &desired_dpnodes,
     unsigned int &max_workers, unsigned int &dpfreq)
 {
     // How many DPnodes should we use for a problem of this size?
-    desired_dpnodes = 2;
+    desired_dpnodes = 1;
     // How many workers would we like to use?
     ZZ sorder = SqrRoot(order >> 46);
     if (NumBits(sorder) > 30) {
@@ -40,6 +40,15 @@ void desired_resources(const ZZ &order, unsigned short &desired_dpnodes,
 static void boundcb(const char *boundaddr, unsigned short boundport)
 {
     cout << "Listening on " << boundaddr << ":" << boundport << "\n";
+    const char *fdenv = getenv("CONTROLLER_BOUNDCB_FD");
+    if (fdenv) {
+	int fd = atoi(fdenv);
+	if (fd > 2) {
+	    write(fd, &boundport, 2);
+	    write(fd, boundaddr, strlen(boundaddr));
+	    close(fd);
+	}
+    }
 }
 
 int main(int argc, char **argv)

dpnode.cc

@@ -22,7 +22,7 @@ extern "C" {
 #include "subproblem.h"
 #include "dpnode.h"
 
-#undef VERBOSE
+// #undef VERBOSE
 
 typedef map<std::string, pair<ZZ,ZZ> > DTable;
 

mpi.cc

@@ -1,4 +1,9 @@
 #include <stdio.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/resource.h>
+#include <sys/wait.h>
+#include <errno.h>
 #include <mpi.h>
 
 #include <NTL/ZZ.h>
@@ -13,6 +18,23 @@ static int mpi_size;
 
 static int controllerfds[2];
 
+// Close fds 3 and up, except for the one given (pass -1 to close them
+// all)
+static void close_highfds_except(int exceptfd)
+{
+    // Find the max fd number
+    struct rlimit limit;
+    getrlimit(RLIMIT_NOFILE, &limit);
+    printf("Closing up to %lu\n", (unsigned long)limit.rlim_cur);
+    for (int fd = 3; fd < limit.rlim_cur; ++fd) {
+	if (fd != exceptfd) {
+	    // There's no ill effect from closing a non-open fd, so just
+	    // do it
+	    close(fd);
+	}
+    }
+}
+
 static void boundcb(const char *boundaddr, unsigned short boundport)
 {
     // Write the port and addr to the pipe
@@ -25,7 +47,7 @@ void desired_resources(const ZZ &order, unsigned short &desired_dpnodes,
     unsigned int &max_workers, unsigned int &dpfreq)
 {
     // How many DPnodes should we use for a problem of this size?
-    desired_dpnodes = 2;
+    desired_dpnodes = 1;
     // How many workers would we like to use?
     ZZ sorder = SqrRoot(order >> 46);
     if (NumBits(sorder) > 30) {
@@ -49,6 +71,16 @@ void desired_resources(const ZZ &order, unsigned short &desired_dpnodes,
     }
 }
 
+static pid_t fork_and_remember(vector<pid_t> &children)
+{
+    pid_t pid = fork();
+    if (pid > 0) {
+	children.push_back(pid);
+    }
+
+    return pid;
+}
+
 int main(int argc, char **argv)
 {
     // Init MPI
@@ -57,7 +89,9 @@ int main(int argc, char **argv)
     char hostname[257];
     gethostname(hostname, 256);
 
-    int rank;
+    vector<pid_t> children;
+
+    int rank = 0;
 
     int ret = 0;
 
@@ -67,8 +101,17 @@ int main(int argc, char **argv)
     if (rank == 0) {
 	// Start the controller
 	pipe(controllerfds);
-	if (fork() == 0) {
-	    // Child; close the write half of the pipe
+	char fdenv[30];
+	sprintf(fdenv, "%d", controllerfds[1]);
+	setenv("CONTROLLER_BOUNDCB_FD", fdenv, 1);
+	if (fork_and_remember(children) == 0) {
+	    // Child; close the read half of the pipe and all other fds
+	    close_highfds_except(controllerfds[1]);
+
+	    execv("./controller", argv);
+	    return 1;
+	} else {
+	    // Parent; close the write half of the pipe
 	    close(controllerfds[1]);
 	    unsigned short boundport;
 	    char boundaddr[257];
@@ -81,33 +124,42 @@ int main(int argc, char **argv)
 	    close(controllerfds[0]);
 	    boundaddr[res] = '\0';
 
-	    std::cerr << "Child bound to " << boundaddr << ":" << boundport << "\n";
+	    std::cerr << "Child reports controller bound to " << boundaddr << ":" << boundport << "\n";
+
+	    char portstr[10];
+	    sprintf(portstr, "%hu", boundport);
 
 	    // The child will spawn two of its own children to be the
 	    // workers
-	    if (fork() == 0) {
-		return worker_main(boundaddr, boundport);
-	    } else if (fork() == 0) {
-		return worker_main(boundaddr, boundport);
-	    }
-
-	    // And now become the dpnode
-	    return dpnode_main(boundaddr, boundport);
-	} else {
-	    // Parent; close the read half of the pipe
-	    close(controllerfds[0]);
-	    unsigned short bindport;
-	    Worklist worklist;
-
-	    if (controller_parse_args(argc, argv, bindport, worklist)) {
-		std::cerr << "Usage: " << argv[0] << " [-p listenport] N1 iter1 N2 iter2 ...\n";
+	    if (fork_and_remember(children) == 0) {
+		close_highfds_except(-1);
+		execl("./worker", "./worker", boundaddr, portstr, "0", NULL);
+		return 1;
+	    } else if (fork_and_remember(children) == 0) {
+		close_highfds_except(-1);
+		execl("./worker", "./worker", boundaddr, portstr, "0", NULL);
+		return 1;
+	    } else if (fork_and_remember(children) == 0) {
+		// And a dpnode
+		close_highfds_except(-1);
+		execl("./dpnode", "./dpnode", boundaddr, portstr, NULL);
+		return 1;
+	    } else if (fork_and_remember(children) == 0) {
+		// And a bonus dpnode
+		close_highfds_except(-1);
+		execl("./dpnode", "./dpnode", boundaddr, portstr, NULL);
 		return 1;
 	    }
-
-	    ret = controller_main(worklist, bindport, boundcb);
 	}
     }
 
+    // Now wait for all the children
+    vector<pid_t>::iterator pidit;
+    for (pidit = children.begin(); pidit != children.end(); ++pidit) {
+	int status;
+	waitpid(*pidit, &status, 0);
+    }
+
     MPI_Finalize();
 
     return ret;