The worker's 'cuda_dev_id' argument selects a GPU.

The workers now select a GPU device based on the 'cuda_dev_id' command line argument. Otherwise without using 'cudaSetDevice' to select a GPU, all workers will default to sharing the same GPU (device 0) on RIPPLE, assuming each worker process doesn't have an independent 'CUDA_VISIBLE_DEVICES' value set.

Makefile

@@ -73,6 +73,9 @@ dpnode: dpnode.o evutils.o dpnode_main.o
 worker: worker.o evutils.o cudadl.o worker_main.o
 	g++ -g -Wall $^ -o $@ -L$(LIBEVENT)/lib -Wl,-rpath=$(LIBEVENT)/lib -levent -levent_pthreads -lntl -L$(GMP) -lgmp -lpthread -L$(CUDA)/lib64 -lcudart
 
+worker.o: worker.cc
+	g++ $(CXXFLAGS) $(CPPFLAGS) -I$(CUDA)/include $^ -c -o $@
+
 desres: controller_main.cc
 	g++ $(CXXFLAGS) $(CPPFLAGS) -DTEST_DESIRED_RESOURCES $^ -o $@ -L$(LIBEVENT)/lib -Wl,-rpath=$(LIBEVENT)/lib -lntl -lgmp
 

worker.cc

@@ -16,6 +16,8 @@ extern "C" {
 #include <errno.h>
 #include <signal.h>
 
+#include <cuda_runtime.h>
+
 #include "cudadl.h"
 
 #include "evutils.h"
@@ -26,6 +28,8 @@ NTL_CLIENT
 
 #undef VERBOSE
 
+static int cuda_device_id = -1;
+
 typedef enum {
     WRKCCSTATE_AWAITCMD,
     WRKCCSTATE_RDPROBLEM,
@@ -89,6 +93,13 @@ bool dpcallback(void *cbdata, unsigned int *dpwords)
 static void *worker_thread_start(void *data)
 {
     ZZ_p::init(wrkctrlstate.current_problem->modulus);
+
+    cudaError_t cudares = cudaSetDevice(cuda_device_id);
+    if (cudares != cudaSuccess) {
+	cerr << "Error setting CUDA device: " << cudaGetErrorString(cudares) << "\n";
+	exit(1);
+    }
+
     cuda_dl(to_ZZ_p(wrkctrlstate.current_problem->base),
 	    to_ZZ_p(wrkctrlstate.current_problem->target),
 	    wrkctrlstate.current_problem->order,
@@ -290,7 +301,7 @@ static void controllerconn_event_cb(struct bufferevent *bev, short events,
     }
 }
 
-int worker_main(const char *controller_host, unsigned short controller_port)
+int worker_main(const char *controller_host, unsigned short controller_port, int gpu_id)
 {
     // Initialize the prng with some randomness from the kernel
     unsigned char randbuf[1024];
@@ -304,6 +315,8 @@ int worker_main(const char *controller_host, unsigned short controller_port)
 
     signal(SIGPIPE, SIG_IGN);
 
+	cuda_device_id = gpu_id;
+
     return controller_client(controller_host, controller_port,
 				controllerconn_event_cb, true);
 }

worker.h

@@ -6,6 +6,6 @@
 // Used to prefix output from this process
 extern std::string output_prefix;
 
-int worker_main(const char *controller_host, unsigned short controller_port);
+int worker_main(const char *controller_host, unsigned short controller_port, int gpu_id);
 
 #endif

worker_main.cc

@@ -26,5 +26,5 @@ int main(int argc, char **argv)
     strcat(prefix, argv[3]);
     output_prefix = std::string(prefix);
 
-    return worker_main(argv[1], controller_port);
+    return worker_main(argv[1], controller_port, strtol(argv[3], NULL, 10));
 }