Both versions now use 'desired_resources(...)'.

The 'desired_resources' function used in the controller/worker version is now used in the dlrho version as well. The dlrho program now requires a 'total_mem_GB' argument.

Makefile

@@ -47,7 +47,7 @@ ptx: $(PTX_TARGETS)
 gen_N: gen_N.o
 	g++ -g -Wall $^ -o $@ -lntl -L$(GMP)/lib -lgmp -lpthread
 
-dlrho: dlrho.o cudadl.o
+dlrho: dlrho.o cudadl.o desired_resources.o
 	g++ -g -Wall $^ -o $@ -lntl -L$(GMP)/lib -lgmp -L$(CUDA)/lib64 -lcudart -lpthread
 
 dlrho.o: dlrho.cc
@@ -65,7 +65,7 @@ test_cuda: parrhoasm.cu cios.asm dpstream.cu
 cios.asm: gencios_reg_20
 	./gencios_reg_20 $(WORDS) > $@
 
-controller: controller.o evutils.o controller_main.o
+controller: controller.o evutils.o controller_main.o desired_resources.o
 	g++ -g -Wall $^ -o $@ -L$(LIBEVENT)/lib -Wl,-rpath=$(LIBEVENT)/lib -levent -lntl -L$(GMP) -lgmp -lpthread
 
 dpnode: dpnode.o evutils.o dpnode_main.o
@@ -77,7 +77,7 @@ worker: worker.o evutils.o cudadl.o worker_main.o
 worker.o: worker.cc
 	g++ $(CXXFLAGS) $(CPPFLAGS) -I$(CUDA)/include $^ -c -o $@
 
-desres: controller_main.cc
+desres: desired_resources.cc
 	g++ $(CXXFLAGS) $(CPPFLAGS) -DTEST_DESIRED_RESOURCES $^ -o $@ -L$(LIBEVENT)/lib -Wl,-rpath=$(LIBEVENT)/lib -lntl -lgmp -lpthread
 
 mpi: mpi.o

controller.cc

@@ -182,8 +182,9 @@ static void ipportsetdump(const IPPortSet &ipps, ostream &os)
     os << "\n";
 }
 
-void desired_resources(const ZZ &order, unsigned short &desired_dpnodes,
-    unsigned int &max_workers, unsigned int &dpfreq);
+void custom_desired_resources(const ZZ &order,
+    unsigned short &desired_dpnodes, unsigned int &max_workers,
+    unsigned int &dpfreq);
 
 struct SubproblemProgress : Subproblem {
 
@@ -211,7 +212,7 @@ struct SubproblemProgress : Subproblem {
 	    // number in the next line is 2^32/1000
 	    Subproblem(id, b, t, m, o, 4294967), solved(false), kernel_launch_count(0), num_workers_replied(0) {
 
-	desired_resources(order, desired_dpnodes, max_workers, dpfreq);
+	custom_desired_resources(order, desired_dpnodes, max_workers, dpfreq);
 
 	// We need to update the binary description of the subproblem,
 	// as we may have just changed dpfreq.

controller_main.cc

@@ -1,11 +1,11 @@
 #include <NTL/ZZ.h>
-
 #include <iostream>
 
 #include <string.h>
 #include <stdlib.h>
 #include <unistd.h>
 
+#include "desired_resources.h"
 #include "controller.h"
 
 NTL_CLIENT
@@ -15,61 +15,17 @@ static unsigned short total_nodes = 1;
 // The amount of memory we can use per dpnode
 static unsigned short GB_mem_per_node = 1;
 
-void desired_resources(const ZZ &order, unsigned short &desired_dpnodes,
-    unsigned int &max_workers, unsigned int &dpfreq)
+// Input: order
+// Output: desired_dpnodes, max_workers, dpfreq
+void custom_desired_resources(const ZZ &order,
+    unsigned short &desired_dpnodes, unsigned int &max_workers,
+    unsigned int &dpfreq)
 {
-    // One point in how many is a DP by default?
-    unsigned int dpscale = 1000;
-    ZZ sorder = SqrRoot(order);
-
-    // How many DPnodes should we use for a problem of this size?
-
-    // 338 is bytes per DP in the table.  10 is a safety factor.
-    ZZ dpnumerator = sorder * 338 * 10;
-    ZZ dpdenominator;
-    dpdenominator = GB_mem_per_node;
-    dpdenominator *= 1000000000UL;  // Convert the above line to GB
-    ZZ dpnodes = (dpnumerator / (dpdenominator * dpscale)) + 1;
-    if (dpnodes > total_nodes) {
-	desired_dpnodes = total_nodes;
-	ZZ zzdpscale = dpnumerator / (dpdenominator * total_nodes);
-	if (NumBits(zzdpscale) > 31) {
-	    dpscale = 4294967295U;
-	} else {
-	    dpscale = trunc_long(zzdpscale, 31);
-	}
-    } else {
-	desired_dpnodes = trunc_long(dpnodes, 31);
-    }
-
-    // How many workers would we like to use?
-    ZZ sorder23 = sorder >> 23;
-    if (NumBits(sorder23) > 30) {
-	// Just use all the workers we can find
-	max_workers = 4294967295U;  // 2^32 - 1
-    } else {
-	max_workers = trunc_long(sorder23,31) + 1;
-    }
-
-    // By default, 1 in dpscale points are distinguished points.
-    dpfreq = 4294967295U / dpscale;
 
-    // Orders smaller than 100*scale^2 behave specially, in order to
-    // avoid DP-free cycles
-    ZZ orderlimit;
-    orderlimit = 100;
-    orderlimit *= dpscale;
-    orderlimit *= dpscale;
 
-    if (order < 1000) {
-	// Just make every point a DP
-	dpfreq = 4294967295U;
-    } else if (order < orderlimit) {
-	// The frequency of DPs should be 10/sqrt(order) to avoid
-	// a DP-free cycle, so dpfreq = (10*2^32)/sqrt(order)
-	ZZ f = (to_ZZ(10) << 32) / SqrRoot(order);
-	dpfreq = trunc_long(f, 31);
-    }
+    return desired_resources(order, total_nodes,
+			    GB_mem_per_node, desired_dpnodes,
+			    max_workers, dpfreq);
 }
 
 static void boundcb(const char *boundaddr, unsigned short boundport)
@@ -86,31 +42,6 @@ static void boundcb(const char *boundaddr, unsigned short boundport)
     }
 }
 
-#ifdef TEST_DESIRED_RESOURCES
-int main(int argc, char **argv)
-{
-    if (argc != 3) {
-	std::cerr << "Usage: " << argv[0] << " num_nodes GB_mem_per_node\n";
-	return 1;
-    }
-
-    total_nodes = strtoul(argv[1], NULL, 10);
-    GB_mem_per_node = strtoul(argv[2], NULL, 10);
-    cout << "# log_2(order) dpnodes workers dpfreq\n";
-    for (int i=44; i<=92; ++i) {
-	unsigned short desired_dpnodes;
-	unsigned int max_workers;
-	unsigned int dpfreq;
-	ZZ order;
-	order = 1;
-	order <<= i;
-	order += 1;
-	desired_resources(order, desired_dpnodes, max_workers, dpfreq);
-	cout << i << " " << desired_dpnodes << " " << max_workers << " "
-		<< dpfreq << "\n";
-    }
-}
-#else
 int main(int argc, char **argv)
 {
     unsigned short bindport = 0;
@@ -124,4 +55,3 @@ int main(int argc, char **argv)
 
     return controller_main(worklist, bindport, boundcb);
 }
-#endif

desired_resources.cc

@@ -0,0 +1,95 @@
+#include <iostream>
+
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "desired_resources.h"
+
+NTL_CLIENT
+
+// Input: order, total_nodes, GB_mem_per_node
+// Output: desired_dpnodes, max_workers, dpfreq
+void desired_resources(const ZZ &order, unsigned short total_nodes,
+    unsigned short GB_mem_per_node, unsigned short &desired_dpnodes,
+    unsigned int &max_workers, unsigned int &dpfreq)
+{
+    // One point in how many is a DP by default?
+    unsigned int dpscale = 1000;
+    ZZ sorder = SqrRoot(order);
+
+    // How many DPnodes should we use for a problem of this size?
+
+    // 338 is bytes per DP in the table.  10 is a safety factor.
+    ZZ dpnumerator = sorder * 338 * 10;
+    ZZ dpdenominator;
+    dpdenominator = GB_mem_per_node;
+    dpdenominator *= 1000000000UL;  // Convert the above line to GB
+    ZZ dpnodes = (dpnumerator / (dpdenominator * dpscale)) + 1;
+    if (dpnodes > total_nodes) {
+	desired_dpnodes = total_nodes;
+	ZZ zzdpscale = dpnumerator / (dpdenominator * total_nodes);
+	if (NumBits(zzdpscale) > 31) {
+	    dpscale = 4294967295U;
+	} else {
+	    dpscale = trunc_long(zzdpscale, 31);
+	}
+    } else {
+	desired_dpnodes = trunc_long(dpnodes, 31);
+    }
+
+    // How many workers would we like to use?
+    ZZ sorder23 = sorder >> 23;
+    if (NumBits(sorder23) > 30) {
+	// Just use all the workers we can find
+	max_workers = 4294967295U;  // 2^32 - 1
+    } else {
+	max_workers = trunc_long(sorder23,31) + 1;
+    }
+
+    // By default, 1 in dpscale points are distinguished points.
+    dpfreq = 4294967295U / dpscale;
+
+    // Orders smaller than 100*scale^2 behave specially, in order to
+    // avoid DP-free cycles
+    ZZ orderlimit;
+    orderlimit = 100;
+    orderlimit *= dpscale;
+    orderlimit *= dpscale;
+
+    if (order < 1000) {
+	// Just make every point a DP
+	dpfreq = 4294967295U;
+    } else if (order < orderlimit) {
+	// The frequency of DPs should be 10/sqrt(order) to avoid
+	// a DP-free cycle, so dpfreq = (10*2^32)/sqrt(order)
+	ZZ f = (to_ZZ(10) << 32) / SqrRoot(order);
+	dpfreq = trunc_long(f, 31);
+    }
+}
+
+#ifdef TEST_DESIRED_RESOURCES
+int main(int argc, char **argv)
+{
+    if (argc != 3) {
+	std::cerr << "Usage: " << argv[0] << " num_nodes GB_mem_per_node\n";
+	return 1;
+    }
+
+    unsigned short total_nodes = strtoul(argv[1], NULL, 10);
+    unsigned short GB_mem_per_node = strtoul(argv[2], NULL, 10);
+    cout << "# log_2(order) dpnodes workers dpfreq\n";
+    for (int i=44; i<=92; ++i) {
+	unsigned short desired_dpnodes;
+	unsigned int max_workers;
+	unsigned int dpfreq;
+	ZZ order;
+	order = 1;
+	order <<= i;
+	order += 1;
+	desired_resources(order, total_nodes, GB_mem_per_node, desired_dpnodes, max_workers, dpfreq);
+	cout << i << " " << desired_dpnodes << " " << max_workers << " "
+		<< dpfreq << "\n";
+    }
+}
+#endif

desired_resources.h

@@ -0,0 +1,7 @@
+#include <NTL/ZZ.h>
+
+NTL_CLIENT
+
+void desired_resources(const ZZ &order, unsigned short total_nodes,
+    unsigned short GB_mem_per_node, unsigned short &desired_dpnodes,
+    unsigned int &max_workers, unsigned int &dpfreq);

dlrho.cc

@@ -31,6 +31,7 @@
 #include <cuda_runtime.h>
 
 #include "atomic_iostream.h"
+#include "desired_resources.h"
 #include "cudadl.h"
 
 NTL_CLIENT
@@ -113,7 +114,8 @@ bool dpcallback(void *cbdata, unsigned int *dpwords)
 // label is "p" or "q", to be printed to report progress.
 // Return 0 on failure, 1 on success.
 static int p_dl(const ZZ_p &target, const ZZ_p &base, ZZ &exp,
-    const vec_ZZ &fvec, const string &label, unsigned int initial_subproblem_id)
+    const vec_ZZ &fvec, const string &label, unsigned int initial_subproblem_id,
+    unsigned short GB_mem)
 {
     const int flen = fvec.length();
 
@@ -130,6 +132,12 @@ static int p_dl(const ZZ_p &target, const ZZ_p &base, ZZ &exp,
     curexp = 0;
 
     for (int i = 0; i < flen; ++i) {
+	unsigned short desired_dpnodes;
+	unsigned int max_workers;
+	unsigned int dpfreq;
+
+	desired_resources(fvec[i], 1, GB_mem, desired_dpnodes, max_workers, dpfreq);
+
 	ZZ quotient;
 	ZZ_p subgroup_base;
 	ZZ_p subgroup_target;
@@ -137,6 +145,7 @@ static int p_dl(const ZZ_p &target, const ZZ_p &base, ZZ &exp,
 	{
 	    AtomicWriter atomic_cout(cout);
 	    atomic_cout << label << " submodulus " << i+1 << " of " << flen << "...\n";
+	    atomic_cout << "Settings (memory, dpfreq): " << GB_mem << ", " << dpfreq << "\n";
 	    // Figure out exp mod fvec[i] by taking each side to the power
 	    // of phirho/fvec[i] so that we're working in the
 	    // subgroup of order fvec[i].
@@ -163,10 +172,6 @@ static int p_dl(const ZZ_p &target, const ZZ_p &base, ZZ &exp,
 	ZZ md = ZZ_p::modulus();
 	CBData cbdata(subgroup_base, subgroup_target, fvec[i]);
 	struct timeval st, et;
-	unsigned int dpfreq = 4294967; // 2^32/1000
-	if (fvec[i] < 1000000) {
-	    dpfreq = 4294967295; // 2^32-1 : every point is a DP
-	}
 	gettimeofday(&st, NULL);
 	unsigned int launch_count = 0;
 	{
@@ -211,7 +216,9 @@ typedef struct {
 
 // Behave like p_dl, but do the work in an asynchronous subprocess
 static PDLHandle* p_dl_fork_start(const ZZ &p, const ZZ_p &target,
-    const ZZ_p &base, const vec_ZZ &fvec, const string &label, int deviceid, unsigned int initial_subproblem_id)
+    const ZZ_p &base, const vec_ZZ &fvec, const string &label,
+    int deviceid, unsigned int initial_subproblem_id,
+    unsigned short GB_mem_per_subprocess)
 {
     PDLHandle *handle = new PDLHandle;
     int fds[2];
@@ -249,7 +256,7 @@ static PDLHandle* p_dl_fork_start(const ZZ &p, const ZZ_p &target,
 	    AtomicWriter(cerr) << "Error setting CUDA device: " << cudaGetErrorString(cudares) << "\n";
 	    exit(1);
 	}
-	int res = p_dl(target_p, base_p, exp, fvec, label, initial_subproblem_id);
+	int res = p_dl(target_p, base_p, exp, fvec, label, initial_subproblem_id, GB_mem_per_subprocess);
 
 	if (res) {
 	    // Write the result back to the parent using wfd
@@ -301,6 +308,13 @@ static int p_dl_fork_join(PDLHandle *handle, ZZ &exp)
 
 int main(int argc, char **argv)
 {
+    if (argc != 2) {
+	cerr << "Usage: " << argv[0] << " total_mem_GB";
+	exit(1);
+    }
+
+    unsigned short GB_mem_per_node = strtol(argv[1], NULL, 10);
+
     // Initialize the prng with some randomness from the kernel
     unsigned char randbuf[1024];
     ifstream urand("/dev/urandom");
@@ -337,8 +351,8 @@ int main(int argc, char **argv)
 
     PDLHandle *handle_p, *handle_q;
 
-    handle_p = p_dl_fork_start(p, target, base, pfvec, "p", 0, 0);
-    handle_q = p_dl_fork_start(q, target, base, qfvec, "q", 1, pfvec.length());
+    handle_p = p_dl_fork_start(p, target, base, pfvec, "p", 0, 0, GB_mem_per_node/2);
+    handle_q = p_dl_fork_start(q, target, base, qfvec, "q", 1, pfvec.length(), GB_mem_per_node/2);
     res_p = p_dl_fork_join(handle_p, exp_p);
     res_q = p_dl_fork_join(handle_q, exp_q);