Improved scheduling.

The scheduler / desired resources now takes into account the total number of workers and the ratio of workers to dpnodes. If that ratio is too high, the dpfreq is scaled back to prevent overloading the dpnodes. This commit makes changes to many of the programs' command line inputs.

controller.cc

@@ -884,10 +884,11 @@ static struct evconnlistener *controller_create(struct event_base *evbase,
 }
 
 int controller_parse_args(int argc, char **argv, unsigned short &bindport,
-    Worklist &worklist, unsigned short &total_nodes,
+    Worklist &worklist, unsigned short &total_workers, unsigned short &total_nodes,
     unsigned short &GB_mem_per_node)
 {
     bindport = 0;
+    total_workers = 1;
     total_nodes = 1;
     GB_mem_per_node = 1;
     int reps = 1;
@@ -898,6 +899,10 @@ int controller_parse_args(int argc, char **argv, unsigned short &bindport,
 	    bindport = strtoul(argv[2], NULL, 10);
 	    argc -= 2;
 	    argv += 2;
+	} else if (!strncmp(argv[1], "-w", 2)) {
+	    total_workers = strtoul(argv[2], NULL, 10);
+	    argc -= 2;
+	    argv += 2;
 	} else if (!strncmp(argv[1], "-n", 2)) {
 	    total_nodes = strtoul(argv[2], NULL, 10);
 	    argc -= 2;

controller.h

@@ -8,7 +8,7 @@ typedef std::pair<const char *, unsigned int> Workentry;
 typedef std::vector<Workentry> Worklist;
 
 int controller_parse_args(int argc, char **argv, unsigned short &bindport,
-    Worklist &worklist, unsigned short &total_nodes,
+    Worklist &worklist, unsigned short &total_workers, unsigned short &total_nodes,
     unsigned short &GB_mem_per_node);
 
 int controller_main(const Worklist &worklist, unsigned short bindport,

controller_main.cc

@@ -10,6 +10,8 @@
 
 NTL_CLIENT
 
+// The total number of workers
+static unsigned short total_workers = 1;
 // The total number of nodes (that is, dpnodes)
 static unsigned short total_nodes = 1;
 // The amount of memory we can use per dpnode
@@ -21,6 +23,7 @@ void custom_desired_resources(const ZZ &order,
     unsigned short &desired_dpnodes, unsigned int &max_workers,
     unsigned int &dpfreq)
 {
+    unsigned short custom_total_workers = total_workers;
     unsigned short custom_total_nodes = total_nodes;
 
 #ifdef MAKE_VERSIONS_COMPARABLE
@@ -30,9 +33,21 @@ void custom_desired_resources(const ZZ &order,
     // results are comparable to the dlrho version
 #endif
 
-    return desired_resources(order, custom_total_nodes,
-			    GB_mem_per_node, desired_dpnodes,
-			    max_workers, dpfreq);
+#if defined(SAVE_DPS) || defined(MAKE_VERSIONS_COMPARABLE)
+    custom_total_workers = 1;
+    // (1) can't use multiple workers when saving dps since
+    //     we want the order of dps to be deterministic
+    // (2) the dlrho version uses 1 worker per subproblem
+#endif
+
+    unsigned short freq_reduction_threshold = 4;
+    // this should be okay in most cases and shouldn't cause
+    // any negative effects if it's too low
+
+    return desired_resources(order, custom_total_workers,
+			    custom_total_nodes, GB_mem_per_node,
+			    freq_reduction_threshold,
+			    desired_dpnodes, max_workers, dpfreq);
 }
 
 static void boundcb(const char *boundaddr, unsigned short boundport)
@@ -61,9 +76,9 @@ int main(int argc, char **argv)
     unsigned short bindport = 0;
     Worklist worklist;
 
-    if (controller_parse_args(argc, argv, bindport, worklist,
+    if (controller_parse_args(argc, argv, bindport, worklist, total_workers,
 				total_nodes, GB_mem_per_node)) {
-	std::cerr << "Usage: " << argv[0] << " [-p listenport] [-n num_nodes] [-m GB_mem_per_node] [-r reps] N1 iter1 N2 iter2 ...\n";
+	std::cerr << "Usage: " << argv[0] << " [-p listenport] [-w num_workers] [-n num_dpnodes] [-m GB_mem_per_node] [-r reps] N1 iter1 N2 iter2 ...\n";
 	return 1;
     }
 

desired_resources.cc

@@ -8,10 +8,20 @@
 
 NTL_CLIENT
 
-// Input: order, total_nodes, GB_mem_per_node
+
+// For a given dpfreq, a worker will always transmit a specific
+// number of dps per second (this needs to be determined
+// experimentally). A dpnode can only accept a specific number
+// of dps per second.
+// freq_reduction_threshold should be equal to the max number of
+// workers that a dpnode can handle when the dpfreq == 4294967
+
+// Input: order, total_workers, total_dpnodes,
+//        GB_mem_per_node, freq_reduction_threshold
 // Output: desired_dpnodes, max_workers, dpfreq
-void desired_resources(const ZZ &order, unsigned short total_nodes,
-    unsigned short GB_mem_per_node, unsigned short &desired_dpnodes,
+void desired_resources(const ZZ &order, unsigned short total_workers,
+    unsigned short total_dpnodes, unsigned short GB_mem_per_node,
+    unsigned short freq_reduction_threshold, unsigned short &desired_dpnodes,
     unsigned int &max_workers, unsigned int &dpfreq)
 {
     // One point in how many is a DP by default?
@@ -24,27 +34,21 @@ void desired_resources(const ZZ &order, unsigned short total_nodes,
     ZZ dpnumerator = sorder * 338 * 10;
     ZZ dpdenominator;
     dpdenominator = GB_mem_per_node;
-    dpdenominator *= 1000000000UL;  // Convert the above line to GB
+    dpdenominator *= 1000000000UL;  // Convert the above line to B
     ZZ dpnodes = (dpnumerator / (dpdenominator * dpscale)) + 1;
-    if (dpnodes > total_nodes) {
-	desired_dpnodes = total_nodes;
-	ZZ zzdpscale = dpnumerator / (dpdenominator * total_nodes);
+    if (dpnodes > total_dpnodes) {
+	desired_dpnodes = total_dpnodes;
+	ZZ zzdpscale = dpnumerator / (dpdenominator * total_dpnodes);
 	if (NumBits(zzdpscale) > 31) {
 	    dpscale = 4294967295U;
 	} else {
 	    dpscale = trunc_long(zzdpscale, 31);
 	}
     } else {
-	desired_dpnodes = trunc_long(dpnodes, 31);
+	desired_dpnodes = total_dpnodes;
     }
 
     // How many workers would we like to use?
-#if defined(SAVE_DPS) || defined(MAKE_VERSIONS_COMPARABLE)
-    max_workers = 1;
-    // (1) can't use multiple workers when saving dps since
-    //     we want the order of dps to be deterministic
-    // (2) the dlrho version uses 1 worker per subproblem
-#else
     ZZ sorder23 = sorder >> 23;
     if (NumBits(sorder23) > 30) {
 	// Just use all the workers we can find
@@ -52,11 +56,22 @@ void desired_resources(const ZZ &order, unsigned short total_nodes,
     } else {
 	max_workers = trunc_long(sorder23,31) + 1;
     }
-#endif
+    if (max_workers > total_workers) {
+	max_workers = total_workers;
+    }
 
     // By default, 1 in dpscale points are distinguished points.
     dpfreq = 4294967295U / dpscale;
 
+    if ((float)max_workers/desired_dpnodes > freq_reduction_threshold) {
+	// if the ratio of workers to dpnodes is too high
+	unsigned int max_dpfreq = (4294967295U/1000) / (((float)max_workers/desired_dpnodes)/freq_reduction_threshold);
+	// max dpfreq for the given ratio of workers to dpnodes
+	if (dpfreq > max_dpfreq) {
+	    dpfreq = max_dpfreq;
+	}
+    }
+
     // Orders smaller than 100*scale^2 behave specially, in order to
     // avoid DP-free cycles
     ZZ orderlimit;
@@ -78,13 +93,16 @@ void desired_resources(const ZZ &order, unsigned short total_nodes,
 #ifdef TEST_DESIRED_RESOURCES
 int main(int argc, char **argv)
 {
-    if (argc != 3) {
-	std::cerr << "Usage: " << argv[0] << " num_nodes GB_mem_per_node\n";
+    if (argc != 5) {
+	std::cerr << "Usage: " << argv[0] << " num_workers num_dpnodes GB_mem_per_node freq_reduction_threshold\n";
 	return 1;
     }
 
-    unsigned short total_nodes = strtoul(argv[1], NULL, 10);
-    unsigned short GB_mem_per_node = strtoul(argv[2], NULL, 10);
+    unsigned short total_workers = strtoul(argv[1], NULL, 10);
+    unsigned short total_dpnodes = strtoul(argv[2], NULL, 10);
+    unsigned short GB_mem_per_node = strtoul(argv[3], NULL, 10);
+    unsigned short freq_reduction_threshold = strtoul(argv[4], NULL, 10);
+
     cout << "# log_2(order) dpnodes workers dpfreq\n";
     for (int i=44; i<=92; ++i) {
 	unsigned short desired_dpnodes;
@@ -94,7 +112,7 @@ int main(int argc, char **argv)
 	order = 1;
 	order <<= i;
 	order += 1;
-	desired_resources(order, total_nodes, GB_mem_per_node, desired_dpnodes, max_workers, dpfreq);
+	desired_resources(order, total_workers, total_dpnodes, GB_mem_per_node, freq_reduction_threshold, desired_dpnodes, max_workers, dpfreq);
 	cout << i << " " << desired_dpnodes << " " << max_workers << " "
 		<< dpfreq << "\n";
     }

desired_resources.h

@@ -2,6 +2,7 @@
 
 NTL_CLIENT
 
-void desired_resources(const ZZ &order, unsigned short total_nodes,
-    unsigned short GB_mem_per_node, unsigned short &desired_dpnodes,
+void desired_resources(const ZZ &order, unsigned short total_workers,
+    unsigned short total_dpnodes, unsigned short GB_mem_per_node,
+    unsigned short freq_factor, unsigned short &desired_dpnodes,
     unsigned int &max_workers, unsigned int &dpfreq);

dlrho.cc

@@ -150,7 +150,11 @@ static int p_dl(const ZZ_p &target, const ZZ_p &base, ZZ &exp,
 	unsigned int max_workers;
 	unsigned int dpfreq;
 
-	desired_resources(fvec[i], 1, GB_mem, desired_dpnodes, max_workers, dpfreq);
+	unsigned short freq_reduction_threshold = 4;
+	// this should be okay in most cases and shouldn't cause
+	// any negative effects if it's too low
+
+	desired_resources(fvec[i], 1, 1, GB_mem, freq_reduction_threshold, desired_dpnodes, max_workers, dpfreq);
 
 	ZZ quotient;
 	ZZ_p subgroup_base;