Rather than pre-allocating an array of randomness pools, just give each thread its own thread-local randomness pool

This way we can use randomness pools from dynamically created background
threads, such as for preprocessing Waksman networks.

App/appconfig.cpp

@@ -140,6 +140,7 @@ bool config_parse(Config &config, const std::string configstr,
         std::cerr << "Could not find my own node entry in config\n";
         ret = false;
     }
+    config.nthreads = nthreads;
 
     if (!ret) return ret;
 

App/appconfig.hpp

@@ -26,6 +26,7 @@ struct Config {
     uint8_t m_priv_in;
     uint8_t m_pub_out;
     uint8_t m_pub_in;
+    uint16_t nthreads;
     // config for each node
     std::vector<NodeConfig> nodes;
     // Which node is this one?

App/start.cpp

@@ -74,7 +74,15 @@ static void route_test(NetIO &netio, char **args)
     // Precompute some WaksmanNetworks
     size_t num_sizes = ecall_precompute_sort(-1);
     for (int i=0;i<int(num_sizes);++i) {
-        ecall_precompute_sort(i);
+        std::vector<boost::thread> ts;
+        for (int j=0; j<config.nthreads; ++j) {
+            ts.emplace_back([i] {
+                ecall_precompute_sort(i);
+            });
+        }
+        for (auto& t: ts) {
+            t.join();
+        }
     }
 
     if (!ecall_ingest_raw(msgs, tot_tokens)) {

Enclave/Enclave.config.xml

@@ -3,8 +3,8 @@
   <ProdID>0</ProdID>
   <ISVSVN>0</ISVSVN>
   <StackMaxSize>0x40000</StackMaxSize>
-  <HeapMaxSize>0x8000000</HeapMaxSize>
-  <TCSNum>10</TCSNum>
+  <HeapMaxSize>0x10000000</HeapMaxSize>
+  <TCSNum>32</TCSNum>
   <TCSPolicy>1</TCSPolicy>
   <DisableDebug>0</DisableDebug>
   <MiscSelect>0</MiscSelect>

Enclave/OblivAlgs/RecursiveShuffle.cpp

@@ -23,7 +23,6 @@ void MarkHalf(uint64_t N, bool *selected_list) {
   
   uint64_t left_to_mark = N/2;
   uint64_t total_left = N;
-  PRB_buffer *randpool = PRB_pool + g_thread_id;
   uint32_t coins[RS_MARKHALF_MAX_COINS];
   size_t coinsleft=0;
   
@@ -36,7 +35,7 @@ void MarkHalf(uint64_t N, bool *selected_list) {
         if (numcoins > RS_MARKHALF_MAX_COINS) {
             numcoins = RS_MARKHALF_MAX_COINS;
         }
-        randpool->getRandomBytes((unsigned char *) coins,
+        PRB_buf.getRandomBytes((unsigned char *) coins,
             sizeof(coins[0])*numcoins);
         coinsleft = numcoins;
     }

Enclave/OblivAlgs/utils.cpp

@@ -5,7 +5,7 @@
 thread_local uint64_t OSWAP_COUNTER=0;
 #endif
 
-PRB_buffer* PRB_pool;
+thread_local PRB_buffer PRB_buf;
 thread_local uint64_t PRB_rand_bits = 0;
 thread_local uint32_t PRB_rand_bits_remaining = 0;
 
@@ -494,19 +494,16 @@ size_t packetsConsumedUptoMSN(signed long msn_no, size_t msns_with_extra_packets
 
 #ifdef USE_PRB
   void PRB_pool_init(int nthreads) {
-    PRB_pool = new PRB_buffer[nthreads];
+    // Nothing needs to be done any more
   }
 
   void PRB_pool_shutdown() {
-    delete [] PRB_pool;
+    // Nothing needs to be done any more
   }
 
   PRB_buffer::PRB_buffer() {
   }
 
-  PRB_buffer::~PRB_buffer() {
-  }
-
   sgx_status_t PRB_buffer::init_PRB_buffer(uint32_t buffer_size = PRB_BUFFER_SIZE) {
     sgx_status_t rt = SGX_SUCCESS;
     if(initialized==false) {
@@ -577,6 +574,7 @@ size_t packetsConsumedUptoMSN(signed long msn_no, size_t msns_with_extra_packets
     }
   }
 
+/*
   sgx_status_t PRB_buffer::getBulkRandomBytes(unsigned char *buffer, size_t size) {
     sgx_status_t rt = SGX_SUCCESS;
     rt = sgx_aes_ctr_encrypt(random_seed, (const uint8_t*) buffer, size,
@@ -622,6 +620,7 @@ size_t packetsConsumedUptoMSN(signed long msn_no, size_t msns_with_extra_packets
     }
     return rt;
   }
+*/
 #else
   sgx_status_t getRandomBytes(unsigned char *random_bytes, size_t size) {
     sgx_status_t rt = SGX_SUCCESS;

Enclave/OblivAlgs/utils.hpp

@@ -161,7 +161,6 @@
 
       public:
         PRB_buffer();
-        ~PRB_buffer();
         sgx_status_t init_PRB_buffer(uint32_t buffer_size);
         /*  Intended for getting random bytes of size << PRB_BUFFER_SIZE at a time.
          Draws random bytes from the (typically) pre-filled random_bytes[PRB_BUFFER_SIZE] 
@@ -174,7 +173,7 @@
         */
         sgx_status_t getBulkRandomBytes(unsigned char *random_bytes, size_t size);
     };
-    extern PRB_buffer* PRB_pool;
+    extern thread_local PRB_buffer PRB_buf;
 
     // Spawn a PRB pool for each thread
     void PRB_pool_init(int nthreads);
@@ -183,8 +182,7 @@
     
     inline sgx_status_t getRandomBytes(unsigned char *random_bytes, size_t size) {
       FOAV_SAFE_CNTXT(PRB, size)
-      FOAV_SAFE_CNTXT(PRB, g_thread_id)
-      return((PRB_pool[g_thread_id]).getRandomBytes(random_bytes, size));
+      return(PRB_buf.getRandomBytes(random_bytes, size));
     }
 
     // Return a random bit

Enclave/config.cpp

@@ -93,7 +93,6 @@ bool ecall_config_load(threadid_t nthreads, bool private_routing,
 
     // Initialize the threadpool and the pseudorandom bytes pools
     threadpool_init(nthreads);
-    PRB_pool_init(nthreads);
 
     if (!route_init()) {
         return false;
@@ -103,6 +102,5 @@ bool ecall_config_load(threadid_t nthreads, bool private_routing,
 
 void ecall_close()
 {
-    PRB_pool_shutdown();
     threadpool_shutdown();
 }