Ugly hack to improve multithreading support in scheduler

Wait until the scheduling loop has completed before notifying
any safe connections that we've written to the buffer.

src/core/or/safe_connection.c

@@ -171,6 +171,8 @@ safe_connection_init(safe_connection_t *safe_conn, uint32_t type_magic,
     safe_conn->inbuf = buf_new();
     safe_conn->outbuf = buf_new();
   }
+
+  safe_conn->care_about_modified = true;
 }
 
 void
@@ -377,13 +379,35 @@ safe_connection_set_write_permission(safe_connection_t *safe_conn,
   tor_mutex_release(&safe_conn->lock);
 }
 
+void
+safe_connection_start_caring_about_modified(safe_connection_t *safe_conn)
+{
+  tor_assert(safe_conn != NULL);
+  tor_mutex_acquire(&safe_conn->lock);
+
+  safe_conn->care_about_modified = true;
+
+  tor_mutex_release(&safe_conn->lock);
+}
+
+void
+safe_connection_stop_caring_about_modified(safe_connection_t *safe_conn)
+{
+  tor_assert(safe_conn != NULL);
+  tor_mutex_acquire(&safe_conn->lock);
+
+  safe_conn->care_about_modified = false;
+
+  tor_mutex_release(&safe_conn->lock);
+}
+
 void
 safe_connection_inbuf_modified(safe_connection_t *safe_conn)
 {
   tor_assert(safe_conn != NULL);
   tor_mutex_acquire(&safe_conn->lock);
 
-  if (safe_conn->inbuf_modified_cb != NULL) {
+  if (safe_conn->inbuf_modified_cb != NULL && safe_conn->care_about_modified) {
     safe_conn->inbuf_modified_cb(safe_conn);
   }
 
@@ -396,7 +420,7 @@ safe_connection_outbuf_modified(safe_connection_t *safe_conn)
   tor_assert(safe_conn != NULL);
   tor_mutex_acquire(&safe_conn->lock);
 
-  if (safe_conn->outbuf_modified_cb != NULL) {
+  if (safe_conn->outbuf_modified_cb != NULL && safe_conn->care_about_modified) {
     safe_conn->outbuf_modified_cb(safe_conn);
   }
   

src/core/or/safe_connection.h

@@ -146,6 +146,12 @@ void
 safe_connection_set_write_permission(safe_connection_t *safe_conn,
                                      bool write_allowed);
 
+void
+safe_connection_start_caring_about_modified(safe_connection_t *safe_conn);
+
+void
+safe_connection_stop_caring_about_modified(safe_connection_t *safe_conn);
+
 void
 safe_connection_inbuf_modified(safe_connection_t *safe_conn);
 

src/core/or/scheduler_kist.c

@@ -138,6 +138,16 @@ channel_outbuf_length(channel_t *chan)
   return len;
 }
 
+static int
+each_channel_update_outbuf(outbuf_table_ent_t *ent, void *data)
+{
+  (void) data;
+  connection_t *conn = TO_CONN(BASE_CHAN_TO_TLS(ent->chan)->conn);
+  tor_assert(conn->safe_conn != NULL);
+  safe_connection_start_caring_about_modified(conn->safe_conn);
+  safe_connection_outbuf_modified(conn->safe_conn);
+}
+
 /* Little helper function for HT_FOREACH_FN. */
 static int
 each_channel_write_to_kernel(outbuf_table_ent_t *ent, void *data)
@@ -622,16 +632,20 @@ kist_scheduler_run(void)
     }
     outbuf_table_add(&outbuf_table, chan);
 
+    connection_t *conn = TO_CONN(BASE_CHAN_TO_TLS(chan)->conn);
+    tor_assert(conn->safe_conn != NULL);
+    safe_connection_stop_caring_about_modified(conn->safe_conn);
+
     /* if we have switched to a new channel, consider writing the previous
      * channel's outbuf to the kernel. */
     if (!prev_chan) {
       prev_chan = chan;
     }
     if (prev_chan != chan) {
-      if (channel_should_write_to_kernel(&outbuf_table, prev_chan)) {
-        //channel_write_to_kernel(prev_chan);
-        outbuf_table_remove(&outbuf_table, prev_chan);
-      }
+      //if (channel_should_write_to_kernel(&outbuf_table, prev_chan)) {
+      //  channel_write_to_kernel(prev_chan);
+      //  outbuf_table_remove(&outbuf_table, prev_chan);
+      //}
       prev_chan = chan;
     }
 
@@ -732,6 +746,8 @@ kist_scheduler_run(void)
   /* Write the outbuf of any channels that still have data */
   //HT_FOREACH_FN(outbuf_table_s, &outbuf_table, each_channel_write_to_kernel,
   //              NULL);
+  HT_FOREACH_FN(outbuf_table_s, &outbuf_table, each_channel_update_outbuf,
+                NULL);
   /* We are done with it. */
   HT_FOREACH_FN(outbuf_table_s, &outbuf_table, free_outbuf_info_by_ent, NULL);
   HT_CLEAR(outbuf_table_s, &outbuf_table);