Added the scheduler as a subsystem

The scheduler now supports the 'initialize' and 'shutdown' subsystem
functions. This required the 'scheduler_init()' function to be broken
up, so now the scheduler must be initialized in three steps:
'scheduler_init()', 'scheduler_attach_mainloop()', and
'scheduler_conf_changed()'.

src/app/config/config.c

@@ -1493,12 +1493,9 @@ options_act_reversible(const or_options_t *old_options, char **msg)
       control_initialize_event_queue();
 
       /*
-       * Initialize the scheduler - this has to come after
-       * options_init_from_torrc() sets up libevent - why yes, that seems
-       * completely sensible to hide the libevent setup in the option parsing
-       * code!  It also needs to happen before init_keys(), so it needs to
-       * happen here too.  How yucky. */
-      scheduler_init();
+       * Initialize the scheduler - It needs to happen before init_keys(),
+       * so it needs to happen here.  How yucky. */
+      scheduler_conf_changed();
     }
 
     /* Adjust the port configuration so we can launch listeners. */

src/app/main/main.c

@@ -81,6 +81,7 @@
 
 #include "core/or/or_connection_st.h"
 #include "core/or/port_cfg_st.h"
+#include "core/or/scheduler.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -1134,6 +1135,7 @@ run_tor_main_loop(void)
   handle_signals();
   timers_initialize();
   initialize_mainloop_events();
+  scheduler_attach_mainloop();
 
   /* load the private keys, if we're supposed to have them, and set up the
    * TLS context. */

src/app/main/shutdown.c

@@ -128,7 +128,6 @@ tor_free_all(int postfork)
   channel_free_all();
   connection_free_all();
   connection_edge_free_all();
-  scheduler_free_all();
   nodelist_free_all();
   microdesc_free_all();
   routerparse_free_all();

src/app/main/subsystem_list.c

@@ -26,6 +26,7 @@
 #include "lib/tls/tortls_sys.h"
 #include "lib/wallclock/wallclock_sys.h"
 #include "lib/evloop/evloop_sys.h"
+#include "core/or/scheduler_sys.h"
 
 #include "feature/dirauth/dirauth_sys.h"
 
@@ -59,6 +60,7 @@ const subsys_fns_t *tor_subsystems[] = {
   &sys_evloop,
 
   &sys_mainloop,
+  &sys_scheduler,
   &sys_or,
 
   &sys_relay,

src/core/or/scheduler.c

@@ -4,6 +4,8 @@
 #include "core/or/or.h"
 #include "app/config/config.h"
 
+#include "core/or/scheduler_sys.h"
+
 #include "lib/evloop/compat_libevent.h"
 #define SCHEDULER_PRIVATE_
 #define SCHEDULER_KIST_PRIVATE
@@ -155,7 +157,7 @@
  *****************************************************************************/
 
 /** DOCDOC */
-STATIC const scheduler_t *the_scheduler;
+STATIC const scheduler_t *the_scheduler = NULL;
 
 /**
  * We keep a list of channels that are pending - i.e, have cells to write
@@ -602,15 +604,26 @@ scheduler_ev_active(void)
 }
 
 /*
- * Initialize everything scheduling-related from config.c. Note this is only
- * called when Tor is starting up, while scheduler_t->init() is called both
- * when Tor is starting up and when we are switching schedulers.
+ * Initialize any global memory needed by the scheduler. In order to use the
+ * scheduler, you must still tell it when the configuration from config.c is
+ * ready with scheduler_conf_changed(), and attach the mainloop event with
+ * scheduler_attach_mainloop(). Note this is only called when Tor is starting
+ * up, while scheduler_t->init() is called when we are switching schedulers.
  */
 void
 scheduler_init(void)
 {
-  log_debug(LD_SCHED, "Initting scheduler");
+  log_debug(LD_SCHED, "Initting scheduler memory");
 
+  channels_pending = smartlist_new();
+}
+
+/*
+ * Create and attach a new mainloop event.
+ */
+void
+scheduler_attach_mainloop(void)
+{
   // Two '!' because we really do want to check if the pointer is non-NULL
   IF_BUG_ONCE(!!run_sched_ev) {
     log_warn(LD_SCHED, "We should not already have a libevent scheduler event."
@@ -619,9 +632,6 @@ scheduler_init(void)
     run_sched_ev = NULL;
   }
   run_sched_ev = mainloop_event_new(scheduler_evt_callback, NULL);
-  channels_pending = smartlist_new();
-
-  set_scheduler();
 }
 
 /*
@@ -766,3 +776,24 @@ scheduler_touch_channel(channel_t *chan)
 }
 
 #endif /* defined(TOR_UNIT_TESTS) */
+
+static int
+subsys_scheduler_initialize(void)
+{
+  scheduler_init();
+  return 0;
+}
+
+static void
+subsys_scheduler_shutdown(void)
+{
+  scheduler_free_all();
+}
+
+const struct subsys_fns_t sys_scheduler = {
+  .name = "scheduler",
+  .supported = true,
+  .level = 10,
+  .initialize = subsys_scheduler_initialize,
+  .shutdown = subsys_scheduler_shutdown,
+};

src/core/or/scheduler.h

@@ -116,6 +116,7 @@ typedef struct scheduler_s {
  *****************************************************************************/
 
 void scheduler_init(void);
+void scheduler_attach_mainloop(void);
 void scheduler_free_all(void);
 void scheduler_conf_changed(void);
 void scheduler_notify_networkstatus_changed(void);

src/core/or/scheduler_sys.h

@@ -0,0 +1,11 @@
+/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_CORE_OR_SCHED_SYS_H
+#define TOR_CORE_OR_SCHED_SYS_H
+
+#include "lib/subsys/subsys.h"
+
+extern const struct subsys_fns_t sys_scheduler;
+
+#endif /* !defined(TOR_CORE_OR_SCHED_SYS_H) */

src/test/test_link_handshake.c

@@ -129,6 +129,8 @@ test_link_handshake_certs_ok(void *arg)
   tor_addr_from_ipv4h(&c2->base_.addr, 0x7f000001);
 
   scheduler_init();
+  scheduler_attach_mainloop();
+  scheduler_conf_changed();
 
   MOCK(tor_tls_cert_matches_key, mock_tls_cert_matches_key);
   MOCK(connection_or_write_var_cell_to_buf, mock_write_var_cell);
@@ -1220,6 +1222,8 @@ authenticate_data_setup(const struct testcase_t *test)
   testing__connection_or_pretend_TLSSECRET_is_supported = 1;
 
   scheduler_init();
+  scheduler_attach_mainloop();
+  scheduler_conf_changed();
 
   MOCK(connection_or_write_var_cell_to_buf, mock_write_var_cell);
   MOCK(tor_tls_get_peer_cert, mock_get_peer_cert);

src/test/test_scheduler.c

@@ -362,6 +362,8 @@ perform_channel_state_tests(int KISTSchedRunInterval, int sched_type)
 
   /* Set up scheduler */
   scheduler_init();
+  scheduler_attach_mainloop();
+  scheduler_conf_changed();
   /*
    * Install the compare channels mock so we can test
    * scheduler_touch_channel().
@@ -576,6 +578,8 @@ test_scheduler_loop_vanilla(void *arg)
 
   /* Set up scheduler */
   scheduler_init();
+  scheduler_attach_mainloop();
+  scheduler_conf_changed();
   /*
    * Install the compare channels mock so we can test
    * scheduler_touch_channel().
@@ -756,6 +760,8 @@ test_scheduler_loop_kist(void *arg)
   mocked_options.KISTSchedRunInterval = 11;
   set_scheduler_options(SCHEDULER_KIST);
   scheduler_init();
+  scheduler_attach_mainloop();
+  scheduler_conf_changed();
 
   tt_assert(ch1);
   ch1->magic = TLS_CHAN_MAGIC;
@@ -857,6 +863,8 @@ test_scheduler_initfree(void *arg)
   set_scheduler_options(SCHEDULER_VANILLA);
 
   scheduler_init();
+  scheduler_attach_mainloop();
+  scheduler_conf_changed();
 
   tt_ptr_op(channels_pending, !=, NULL);
   tt_ptr_op(run_sched_ev, !=, NULL);
@@ -1085,6 +1093,8 @@ test_scheduler_kist_pending_list(void *arg)
 
   /* Init scheduler. */
   scheduler_init();
+  scheduler_attach_mainloop();
+  scheduler_conf_changed();
 
   /* Initialize a channel. We'll need a second channel for the #24700 bug
    * test. */