cudadl/worker.cc

#include <pthread.h>

extern "C" {
#include <event2/thread.h>
#include <event2/bufferevent.h>
#include <event2/buffer.h>
#include <event2/event.h>
}

#include <NTL/ZZ_p.h>

#include <vector>
#include <stdio.h>

#include "cudadl.h"

#include "evutils.h"
#include "subproblem.h"

NTL_CLIENT

typedef enum {
    WRKCCSTATE_AWAITCMD,
    WRKCCSTATE_RDPROBLEM,
    WRKCCSTATE_RDDPNODES,
    WRKCCSTATE_END
} WrkCCState;

struct WrkControllerConnInfo {
    WrkCCState state;
    unsigned short num_dpnodes;

    WrkControllerConnInfo() : state(WRKCCSTATE_AWAITCMD), num_dpnodes(0) {}
};

typedef enum { WT_NOT_RUNNING, WT_RUNNING, WT_SHOULD_STOP} WTState;

static struct WrkControllerState {
    Subproblem *current_problem;
    unsigned short num_expected_dpnodes;
    vector<struct bufferevent *> dpnodes;
    unsigned short num_connected_dpnodes;
    WTState worker_thread_state;
    pthread_t worker_thread;

    WrkControllerState(): current_problem(NULL),
	worker_thread_state(WT_NOT_RUNNING) {}
} wrkctrlstate;

// ----- Below this line are the functions running in the worker thread.


// This function is called from inside cuda_dl for each DP it encounters.
// It calls the function named "dpcallback" directly.  It would be
// cleaner if this were passed as a function pointer to cuda_dl, but
// that makes nvcc 3.1 segfault. :-p
bool dpcallback(void *cbdata, unsigned short threadId,
    unsigned short blockId, unsigned int demux, unsigned int *dpwords,
    unsigned int a_0, unsigned int a_1, unsigned int a_2,
    unsigned int b_0, unsigned int b_1, unsigned int b_2)
{
    struct bufferevent *bev =
	wrkctrlstate.dpnodes[demux % wrkctrlstate.num_connected_dpnodes];

    unsigned char DPbuf[(WORDS+6)*sizeof(unsigned int)];
    memmove(DPbuf, dpwords, WORDS*sizeof(unsigned int));
    memmove(DPbuf+WORDS*sizeof(unsigned int), &a_0, sizeof(unsigned int));
    memmove(DPbuf+(WORDS+1)*sizeof(unsigned int), &a_1, sizeof(unsigned int));
    memmove(DPbuf+(WORDS+2)*sizeof(unsigned int), &a_2, sizeof(unsigned int));
    memmove(DPbuf+(WORDS+3)*sizeof(unsigned int), &b_0, sizeof(unsigned int));
    memmove(DPbuf+(WORDS+4)*sizeof(unsigned int), &b_1, sizeof(unsigned int));
    memmove(DPbuf+(WORDS+5)*sizeof(unsigned int), &b_2, sizeof(unsigned int));

    bufferevent_write(bev, DPbuf, (WORDS+6)*sizeof(unsigned int));

    // If worker_thread_state changes to WT_SHOULD_STOP, then signal to
    // stop computation by returning true.  If for some reason, it
    // becomes WT_NOT_RUNNING (which it shouldn't), stop as well.
    return wrkctrlstate.worker_thread_state != WT_RUNNING;
}

static void *worker_thread_start(void *data)
{
    ZZ_p::init(wrkctrlstate.current_problem->modulus);
    cuda_dl(to_ZZ_p(wrkctrlstate.current_problem->base),
	    to_ZZ_p(wrkctrlstate.current_problem->target),
	    wrkctrlstate.current_problem->order,
	    wrkctrlstate.current_problem->modulus, NULL);

    return NULL;
}

// ----- Above this line are the functions running in the worker thread.
//       Below are the functions running in the main (communication) thread.

static void stop_working(void)
{
    cerr << "Stopping work\n";
    if (wrkctrlstate.worker_thread_state != WT_NOT_RUNNING) {
	// Tell the worker thread to stop after its next kernel launch
	wrkctrlstate.worker_thread_state = WT_SHOULD_STOP;
	pthread_join(wrkctrlstate.worker_thread, NULL);
	wrkctrlstate.worker_thread_state = WT_NOT_RUNNING;
    }

    // Close the connections to the dpnodes
    vector<struct bufferevent *>::iterator bevit;
    for (bevit = wrkctrlstate.dpnodes.begin();
	    bevit != wrkctrlstate.dpnodes.end(); ++bevit) {
	bufferevent_free(*bevit);
    }
    wrkctrlstate.dpnodes.clear();
    wrkctrlstate.num_connected_dpnodes = 0;
    wrkctrlstate.num_expected_dpnodes = 0;
    // Careful!  Subproblem uses NTL, so we must be sure we're not
    // multithreaded at this point.
    delete wrkctrlstate.current_problem;
    wrkctrlstate.current_problem = NULL;
}

static void start_working(void)
{
    cerr << "Starting work\n";
    wrkctrlstate.worker_thread_state = WT_RUNNING;
    if (pthread_create(&wrkctrlstate.worker_thread, NULL,
			worker_thread_start, NULL)) {
	wrkctrlstate.worker_thread = WT_NOT_RUNNING;
	cerr << "Could not start worker thread\n";
    }
}

static void dpconn_event_cb(struct bufferevent *bev, short events,
    void *ctx)
{
    if (events & BEV_EVENT_CONNECTED) {
	// We have successfully connected to the dpnode
	bufferevent_enable(bev, EV_WRITE);
	++wrkctrlstate.num_connected_dpnodes;
	if (wrkctrlstate.num_connected_dpnodes ==
		wrkctrlstate.num_expected_dpnodes) {
	    start_working();
	}
    } else if (events & (BEV_EVENT_EOF|BEV_EVENT_ERROR)) {
	fprintf(stderr, "Closing connection to dpnode\n");
	stop_working();
	bufferevent_free(bev);
    }
}

static void controllerconn_reader(struct bufferevent *bev, void *ctx)
{
    struct evbuffer *input = bufferevent_get_input(bev);
    WrkControllerConnInfo *info = (WrkControllerConnInfo *)ctx;
    unsigned char cmd[1];
    unsigned char subproblem[SUBPROBLEM_DESC_LEN];

    while(1) {
	size_t len = evbuffer_get_length(input);
	switch(info->state) {
	    case WRKCCSTATE_AWAITCMD:
		if (len < 1) return;
		bufferevent_read(bev, cmd, 1);
		switch(cmd[0]) {
		    case 'P':
			info->state = WRKCCSTATE_RDPROBLEM;
			break;
		    case 'S':
			stop_working();
			break;
		    default:
			/* Unknown command received */
			fprintf(stderr, "Unknown command in "
				"controllerconn_reader: %c\n", cmd[0]);
			info->state = WRKCCSTATE_END;
			break;
		}
		break;

	    case WRKCCSTATE_RDPROBLEM:
		if (len < SUBPROBLEM_DESC_LEN+2) return;
		stop_working();
		bufferevent_read(bev, subproblem, SUBPROBLEM_DESC_LEN);
		bufferevent_read(bev, &(info->num_dpnodes), 2);
		// Careful!  Subproblem uses NTL, so we must be sure
		// we're not multithreaded at this point.
		wrkctrlstate.current_problem = new Subproblem(subproblem);
		info->state = WRKCCSTATE_RDDPNODES;
		/* FALLTHROUGH */

	    case WRKCCSTATE_RDDPNODES:
		if (len < 6*(info->num_dpnodes)) return;
		wrkctrlstate.num_expected_dpnodes = info->num_dpnodes;
		{
		    unsigned short i;
		    for(i=0;i<info->num_dpnodes;++i) {
			unsigned char ipport[6];
			bufferevent_read(bev, ipport, 6);
			cerr << "Connecting to DPnode " <<
				    int(ipport[0]) << "." <<
				    int(ipport[1]) << "." <<
			            int(ipport[2]) << "." <<
				    int(ipport[3]) << ":" <<
				    ((ipport[4] << 8) + ipport[5]) <<
				    "\n";
			struct bufferevent *dpbev = client_create(
				bufferevent_get_base(bev), ipport,
				dpconn_event_cb, true);
			if (dpbev) {
			    wrkctrlstate.dpnodes.push_back(dpbev);
			} else {
			    stop_working();
			}
		    }
		}
		info->state = WRKCCSTATE_AWAITCMD;
		break;

	    case WRKCCSTATE_END:
		// Shut down
		delete info;
		event_base_loopbreak(bufferevent_get_base(bev));
		bufferevent_free(bev);
		return;
	}
    }
}

static void controllerconn_event_cb(struct bufferevent *bev, short events,
    void *ctx)
{
    if (events & BEV_EVENT_CONNECTED) {
	// We have successfully connected to the controller
	char id[1] = { 'W' };

        bufferevent_enable(bev, EV_READ|EV_WRITE);
	bufferevent_write(bev, id, 1);
	bufferevent_setcb(bev, controllerconn_reader, NULL,
		controllerconn_event_cb, new WrkControllerConnInfo());
    } else if (events & (BEV_EVENT_EOF|BEV_EVENT_ERROR)) {
	fprintf(stderr, "Closing connection to controller and exiting\n");
	event_base_loopbreak(bufferevent_get_base(bev));
	bufferevent_free(bev);
    }
}

int main(int argc, char **argv)
{
    if (argc != 3) {
	fprintf(stderr, "Usage: %s controller_host controller_port\n", argv[0]);
	return 1;
    }

    unsigned short controller_port = strtoul(argv[2], NULL, 10);

    evthread_use_pthreads();

    return controller_client(argv[1], controller_port,
				controllerconn_event_cb, true);
}