fixed bugs, added crypto locks for multithreading

server/Makefile

@@ -4,12 +4,12 @@ TARGETS=slitheen-proxy
 
 all: $(TARGETS)
 
-slitheen-proxy.o flow.o rserv.o ptwist168.o util.o crypto.o relay.o:: ptwist.h rserv.h flow.h slitheen.h util.h crypto.h relay.h
+slitheen-proxy.o flow.o rserv.o ptwist168.o util.o crypto.o relay.o cryptothread.o :: ptwist.h rserv.h flow.h slitheen.h util.h crypto.h relay.h cryptothread.h
 
 rserv: rserv.o ptwist168.o
 	gcc -g -ggdb -o $@ $^ -lssl -lcrypto
 
-slitheen-proxy: slitheen-proxy.o flow.o rserv.o ptwist168.o crypto.o relay.o relay.h crypto.h ptwist.h rserv.h flow.h slitheen.h
+slitheen-proxy: slitheen-proxy.o flow.o rserv.o ptwist168.o crypto.o relay.o cryptothread.o relay.h crypto.h ptwist.h rserv.h flow.h slitheen.h cryptothread.h
 	gcc -g -ggdb -o $@ $^ -I/home/slitheen/Documents/include/openssl libssl.a libcrypto.a -lpcap -lpthread -ldl
 
 clean:

server/crypto.c

@@ -592,8 +592,9 @@ int compute_master_secret(flow *f){
 	
 	PRF(f, pre_master_secret, pre_master_len, (uint8_t *) TLS_MD_MASTER_SECRET_CONST, TLS_MD_MASTER_SECRET_CONST_SIZE, f->client_random, SSL3_RANDOM_SIZE, f->server_random, SSL3_RANDOM_SIZE, NULL, 0, f->master_secret, SSL3_MASTER_SECRET_SIZE);
 
-	if(f->current_session != NULL)
-		memcpy(f->current_session, f->master_secret, SSL3_MASTER_SECRET_SIZE);
+	if(f->current_session != NULL){
+		memcpy(f->current_session->master_secret, f->master_secret, SSL3_MASTER_SECRET_SIZE);
+	}
 
 #ifdef DEBUG
 	fprintf(stdout, "Premaster Secret:\n");
@@ -642,6 +643,11 @@ err:
 		EC_KEY_free(clnt_ecdh);
 	}
 
+///???
+	if(priv_key != NULL){
+		BN_free(priv_key);
+	}
+
 	return ok;
 }
 
@@ -926,10 +932,18 @@ int init_ciphers(flow *f){
     }
 #endif 
 
-	EVP_CipherInit_ex(r_ctx, c, NULL, read_key, NULL, 0);
-	EVP_CipherInit_ex(w_ctx, c, NULL, write_key, NULL, 1);
-	EVP_CipherInit_ex(w_ctx_srvr, c, NULL, read_key, NULL, 1);
-	EVP_CipherInit_ex(r_ctx_srvr, c, NULL, write_key, NULL, 0);
+	if(!EVP_CipherInit_ex(r_ctx, c, NULL, read_key, NULL, 0)){
+		printf("FAIL r_ctx\n");
+	}
+	if(!EVP_CipherInit_ex(w_ctx, c, NULL, write_key, NULL, 1)){
+		printf("FAIL w_ctx\n");
+	}
+	if(!EVP_CipherInit_ex(w_ctx_srvr, c, NULL, read_key, NULL, 1)){
+		printf("FAIL w_ctx_srvr\n");
+	}
+	if(!EVP_CipherInit_ex(r_ctx_srvr, c, NULL, write_key, NULL, 0)){
+		printf("FAIL r_ctx_srvr\n");
+	}
 	EVP_CIPHER_CTX_ctrl(r_ctx, EVP_CTRL_GCM_SET_IV_FIXED, EVP_GCM_TLS_FIXED_IV_LEN, read_iv);
 	EVP_CIPHER_CTX_ctrl(w_ctx, EVP_CTRL_GCM_SET_IV_FIXED, EVP_GCM_TLS_FIXED_IV_LEN, write_iv);
 	EVP_CIPHER_CTX_ctrl(w_ctx_srvr, EVP_CTRL_GCM_SET_IV_FIXED, EVP_GCM_TLS_FIXED_IV_LEN, read_iv);

server/cryptothread.c

@@ -0,0 +1,52 @@
+
+#include <pthread.h>
+#include <openssl/crypto.h>
+#include "cryptothread.h"
+
+static pthread_mutex_t *crypto_locks;
+static long *lock_count;
+
+void init_crypto_locks(void){
+
+	crypto_locks = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t));
+	if(!crypto_locks)
+		exit(1);
+	lock_count = OPENSSL_malloc(CRYPTO_num_locks() * sizeof(long));
+	if(!lock_count)
+		exit(1);
+	for (int i = 0; i < CRYPTO_num_locks(); i++) {
+		lock_count[i] = 0;
+		pthread_mutex_init(&(crypto_locks[i]), NULL);
+	}
+
+	CRYPTO_THREADID_set_callback(pthreads_thread_id);
+	CRYPTO_set_locking_callback(pthreads_locking_callback);
+}
+
+void crypto_locks_cleanup(void){
+	int i;
+
+	CRYPTO_set_locking_callback(NULL);
+	for (i = 0; i < CRYPTO_num_locks(); i++) {
+		pthread_mutex_destroy(&(crypto_locks[i]));
+	}
+	OPENSSL_free(crypto_locks);
+	OPENSSL_free(lock_count);
+
+}
+
+/** If the mode is CRYPTO_LOCK, the lock indicated by type will be acquired, otherwise it will be released */
+void pthreads_locking_callback(int mode, int type, const char *file, int line){
+
+	if(mode & CRYPTO_LOCK){
+		pthread_mutex_lock(&(crypto_locks[type]));
+		lock_count[type]++;
+	} else {
+		pthread_mutex_unlock(&(crypto_locks[type]));
+	}
+}
+
+void pthreads_thread_id(CRYPTO_THREADID *tid){
+	CRYPTO_THREADID_set_numeric(tid, (unsigned long)pthread_self());
+}
+

server/cryptothread.h

@@ -0,0 +1,10 @@
+#ifndef _CRYPTOTHREAD_H_
+#define _CRYPTOTHREAD_H_
+
+#include <openssl/crypto.h>
+void init_crypto_locks(void);
+void crypto_locks_cleanup(void);
+
+void pthreads_thread_id(CRYPTO_THREADID *tid);
+void pthreads_locking_callback(int mode, int type, const char *file, int line);
+#endif /* _CRYPTOTHREAD_H_ */

server/flow.c

@@ -57,6 +57,7 @@ flow *add_flow(flow newFlow) {
 	sem_init(&(newFlow.packet_chain_lock), 0, 1);
 	newFlow.upstream_queue = NULL;
 	newFlow.upstream_remaining = 0;
+	sem_init(&(newFlow.upstream_queue_lock), 0, 1);
 	newFlow.outbox = NULL;
 	newFlow.outbox_len = 0;
 	newFlow.outbox_offset = 0;
@@ -67,6 +68,8 @@ flow *add_flow(flow newFlow) {
 	newFlow.httpstate = PARSE_HEADER;
 	newFlow.replace_response = 0;
 
+	newFlow.ecdh = NULL;
+
 
 	newFlow.finish_md_ctx = EVP_MD_CTX_create();
 	const EVP_MD *md = EVP_sha384();
@@ -133,17 +136,21 @@ int update_flow(flow *f) {
 	for(int i=0; (i<data_len) && (i<record_len); i++){
 		record[i] = p[i];
 	}
+	printf("Processing seq num %u\n", current->seq_num);
 
 	while(record_len > data_len) {
 		if(current->next == NULL){
+			printf("Not enough data\n");
 			goto err;
 		}
 		if(current->next->seq_num != current->seq_num + current->len){
-			printf("Missing packet: seq_num= %d, datalen= %d, nextseq= %d\n", current->seq_num, current->len, current->next->seq_num);
+			printf("Missing packet: seq_num= %u, datalen= %d, nextseq= %u\n", current->seq_num, current->len, current->next->seq_num);
 			goto err;
 		}
 
 		current = current->next;
+		printf("Processing seq num %u\n", current->seq_num);
+		
 		p = current->data;
 		int i;
 		for(i=0; (i<current->data_len) && (i+data_len < record_len); i++){
@@ -172,6 +179,9 @@ int update_flow(flow *f) {
 					printf("%02x ", p[i]);
 				}
 				printf("\n");
+				if(p[0] != 0x14){
+					p[0] = 0x20; //trigger error
+				}
 
 				update_context(f, p, n, incoming, 0x16, 0);
 				if(incoming) f->in_encrypted = 2;
@@ -238,12 +248,17 @@ int update_flow(flow *f) {
 					if((f->in_encrypted == 2) && (f->out_encrypted == 2)){
 						printf("Handshake complete!\n");
 						f->application = 1;
-						if(current->incoming)
-							f->seq_num = current->seq_num + current->len;
+						if(current->incoming){
+							f->downstream_seq_num = current->seq_num + current->len;
+						} else {
+							f->upstream_seq_num = current->seq_num + current->len;
+						}
 						while(current->next != NULL){
 							current = current->next;
 							if(current->incoming)
-								f->seq_num = current->seq_num+ current->len;
+								f->downstream_seq_num = current->seq_num+ current->len;
+							else
+								f->upstream_seq_num = current->seq_num+ current->len;
 						}
 					}
 					break;
@@ -298,7 +313,11 @@ int update_flow(flow *f) {
 	}
 
 	//if(!f->application){
-		f->seq_num = current->seq_num;
+		if(current->incoming){
+			f->downstream_seq_num = current->seq_num;
+		} else {
+			f->upstream_seq_num = current->seq_num;
+		}
 
 		if(record_len == data_len){
 			/* record ended on packet boundary */
@@ -360,6 +379,10 @@ int remove_flow(flow *f) {
 		EVP_CIPHER_CTX_free(f->srvr_write_ctx);
 	}
 
+	if(f->ecdh != NULL){
+		EC_KEY_free(f->ecdh);
+	}
+
 	sem_wait(&flow_table_lock);
 	flow_entry *entry = table->first_entry;
 	if(entry->f == f){
@@ -498,11 +521,19 @@ int verify_session_id(flow *f, uint8_t *hs){
 	//check to see if it matches flow's session id set by ClientHello
 	if(f->current_session != NULL && f->current_session->session_id_len > 0 && !memcmp(f->current_session->session_id, p, id_len)){
 		//if it matched, update flow with master secret :D
+		printf("Session id matched!\n");
+		printf("First session id (%p->%p):", sessions, sessions->first_session);
 		session *last = sessions->first_session;
 		int found = 0;
 		for(int i=0; ((i<sessions->length) && (!found)); i++){
+			printf("Checking saved session id: ");
+			for (int j=0; j< last->session_id_len; j++){
+				printf("%02x ", last->session_id[j]);
+			}
+			printf("\n");
 			if(!memcmp(last->session_id, f->current_session->session_id, id_len)){
 				memcpy(f->master_secret, last->master_secret, SSL3_MASTER_SECRET_SIZE);
+				printf("Found session id number!\n");
 				found = 1;
 			}
 			last = last->next;
@@ -561,6 +592,11 @@ int check_session(flow *f, uint8_t *hs, uint32_t len){
 		memcpy(new_session->session_id, p, new_session->session_id_len);
 		new_session->next = NULL;
 		printf("Requested new session (%x:%d -> %x:%d)\n", f->src_ip.s_addr, f->src_port, f->dst_ip.s_addr, f->dst_port);
+		printf("session id: \n");
+		for(int i=0; i< new_session->session_id_len; i++){
+			printf("%02x ", p[i]);
+		}
+		printf("\n");
 
 		f->current_session = new_session;
 	}
@@ -646,12 +682,19 @@ int save_session_id(flow *f, uint8_t *hs){
 
 	if(sessions->first_session == NULL){
 		sessions->first_session = new_session;
+		printf("First session id (%p->%p):", sessions, sessions->first_session);
+		for(int i=0; i< new_session->session_id_len; i++){
+			printf(" %02x", sessions->first_session->session_id[i]);
+		}
+		printf("\n");
 	} else {
 		session *last = sessions->first_session;
 
-		for(int i=0; i< sessions->length; i++){
-			if(last == NULL)
+		for(int i=0; i< sessions->length -1; i++){
+			if(last == NULL){
 				printf("UH OH: last is null?\n");
+				fflush(stdout);
+			}
 			last = last->next;
 		}
 		last->next = new_session;
@@ -661,7 +704,7 @@ int save_session_id(flow *f, uint8_t *hs){
 
 	printf("Saved session id:");
 	for(int i=0; i< new_session->session_id_len; i++){
-		printf(" %02x", p[i]);
+		printf(" %02x", new_session->session_id[i]);
 	}
 	printf("\n");
 
@@ -682,13 +725,18 @@ int save_session_id(flow *f, uint8_t *hs){
  *  	0 if success, 1 if failed
  */
 int save_session_ticket(flow *f, uint8_t *hs, uint32_t len){
+	printf("TICKET HDR:");
+	for(int i=0; i< HANDSHAKE_HEADER_LEN; i++){
+		printf("%02x ", hs[i]);
+	}
+	printf("\n");
 	uint8_t *p = hs + HANDSHAKE_HEADER_LEN;
 	p += 4; //skip lifetime TODO: add to session struct
 	session *new_session = calloc(1,sizeof(session));
 	new_session->session_id_len = 0;
 	
 	new_session->session_ticket_len = (p[0] << 8) + p[1];
-	printf("saving ticket of size %d\n", new_session->session_ticket_len);
+	printf("saving ticket of size %d (msg size %d)\n", new_session->session_ticket_len, len);
 	fflush(stdout);
 	p += 2;
 
@@ -741,7 +789,7 @@ int add_packet(flow *f, struct packet_info *info){
 	}
 
 	packet *new_packet = calloc(1, sizeof(packet));
-	new_packet->seq_num = htonl(info->tcp_hdr->sequence_num);
+	new_packet->seq_num = ntohl(info->tcp_hdr->sequence_num);
 	new_packet->len = info->app_data_len;
 
 	uint8_t *packet_data = calloc(1, new_packet->len);

server/flow.h

@@ -69,7 +69,8 @@ typedef struct flow_st {
 	struct in_addr src_ip, dst_ip; /* Source (client) and Destination (server) addresses */
 	uint16_t src_port, dst_port;	/* Source and Destination ports */
 
-	uint32_t seq_num;		/* sequence number */
+	uint32_t upstream_seq_num;		/* sequence number */
+	uint32_t downstream_seq_num;		/* sequence number */
 
 	byte key[16];		/* negotiated key */
 	int state;		/* TLS handshake state */
@@ -85,6 +86,7 @@ typedef struct flow_st {
 	uint32_t upstream_remaining;
 	DH *dh;
 	EC_KEY *ecdh;
+	sem_t upstream_queue_lock;
 
 	const EVP_CIPHER *cipher;
 	const EVP_MD *message_digest;

server/relay.c

@@ -18,6 +18,8 @@
 #include "flow.h"
 #include "crypto.h"
 
+#define DEBUG
+
 /** Called when a TLS application record is received for a
  *  tagged flow. Upstream packets will be checked for covert
  *  requests to censored sites, downstream packets will be
@@ -37,10 +39,10 @@ int replace_packet(flow *f, struct packet_info *info){
 	}
 
 #ifdef DEBUG
-	fprintf(stderr,"Flow: %d > %d (%s)\n", info->ip_hdr->src.s_addr, info->ip_hdr->dst.s_addr, (info->ip_hdr->src.s_addr != f->src_ip.s_addr)? "incoming":"outgoing");
-	fprintf(stderr,"ID number: %u\n", htonl(info->ip_hdr->id));
-	fprintf(stderr,"Sequence number: %u\n", htonl(info->tcp_hdr->sequence_num));
-	fprintf(stderr,"Acknowledgement number: %u\n", htonl(info->tcp_hdr->ack_num));
+	fprintf(stdout,"Flow: %d > %d (%s)\n", info->ip_hdr->src.s_addr, info->ip_hdr->dst.s_addr, (info->ip_hdr->src.s_addr != f->src_ip.s_addr)? "incoming":"outgoing");
+	fprintf(stdout,"ID number: %u\n", htonl(info->ip_hdr->id));
+	fprintf(stdout,"Sequence number: %u\n", htonl(info->tcp_hdr->sequence_num));
+	fprintf(stdout,"Acknowledgement number: %u\n", htonl(info->tcp_hdr->ack_num));
 #endif
 
 	if(info->app_data_len <= 0){
@@ -54,13 +56,13 @@ int replace_packet(flow *f, struct packet_info *info){
 	} else {
 
 #ifdef DEBUG
-		printf("Current sequence number: %d\n", f->seq_num);
-		printf("Received sequence number: %d\n", htonl(tcp_hdr->sequence_num));
+		printf("Current sequence number: %d\n", f->downstream_seq_num);
+		printf("Received sequence number: %d\n", htonl(info->tcp_hdr->sequence_num));
 #endif
 
-		uint32_t offset = htonl(info->tcp_hdr->sequence_num) - f->seq_num;
+		uint32_t offset = htonl(info->tcp_hdr->sequence_num) - f->downstream_seq_num;
 		if(offset == 0)
-			f->seq_num += info->app_data_len;
+			f->downstream_seq_num += info->app_data_len;
 
 		/* if incoming, replace with data from queue */
 		//if(htonl(tcp_hdr->sequence_num) >= f->seq_num){
@@ -68,11 +70,11 @@ int replace_packet(flow *f, struct packet_info *info){
 		//}//TODO: need to do something about replaying packets (maybe store previously sent data??
 
 
-#ifdef DEBUG //TODO: fix
+#ifdef DEBUG2 //TODO: fix
 		uint8_t *p = (uint8_t *) info->tcp_hdr;
 		fprintf(stdout, "ip hdr length: %d\n", htons(info->ip_hdr->len));
 		fprintf(stdout, "Injecting the following packet:\n");
-		for(int i=0; i< htons(info->ip_hdr->len); i++){
+		for(int i=0; i< htons(info->ip_hdr->len)-1; i++){
 			fprintf(stdout, "%02x ", p[i]);
 		}
 		fprintf(stdout, "\n");
@@ -138,7 +140,6 @@ int read_header(flow *f, struct packet_info *info){
 
 		} else {
 			//process what we have
-		printf("US: %d bytes of packet unused\n", info->app_data_len - f->upstream_remaining);
 			record_hdr = (struct record_header*) f->upstream_queue->data;
 			record_length = RECORD_LEN(record_hdr);
 			record_ptr = calloc(1, record_length+ RECORD_HEADER_LEN);
@@ -153,6 +154,7 @@ int read_header(flow *f, struct packet_info *info){
 			}
 			memcpy(record_ptr+offset, p, f->upstream_remaining);
 			p = record_ptr;
+			record_hdr = (struct record_header*) p;
 			f->upstream_remaining = 0;
 		}
 	} else {
@@ -166,7 +168,7 @@ int read_header(flow *f, struct packet_info *info){
 			uint8_t *block_data = calloc(1, info->app_data_len);
 			memcpy(block_data, p, info->app_data_len);
 
-			new_block->len = info->app_data_len - RECORD_HEADER_LEN;
+			new_block->len = info->app_data_len;
 			new_block->offset = record_length; //re-appropriate this for len of record
 			new_block->data = block_data;
 			new_block->next = NULL;
@@ -209,16 +211,17 @@ int read_header(flow *f, struct packet_info *info){
 	}
 
 #ifdef DEBUG
-	printf("Upstream data:\n");
+	printf("Upstream data: (%x:%d > %x:%d )\n",info->ip_hdr->src.s_addr,info->tcp_hdr->src_port, info->ip_hdr->dst.s_addr, info->tcp_hdr->dst_port);
 	printf("%s\n", decrypted_data+EVP_GCM_TLS_EXPLICIT_IV_LEN);
 #endif
 
-	//TODO: re-write this to take a SOCKS connection request
 	/* search through decrypted data for x-ignore */
-	char *header_ptr = strstr((const char *) decrypted_data, "X-Slitheen");
+	char *header_ptr = strstr((const char *) decrypted_data+EVP_GCM_TLS_EXPLICIT_IV_LEN, "X-Slitheen");
 
 	uint8_t *upstream_data;
 	if(header_ptr == NULL){
+		printf("Slitheen header not found(%x:%d > %x:%d) \n",info->ip_hdr->src.s_addr,info->tcp_hdr->src_port, info->ip_hdr->dst.s_addr, info->tcp_hdr->dst_port);
+		fflush(stdout);
 		if(record_ptr != NULL)
 			free(record_ptr);
 		free(decrypted_data);
@@ -228,8 +231,9 @@ int read_header(flow *f, struct packet_info *info){
 
 #ifdef DEBUG
 	printf("UPSTREAM: Found x-slitheen header\n");
-	fprintf(stdout,"UPSTREAM Flow: %x > %x (%s)\n", info->ip_hdr->src.s_addr, info->ip_hdr->dst.s_addr, (info->ip_hdr->src.s_addr != f->src_ip.s_addr)? "incoming":"outgoing");
-	fprintf(stdout, "Sequence number: %d\n", info->tcp_hdr->sequence_num);
+	fflush(stdout);
+	fprintf(stdout,"UPSTREAM Flow: %x:%d > %x:%d (%s)\n", info->ip_hdr->src.s_addr,info->tcp_hdr->src_port, info->ip_hdr->dst.s_addr, info->tcp_hdr->dst_port,(info->ip_hdr->src.s_addr != f->src_ip.s_addr)? "incoming":"outgoing");
+	fprintf(stdout, "Sequence number: %d\n", ntohs(info->tcp_hdr->sequence_num));
 #endif
 
 	header_ptr += strlen("X-Slitheen: ");
@@ -247,8 +251,10 @@ int read_header(flow *f, struct packet_info *info){
 	}
 	c++;
 	*c = '\0';
+	printf("UPSTREAM: Found %d messages\n", num_messages);
 
 	for(int i=0; i< num_messages-1; i++){
+		printf("Decoding message: %s\n", messages[i]);
 		char *message = messages[i];
 
 		//b64 decode the data
@@ -272,8 +278,6 @@ int read_header(flow *f, struct packet_info *info){
 		int32_t output_len = BIO_read(bio, upstream_data, strlen(message));
 
 		BIO_free_all(bio);
-		if(output_len != decode_len)
-			printf("UH OH, lens dont match\n. %d != %d", decode_len, output_len);
 
 #ifdef DEBUG
 		printf("Decoded to get %d bytes:\n", output_len);
@@ -297,38 +301,50 @@ int read_header(flow *f, struct packet_info *info){
 
 			//If a thread for this stream id exists, get the thread info and pipe data
 			int32_t stream_pipe = -1;
+			stream *last = streams->first;
+			stream *prev = last;
 			if(streams->first != NULL){
-				stream *last = streams->first;
 				if(last->stream_id == stream_id){
 					stream_pipe = last->pipefd;
-				}
-				while(last->next != NULL){
-					last = last->next;
-					if(last->stream_id == stream_id){
-						stream_pipe = last->pipefd;
-						break;
+				} else {
+					while(last->next != NULL){
+						prev = last;
+						last = last->next;
+						if(last->stream_id == stream_id){
+							stream_pipe = last->pipefd;
+							break;
+						}
 					}
 				}
 			}
 
 			if(stream_pipe != -1){
-				int32_t bytes_sent = write(stream_pipe, p, stream_len);
-				if(bytes_sent < 0){
-					printf("Error writing to stream pipe\n");
-				fflush(stdout);
+				//check to see if this is a close message
+				if(stream_len == 0){
+					close(stream_pipe);
+					//remove from stream id table
+					if(last == streams->first){
+						streams->first = last->next;
+						printf("streams->first is now %p\n", streams->first);
+					} else {
+						prev->next = last->next;
+						printf("prev->next is now %p\n", prev->next);
+					}
+					printf("freeing %p\n", last);
+					free(last);
+					break;
 				}
+				int32_t bytes_sent = write(stream_pipe, p, stream_len);
 
-			} else {
+			} else if(stream_len > 0){
 
 				/*Else, spawn a thread to handle the proxy to this site*/
-				pthread_t *proxy_thread = calloc(1, sizeof(pthread_t));
+				pthread_t proxy_thread;
 				int32_t pipefd[2];
 				if(pipe(pipefd) < 0){
-					printf("Failed to create pipe for new thread\n");
 					free(decrypted_data);
 					if(record_ptr != NULL)
 						free(record_ptr);
-
 					return 1;
 				}
 				uint8_t *initial_data = calloc(1,stream_len);
@@ -341,12 +357,15 @@ int read_header(flow *f, struct packet_info *info){
 				thread_data->stream_id = stream_id;
 				thread_data->pipefd = pipefd[0];
 				
-				pthread_create(proxy_thread, NULL, proxy_covert_site, (void *) thread_data);
+				
+				pthread_create(&proxy_thread, NULL, proxy_covert_site, (void *) thread_data);
 
+				pthread_detach(proxy_thread);
 				//add stream to table
 				stream *new_stream = calloc(1, sizeof(stream));
 				new_stream->stream_id = stream_id;
 				new_stream->pipefd = pipefd[1];
+				new_stream->next = NULL;
 
 				if(streams->first == NULL){
 					streams->first = new_stream;
@@ -358,6 +377,8 @@ int read_header(flow *f, struct packet_info *info){
 					last->next = new_stream;
 				}
 
+			} else{
+				break;
 			}
 			output_len -= stream_len;
 			p += stream_len;
@@ -401,6 +422,8 @@ void *proxy_covert_site(void *data){
 	struct socks_req *clnt_req = (struct socks_req *) p;
 	p += 4;
 
+	int32_t handle = -1;
+
 	//see if it's a connect request
 	if(clnt_req->cmd != 0x01){
 		goto err;
@@ -440,7 +463,7 @@ void *proxy_covert_site(void *data){
 	//now set the port
 	dest.sin_port = *((uint16_t *) p);
 
-    int32_t handle = socket(AF_INET, SOCK_STREAM, 0);
+    handle = socket(AF_INET, SOCK_STREAM, 0);
     if(handle < 0){
 		goto err;
     }
@@ -467,7 +490,7 @@ void *proxy_covert_site(void *data){
 
 #ifdef DEBUG
 	printf("Bound to %x:%d\n", my_addr.sin_addr.s_addr, ntohs(my_addr.sin_port));
-	printf("Downstream response:\n");
+	printf("Downstream response (id %d):\n", stream_id);
 	for(int i=0; i< 10; i++){
 		printf("%02x ", response[i]);
 	}
@@ -519,17 +542,25 @@ void *proxy_covert_site(void *data){
 			int32_t bytes_read = read(thread_data->pipefd, buffer, buffer_len);
 
 			if(bytes_read > 0){
+#ifdef DEBUG
+				printf("PROXY (id %d): read %d bytes from pipe\n", stream_id, bytes_read);
+				for(int i=0; i< bytes_read; i++){
+					printf("%02x ", buffer[i]);
+				}
+				printf("\n");
+				printf("%s\n", buffer);
+#endif
 				int32_t bytes_sent = send(handle, buffer,
 						bytes_read, 0);
 				if( bytes_sent <= 0){
-					printf("error sending request\n");
 					break;
 				} else if (bytes_sent < bytes_read){
 					//TODO: should update buffer and keep
 					//track of length of upstream data
-					printf("sent less than full upstream bytes\n");
 					break;
 				}
+			} else {
+				break;
 			}
 
 		}
@@ -538,12 +569,16 @@ void *proxy_covert_site(void *data){
 			//we have downstream data read for saving
 			int32_t bytes_read;
 			bytes_read = recv(handle, buffer, buffer_len, 0);
-			if(bytes_read <= 0){
-				break;
-			}
 			if(bytes_read > 0){
 				uint8_t *new_data = calloc(1, bytes_read);
 				memcpy(new_data, buffer, bytes_read);
+#ifdef DEBUG
+				printf("PROXY (id %d): read %d bytes from censored site\n",stream_id, bytes_read);
+				for(int i=0; i< bytes_read; i++){
+					printf("%02x ", buffer[i]);
+				}
+				printf("\n");
+#endif
 
 				//make a new queue block
 				new_block = calloc(1, sizeof(queue_block));
@@ -562,21 +597,69 @@ void *proxy_covert_site(void *data){
 					last->next = new_block;
 				}
 			} else {
-				printf("read 0 bytes\n");
+				
 				break;
 			}
 
 		}
 	}
+	//remove self from list 
+	stream *last = streams->first;
+	stream *prev = last;
+	if(streams->first != NULL){
+		if(last->stream_id == stream_id){
+			streams->first = last->next;
+			printf("Freeing (2) %p\n", last);
+			free(last);
+		} else {
+			while(last->next != NULL){
+				prev = last;
+				last = last->next;
+				if(last->stream_id == stream_id){
+					prev->next = last->next;
+					printf("Freeing (2) %p\n", last);
+					free(last);
+					break;
+				}
+			}
+		}
+	}
+	if(thread_data->initial_data != NULL){
+		free(thread_data->initial_data);
+	}
 	free(thread_data);
 	free(buffer);
 	close(handle);
+	pthread_exit(NULL);
 	return 0;
 err:
+	//remove self from list 
+	last = streams->first;
+	prev = last;
+	if(streams->first != NULL){
+		if(last->stream_id == stream_id){
+			streams->first = last->next;
+			free(last);
+		} else {
+			while(last->next != NULL){
+				prev = last;
+				last = last->next;
+				if(last->stream_id == stream_id){
+					prev->next = last->next;
+					free(last);
+					break;
+				}
+			}
+		}
+	}
 	if(thread_data->initial_data != NULL){
 		free(thread_data->initial_data);
 	}
-	close(handle);
+	free(thread_data);
+	if(handle > 0){
+		close(handle);
+	}
+	pthread_exit(NULL);
 	return 0;
 }
 
@@ -602,6 +685,7 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 	uint8_t *p = info->app_data;
 	uint32_t remaining_packet_len = info->app_data_len;
 
+
 	if(f->remaining_record_len > 0){
 		//ignore bytes until the end of the record
 		if(f->remaining_record_len > remaining_packet_len){ //ignore entire packet
@@ -632,12 +716,14 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 
 	while(remaining_packet_len > 0){ //while bytes remain in the packet
 		if(remaining_packet_len < RECORD_HEADER_LEN){
-			printf("Error: partial record header: \n");
+#ifdef DEBUG
+			printf("partial record header: \n");
 			for(int i= 0; i< remaining_packet_len; i++){
 				printf("%02x ", p[i]);
 			}
 			printf("\n");
 			fflush(stdout);
+#endif
 			f->partial_record_header = calloc(1, RECORD_HEADER_LEN);
 			memcpy(f->partial_record_header, p, remaining_packet_len);
 			f->partial_record_header_len = remaining_packet_len;
@@ -651,14 +737,13 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 			memcpy(f->partial_record_header+ f->partial_record_header_len, 
 					p, RECORD_HEADER_LEN - f->partial_record_header_len);
 			record_hdr = (struct record_header *) f->partial_record_header;
-			printf("Using partial record\n");
 		} else {
 		
 			record_hdr = (struct record_header*) p;
 		}
 		uint32_t record_len = RECORD_LEN(record_hdr);
 
-//#ifdef DEBUG
+#ifdef DEBUG
 	fprintf(stdout,"Flow: %x > %x (%s)\n", info->ip_hdr->src.s_addr, info->ip_hdr->dst.s_addr, (info->ip_hdr->src.s_addr != f->src_ip.s_addr)? "incoming":"outgoing");
 	fprintf(stdout,"ID number: %u\n", htonl(info->ip_hdr->id));
 	fprintf(stdout,"Sequence number: %u\n", htonl(info->tcp_hdr->sequence_num));
@@ -668,7 +753,7 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 			printf("%02x ", ((uint8_t *) record_hdr)[i]);
 		}
 		printf("\n");
-//#endif
+#endif
 
 		p += (RECORD_HEADER_LEN - f->partial_record_header_len);
 		remaining_packet_len -= (RECORD_HEADER_LEN - f->partial_record_header_len);
@@ -720,6 +805,10 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 				}
 			}
 			remaining_packet_len -= remaining_packet_len;
+			if(f->partial_record_header_len > 0){
+				f->partial_record_header_len = 0;
+				free(f->partial_record_header);
+			}
 			break;
 		}
 
@@ -729,6 +818,10 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 						record_hdr->type, 0);
 		if(n < 0){
 			//do something smarter here
+			if(f->partial_record_header_len > 0){
+				f->partial_record_header_len = 0;
+				free(f->partial_record_header);
+			}
 			return 0;
 		}
 		changed = 1;
@@ -773,6 +866,7 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 					if(len_ptr != NULL){
 						if(!memcmp(len_ptr + 19, "chunked", 7)){
 							//now find end of header
+							//printf("chunked encoding\n");
 							
 							len_ptr = strstr((const char *) p, "\r\n\r\n");
 							if(len_ptr != NULL){
@@ -786,6 +880,7 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 						if(len_ptr != NULL){
 							len_ptr += 15;
 							f->remaining_response_len = strtol((const char *) len_ptr, NULL, 10);
+							//printf("content-length: %d\n", f->remaining_response_len);
 							len_ptr = strstr((const char *) p, "\r\n\r\n");
 							if(len_ptr != NULL){
 								f->httpstate = MID_CONTENT;
@@ -793,9 +888,11 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 								p = (uint8_t *) len_ptr + 4;
 							} else {
 								remaining_record_len = 0;
+								//printf("Missing end of header. Sending to FORFEIT_REST\n");
 								f->httpstate = FORFEIT_REST;
 							}
 						} else {
+							//printf("No content length of transfer encoding field, sending to FORFEIT_REST\n");
 							f->httpstate = FORFEIT_REST;
 							remaining_record_len = 0;
 						}
@@ -856,6 +953,7 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 						p = (uint8_t *) needle + 2;
 					} else {
 						remaining_record_len = 0;
+						printf("Couldn't find chunk, sending to FORFEIT_REST\n");
 						f->httpstate = FORFEIT_REST;
 					}
 					}
@@ -880,11 +978,11 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 						remaining_record_len = 0;
 					} else {
 						if(f->replace_response){
-							fill_with_downstream(p, remaining_record_len);
+							fill_with_downstream(p, f->remaining_response_len);
 
 #ifdef DEBUG
 							printf("Replaced with:\n");
-							for(int i=0; i< remaining_record_len; i++){
+							for(int i=0; i< f->remaining_response_len; i++){
 								printf("%02x ", p[i]);
 							}
 							printf("\n");
@@ -892,6 +990,7 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 						}
 						remaining_record_len -= f->remaining_response_len;
 						p += f->remaining_response_len;
+						f->remaining_response_len = 0;
 						f->httpstate = END_CHUNK;
 					}
 					break;
@@ -904,6 +1003,7 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 						remaining_record_len -= 2;
 					} else {
 						remaining_record_len = 0;
+						//printf("Couldn't find end of chunk, sending to FORFEIT_REST\n");
 						f->httpstate = FORFEIT_REST;
 					}
 					break;
@@ -916,6 +1016,7 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 						remaining_record_len -= 2;
 					} else {
 						remaining_record_len = 0;
+						//printf("Couldn't find end of body, sending to FORFEIT_REST\n");
 						f->httpstate = FORFEIT_REST;
 					}
 					break;
@@ -935,6 +1036,10 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 		if((n = encrypt(f, record_ptr, record_ptr,
 						n + EVP_GCM_TLS_EXPLICIT_IV_LEN, 1, record_hdr->type,
 						1)) < 0){
+			if(f->partial_record_header_len > 0){
+				f->partial_record_header_len = 0;
+				free(f->partial_record_header);
+			}
 			return 0;
 		}
 
@@ -943,7 +1048,6 @@ int process_downstream(flow *f, int32_t offset, struct packet_info *info){
 		if(f->partial_record_header_len > 0){
 			f->partial_record_header_len = 0;
 			free(f->partial_record_header);
-			printf("freed partial record\n");
 		}
 
 	}
@@ -1011,13 +1115,18 @@ int fill_with_downstream(uint8_t *data, int32_t length){
 		}
 		sl_hdr->len = htons(sl_hdr->len);
 
-#ifdef DEBUG
+//#ifdef DEBUG
 		printf("DWNSTRM: slitheen header: ");
 		for(int i=0; i< SLITHEEN_HEADER_LEN; i++){
 			printf("%02x ",((uint8_t *) sl_hdr)[i]);
 		}
 		printf("\n");
-#endif
+		printf("Sending %d downstream bytes:", ntohs(sl_hdr->len));
+		for(int i=0; i< ntohs(sl_hdr->len); i++){
+			printf("%02x ", ((uint8_t *) sl_hdr)[i+SLITHEEN_HEADER_LEN]);
+		}
+		printf("\n");
+//#endif
 	}
 	//now, if we need more data, fill with garbage
 	if(remaining > SLITHEEN_HEADER_LEN ){
@@ -1028,13 +1137,13 @@ int fill_with_downstream(uint8_t *data, int32_t length){
 		sl_hdr->len = htons(remaining);
 		sl_hdr->garbage = htons(remaining);
 
-#ifdef DEBUG
+//#ifdef DEBUG
 		printf("DWNSTRM: slitheen header: ");
 		for(int i=0; i< SLITHEEN_HEADER_LEN; i++){
 			printf("%02x ", p[i]);
 		}
 		printf("\n");
-#endif
+//#endif
 
 		p += SLITHEEN_HEADER_LEN;
 		memset(p, 'A', remaining);

server/slitheen-proxy.c

@@ -9,6 +9,7 @@
 #include "slitheen.h"
 #include "relay.h"
 #include "crypto.h"
+#include "cryptothread.h"
 
 void usage(void){
 	printf("Usage: slitheen-proxy [internal network interface] [NAT interface]\n");
@@ -16,8 +17,8 @@ void usage(void){
 
 int main(int argc, char *argv[]){
 	pthread_t t1, t2;
-	char filter1[33] = "ether src host 08:00:27:0e:89:ea";
-	char filter2[33] = "ether src host 08:00:27:0e:89:ea";
+	char *filter1 = calloc(1, 33);
+	char *filter2 = calloc(1, 33);
 
 	char *dev1 = NULL; /* Device that leads to the internal network */
 	char *dev2 = NULL; /* Device that leads out to the world */
@@ -32,11 +33,12 @@ int main(int argc, char *argv[]){
 	dev1 = argv[1];
 	dev2 = argv[2];
 
-	snprintf(filter1, 33, "ether src host %s", macaddr);
-	snprintf(filter2, 33, "ether dst host %s", macaddr);
+	snprintf(filter1, 33, "ether src host %s", macaddr1);
+	snprintf(filter2, 33, "ether src host %s", macaddr2);
 
 	init_tables();
 	init_session_cache();
+	init_crypto_locks();
 
 	/* Create threads */
 	outbound.readdev = dev1;
@@ -52,6 +54,10 @@ int main(int argc, char *argv[]){
 	pthread_join(t2, NULL);
 
 	pthread_exit(NULL);
+	free(filter1);
+	free(filter2);
+
+	crypto_locks_cleanup();
 
 	return(0);
 }
@@ -153,7 +159,8 @@ void process_packet(struct packet_info *info){
 	newFlow.src_port = info->tcp_hdr->src_port;
 	newFlow.dst_port = info->tcp_hdr->dst_port;
 
-	newFlow.seq_num = info->tcp_hdr->sequence_num;
+	newFlow.upstream_seq_num = ntohl(info->tcp_hdr->sequence_num);
+	newFlow.downstream_seq_num = 0;
 
 	/* Checks to see if this is a possibly tagged hello msg */
 	if ((info->record_hdr != NULL) && (info->record_hdr->type == HS)){ /* This is a TLS handshake */
@@ -167,9 +174,15 @@ void process_packet(struct packet_info *info){
 		if(observed->application){
 			replace_packet(observed, info);
 		} else {
+			int32_t incoming = 
+			(info->ip_hdr->src.s_addr == observed->src_ip.s_addr) ? 0 : 1;
 
 			/* Pass data to packet chain */
-			add_packet(observed, info);
+			if(incoming && (observed->downstream_seq_num <= htonl(info->tcp_hdr->sequence_num))){
+				add_packet(observed, info);
+			} else if ((!incoming) && (observed->upstream_seq_num <= htonl(info->tcp_hdr->sequence_num))){
+				add_packet(observed, info);
+			}
 
 			/* Update flow state */
 			if(observed->packet_chain != NULL){

server/slitheen.h

@@ -4,7 +4,11 @@
 #include <netinet/in.h>
 #include <pcap.h>
 
-#define macaddr "08:00:27:0e:89:ea"
+#define macaddr1 "00:25:90:5a:26:99"
+#define macaddr2 "00:25:90:c9:5a:09"
+
+//#define macaddr1 "08:00:27:0e:89:ea"
+//#define macaddr2 "08:00:27:0e:89:ea"
 
 /* Ethernet addresses are 6 bytes */
 #define ETHER_ADDR_LEN	6