added code to delay injection of misordered packets during handshake

relay_station/flow.c

@@ -79,6 +79,11 @@ flow *add_flow(struct packet_info *info) {
 	new_flow->upstream_seq_num = ntohl(info->tcp_hdr->sequence_num);
 	new_flow->downstream_seq_num = ntohl(info->tcp_hdr->ack_num);
 
+    new_flow->us_frame_queue = emalloc(sizeof(frame_queue));
+    new_flow->us_frame_queue->first_frame = NULL;
+    new_flow->ds_frame_queue = emalloc(sizeof(frame_queue));
+    new_flow->ds_frame_queue->first_frame = NULL;
+
 	new_flow->streams=NULL;
 	new_flow->downstream_queue=NULL;
 	new_flow->client_ptr=NULL;
@@ -469,6 +474,24 @@ int remove_flow(flow *f) {
 	if(f->removed)
 		printf("Trying again to free\n");
 
+    frame *first_frame = f->us_frame_queue->first_frame;
+    while(first_frame != NULL){
+        inject_packet(first_frame->handle, first_frame->header, first_frame->packet);
+        frame *tmp = first_frame->next;
+        free(first_frame);
+        first_frame = tmp;
+    }
+    free(f->us_frame_queue);
+
+    first_frame = f->ds_frame_queue->first_frame;
+    while(first_frame != NULL){
+        inject_packet(first_frame->handle, first_frame->header, first_frame->packet);
+        frame *tmp = first_frame->next;
+        free(first_frame);
+        first_frame = tmp;
+    }
+    free(f->ds_frame_queue);
+
 	//Empty application data queues
 	packet *tmp = f->upstream_app_data->first_packet;
 	while(tmp != NULL){
@@ -1007,8 +1030,12 @@ int save_session_ticket(flow *f, uint8_t *hs, uint32_t len){
 	return 0;
 }
 
-/* Adds a packet the flow's packet chain. If it can complete a record, gives
- * this record to update_flow */
+/* Adds a (handshake) packet to the flow's packet chain. If it can complete a record, passes
+ * this record to update_flow
+ *
+ * Note: the code in slitheen-proxy.c should ensure that this function only ever gets the next
+ * expected sequence number
+ */
 int add_packet(flow *f, struct packet_info *info){
 	if (info->tcp_hdr == NULL || info->app_data_len <= 0){
 		return 0;
@@ -1030,13 +1057,20 @@ int add_packet(flow *f, struct packet_info *info){
 
 	if(new_packet->seq_num < chain->expected_seq_num){
 		//see if this packet contains any data we are missing
-		printf("Received replayed packet O.o\n");
+        //TODO: figure out how/why this happens and what should follow
+		printf("ERROR: Received replayed packet O.o\n");
+
 		free(new_packet->data);
 		free(new_packet);
 
-	} else {//new_packet->seq_num >= chain->expected_seq_num
+    } else {//new_packet->seq_num >= chain->expected_seq_num
+        
+        if(new_packet->seq_num > chain->expected_seq_num) {
+            printf("ERROR: Received future packet O.o\n");
+        }
 	
 		//Find appropriate place in chain
+        //TODO: this can be simplified; slitheen-proxy code already takes care of it
 		packet *previous = NULL;
 		packet *next = chain->first_packet;
 		while(next != NULL && (next->seq_num <= new_packet->seq_num)){
@@ -1135,7 +1169,7 @@ int add_packet(flow *f, struct packet_info *info){
 							return 1;//error occurred and flow was removed
 						}
 
-						//check to see if last finished message received
+						//check to see if server finished message received
 						if(f->in_encrypted ==2){
 
 #ifdef DEBUG
@@ -1192,7 +1226,7 @@ int add_packet(flow *f, struct packet_info *info){
 		
 		} else {//
 			//add to end of packet_chain
-			//printf("Missing packet (expected %d, received %d)\n", chain->expected_seq_num, new_packet->seq_num);
+			printf("Missing packet (expected %d, received %d)\n", chain->expected_seq_num, new_packet->seq_num);
 		}
 	}
 	return 0;

relay_station/flow.h

@@ -67,6 +67,18 @@ typedef struct app_data_queue_st {
 	packet *first_packet;
 } app_data_queue;
 
+typedef struct frame_st {
+    uint8_t *packet;
+    const struct pcap_pkthdr *header;
+    pcap_t *handle;
+	uint32_t seq_num;
+    struct frame_st *next;
+} frame;
+
+typedef struct frame_queue_st {
+    frame *first_frame;
+} frame_queue;
+
 typedef struct session_st {
 	uint8_t session_id_len;
 	uint8_t session_id[SSL_MAX_SSL_SESSION_ID_LENGTH];
@@ -98,6 +110,9 @@ typedef struct flow_st {
 	app_data_queue *upstream_app_data;	/* Saved application-layer data for packet retransmits */
 	app_data_queue *downstream_app_data;
 
+    frame_queue *us_frame_queue; /*Held misordered Ethernet frames to be processed and written out later */
+    frame_queue *ds_frame_queue; /*Held misordered Ethernet frames to be processed and written out later */
+
 	byte key[16];		/* negotiated key */
 	int state;		/* TLS handshake state */
 	int in_encrypted;		/* indicates whether incoming flow is encrypted */

relay_station/slitheen-proxy.c

@@ -121,6 +121,28 @@ void *sniff_packets(void *args){
 	return NULL;
 }
 
+
+/*
+ * Injects a packet back out the opposite interface
+ */
+void inject_packet(pcap_t *handle, const struct pcap_pkthdr *header, uint8_t *packet){
+
+
+	if((pcap_inject(handle, packet, header->len)) < 0 ){
+		fprintf(stderr, "Error: %s\n", pcap_geterr(handle));
+	}
+
+#ifdef DEBUG
+	fprintf(stderr, "injected the following packet:\n");
+	for(int i=0; i< header->len; i++){
+		fprintf(stderr, "%02x ", packet[i]);
+	}
+	fprintf(stderr, "\n");
+
+#endif
+	free(packet);
+}
+
 /**
  * Runs when pcap_loop receives a packet from the specified interface
  * If the received packet is a tcp packet, processes it and then writes it back out
@@ -130,39 +152,12 @@ void *sniff_packets(void *args){
 void got_packet(uint8_t *args, const struct pcap_pkthdr *header, const uint8_t *packet){
 	pcap_t *handle = (pcap_t *) args;
 
-	struct packet_info *info = emalloc(sizeof(struct packet_info));
 	uint8_t *tmp_packet = emalloc(header->len);
-	//printf("Allocated %d bytes to %p\n", header->len, tmp_packet);
 	memcpy(tmp_packet, packet, header->len);
-	extract_packet_headers(tmp_packet, info);
 
-	// Check to make sure it is a TCP packet 
-	if((info->ip_hdr == NULL) || (info->tcp_hdr == NULL))
-		goto end;
+	process_packet(handle, header, tmp_packet);
 
-	process_packet(info);
-
-end:
-	if((pcap_inject(handle, tmp_packet, header->len)) < 0 ){
-		fprintf(stderr, "Error: %s\n", pcap_geterr(handle));
-	}
-#ifdef DEBUG
-	fprintf(stderr, "injected the following packet:\n");
-	for(int i=0; i< header->len; i++){
-		fprintf(stderr, "%02x ", tmp_packet[i]);
-	}
-	fprintf(stderr, "\n");
 
-	if((info->tcp_hdr != NULL) && (info->ip_hdr != NULL)){
-	fprintf(stdout,"Injected packet: %x:%d > %x:%d\n", info->ip_hdr->src.s_addr, ntohs(info->tcp_hdr->src_port), info->ip_hdr->dst.s_addr, ntohs(info->tcp_hdr->dst_port));
-	fprintf(stdout,"ID number: %u\n", htonl(info->ip_hdr->id));
-	fprintf(stdout,"Sequence number: %u\n", htonl(info->tcp_hdr->sequence_num));
-	fprintf(stdout,"Acknowledgement number: %u\n", htonl(info->tcp_hdr->ack_num));
-	fflush(stdout);
-	}
-#endif
-	free(info);//Note: don't free this while a thread is using it
-	free(tmp_packet);
 
 }
 
@@ -171,7 +166,14 @@ end:
  * 	2) adds the packet to the flow's data chain
  * 	3) updates the flow's state
  */
-void process_packet(struct packet_info *info){
+void process_packet(pcap_t *handle, const struct pcap_pkthdr *header, uint8_t *packet){
+
+	struct packet_info *info = emalloc(sizeof(struct packet_info));
+	extract_packet_headers(packet, info);
+
+    //Ignore non-TCP packets (shouldn't actually get any)
+	if((info->ip_hdr == NULL) || (info->tcp_hdr == NULL))
+		goto err;
 
 	/* Checks to see if this is a possibly tagged hello msg */
 	if ((info->record_hdr != NULL) && (info->record_hdr->type == HS)){ /* This is a TLS handshake */
@@ -183,20 +185,20 @@ void process_packet(struct packet_info *info){
 	flow *observed;
 	if((observed = check_flow(info)) != NULL){
 	
-//#ifdef DEBUG
+#ifdef DEBUG
 		/*Check sequence number and replay application data if necessary*/
 		fprintf(stdout,"Flow: %x:%d > %x:%d (%s)\n", info->ip_hdr->src.s_addr, ntohs(info->tcp_hdr->src_port), info->ip_hdr->dst.s_addr, ntohs(info->tcp_hdr->dst_port), (info->ip_hdr->src.s_addr != observed->src_ip.s_addr)? "incoming":"outgoing");
 		fprintf(stdout,"ID number: %u\n", htonl(info->ip_hdr->id));
 		fprintf(stdout,"Sequence number: %u\n", htonl(info->tcp_hdr->sequence_num));
 		fprintf(stdout,"Acknowledgement number: %u\n", htonl(info->tcp_hdr->ack_num));
-//#endif
+#endif
 
 		uint8_t incoming = (info->ip_hdr->src.s_addr != observed->src_ip.s_addr)? 1 : 0;
 		uint32_t seq_num = htonl(info->tcp_hdr->sequence_num);
 		uint32_t expected_seq = (incoming)? observed->downstream_seq_num : observed->upstream_seq_num;
-//#ifdef DEBUG
+#ifdef DEBUG
 		fprintf(stdout,"Expected sequence number: %u\n", expected_seq);
-//#endif
+#endif
 
 		/* Remove acknowledged data from queue after TCP window is exceeded */
         update_window_expiration(observed, info);
@@ -234,20 +236,49 @@ void process_packet(struct packet_info *info){
                 if(seq_num > expected_seq){
                     //For now, enters into FORFEIT state
                     //TODO: change upstream behaviour to try to mask slitheen hdr
-                    printf("ERROR: future packet in app data, forfeiting flow\n");
+                    //printf("ERROR: future packet in app data, forfeiting flow\n");
                     remove_flow(observed);
-                    return;
+                    goto err;
                 }
 
 				replace_packet(observed, info);
 			} else {
+                //We're still in the TLS handshake; hold packets misordered packets
+
+                if(seq_num > expected_seq){
+                    //Delay and process later
+                    frame *new_frame = ecalloc(1, sizeof(frame));
+                    new_frame->handle = handle;
+                    new_frame->packet = packet;
+                    new_frame->header = header;
+                    new_frame->seq_num = seq_num;
+                    new_frame->next = NULL;
+                    frame_queue *queue = (incoming) ? observed->ds_frame_queue : observed->us_frame_queue;
+
+                    printf("Used frame queue\n");
+
+                    //add to end of list
+                    if(queue->first_frame == NULL){
+                        queue->first_frame = new_frame;
+                    } else {
+                        frame *last = queue->first_frame;
+                        while(last->next != NULL){
+                            last = last->next;
+                        }
+                        last->next = new_frame;
+                    }
+
+                    free(info);
+                    observed->ref_ctr--;
+                    return; //TODO: fix terrible spaghetti returns
+                }
 
 				/* Pass data to packet chain */
 				if(observed->stall){
 
 				}
 				if(add_packet(observed, info)){//removed_flow
-					return;
+					goto err;
 				}
 			}
 
@@ -255,20 +286,53 @@ void process_packet(struct packet_info *info){
 			if(info->tcp_hdr->flags & (FIN | RST) ){
 				/* Remove flow from table, connection ended */
 				remove_flow(observed);
-				return;
+				goto err;
 			}
 
 			/* add packet to application data queue */
             save_packet(observed, info);
-
 		}
+        
+
+        /*process and release held frames with current sequence numbers*/
+        frame_queue *queue = (incoming) ? observed->ds_frame_queue : observed->us_frame_queue;
+        frame *first = queue->first_frame;
+        frame *prev = queue->first_frame;
+		expected_seq = (incoming)? observed->downstream_seq_num : observed->upstream_seq_num;
+
+        while (first != NULL){
+
+            if(first->seq_num <= expected_seq){
+                //remove from queue and process
+                if(first == queue->first_frame) {
+                    queue->first_frame = first->next;
+                } else {
+                    prev->next = first->next;
+                }
+                process_packet(handle, first->header, first->packet);
+                free(first);
+                first = queue->first_frame;
+                prev = queue->first_frame;
+            } else {
+                prev = first;
+                first = first->next;
+            }
+        }
 
 		observed->ref_ctr--;
 	}
 
+err:
+	free(info);//Note: don't free this while a thread is using it
+
+    inject_packet(handle, header, packet);
+
+    return;
+
 
 }
 
+//TODO: rewrite this function to remove bloat
 void save_packet(flow *f, struct packet_info *info){
 
     uint8_t incoming = (info->ip_hdr->src.s_addr != f->src_ip.s_addr)? 1 : 0;
@@ -383,11 +447,11 @@ void update_window_expiration(flow *f, struct packet_info *info){
     uint32_t end_seq = htonl(info->tcp_hdr->sequence_num) + info->app_data_len - 1;
     uint32_t window = ack_num + htons(info->tcp_hdr->win_size);
 
-//#ifdef DEBUG
+#ifdef DEBUG
     printf("Received sequence number %u\n", htonl(info->tcp_hdr->sequence_num));
     printf("Acknowledged up to %u with window expiring at %u\n", ack_num, window);
     printf("Removing all packets up to %u\n", end_seq);
-//#endif
+#endif
 
     packet *saved_data = (incoming)? f->downstream_app_data->first_packet :
         f->upstream_app_data->first_packet;
@@ -441,8 +505,6 @@ void retransmit(flow *f, struct packet_info *info, uint32_t data_to_fill){
     packet *saved_data = (incoming)? f->downstream_app_data->first_packet :
         f->upstream_app_data->first_packet;
 
-    printf("Filling with %d retransmitted bytes\n", data_to_fill);
-
     while(data_to_fill > 0){
         if(saved_data == NULL){
             //have already acked all data

relay_station/slitheen.h

@@ -102,8 +102,9 @@ struct sniff_args {
 
 void got_packet(uint8_t *args, const struct pcap_pkthdr *header, const uint8_t *packet);
 void *sniff_packets(void *);
-void process_packet(struct packet_info *info);
+void process_packet(pcap_t *handle, const struct pcap_pkthdr *header, uint8_t *packet);
 void extract_packet_headers(uint8_t *packet, struct packet_info *info);
 struct packet_info *copy_packet_info(struct packet_info *src_info);
+void inject_packet(pcap_t *handle, const struct pcap_pkthdr *header, uint8_t *packet);
 
 #endif /* _SLITHEEN_H_ */