From a603a5762c0ccaa35ffb2a007ba62dc482cef269 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Fri, 9 Aug 2019 17:01:28 +0200
Subject: [PATCH 01/35] Refactor core

---
 CMakeLists.txt     |   2 +-
 src/algo_dup2.c    |   3 +-
 src/algo_naive.c   |   3 +-
 src/algo_rr.c      | 379 ---------------------------------------------
 src/algo_thunder.c |  40 +++++
 src/packet.c       |  80 +++++-----
 src/packet.h       |  31 ++--
 src/proxy.c        |   9 +-
 src/proxy.h        |  24 +--
 9 files changed, 122 insertions(+), 449 deletions(-)
 delete mode 100644 src/algo_rr.c
 create mode 100644 src/algo_thunder.c

diff --git a/CMakeLists.txt b/CMakeLists.txt
index d9c9490..e6661e6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -25,8 +25,8 @@ list(APPEND CSOURCES
 	src/url.c
 	src/donar_init.h
 	src/donar_init.c
-	src/algo_rr.c
 	src/algo_dup2.c
+	src/algo_thunder.c
 	src/algo_utils.h
 	src/algo_utils.c
 	src/proxy.h
diff --git a/src/algo_dup2.c b/src/algo_dup2.c
index 27fd1d9..e179f79 100644
--- a/src/algo_dup2.c
+++ b/src/algo_dup2.c
@@ -19,10 +19,11 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
   char url[256];
   struct evt_core_fdinfo *to_fdinfo = NULL;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
 
   // Check that we didn't already received the packet
   struct dup2_ctx* dup2c = app_ctx->misc;
-  if (ring_ge(dup2c->recv_id, bp->ip.ap.fmt.content.clear.id)) {
+  if (ring_ge(dup2c->recv_id, ap->fmt.content.clear.id)) {
     mv_buffer_rtof(&app_ctx->br, fdinfo);
     return 0;
   }
diff --git a/src/algo_naive.c b/src/algo_naive.c
index 6a7bac1..fb85873 100644
--- a/src/algo_naive.c
+++ b/src/algo_naive.c
@@ -9,9 +9,10 @@ int algo_naive_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinf
   char url[256];
   struct evt_core_fdinfo *to_fdinfo = NULL;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
 
   // 1. Find destination
-  sprintf(url, "udp:write:127.0.0.1:%d", bp->ip.ap.fmt.content.clear.port);
+  sprintf(url, "udp:write:127.0.0.1:%d", ap->fmt.content.udp_encapsulated.port);
   to_fdinfo = evt_core_get_from_url (ctx, url);
   if (to_fdinfo == NULL) {
     fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
diff --git a/src/algo_rr.c b/src/algo_rr.c
deleted file mode 100644
index 788d3cb..0000000
--- a/src/algo_rr.c
+++ /dev/null
@@ -1,379 +0,0 @@
-#include <sys/timerfd.h>
-#include "algo_utils.h"
-#include "utils.h"
-#include "url.h"
-#include "proxy.h"
-#include "timer.h"
-
-struct timer_info {
-  uint16_t health_id;
-  uint8_t prevlink;
-  uint16_t min_blocked_pkt;
-  struct algo_ctx* algo;
-};
-
-struct queued_pkt {
-  uint8_t on;
-  int link_fd;
-  int idx;
-  uint16_t id;
-  struct algo_ctx* algo;
-};
-
-struct rr_ctx {
-  uint8_t my_links;
-  uint8_t remote_links;
-  uint8_t current_link;
-  int64_t mjit;
-  uint16_t health_id;
-  uint16_t health_id_late;
-  uint16_t content_id;
-  uint16_t sent_health_id;
-  uint16_t sent_content_id;
-  struct internet_packet prev_packet;
-  struct timespec emit_time;
-  struct queued_pkt real[PACKET_BUFFER_SIZE];
-  struct timer_info wait[PACKET_BUFFER_SIZE];
-};
-
-void show_link_availability(struct rr_ctx* rr) {
-  printf("Links availability: my_links[");
-  for (int i = 0; i < 8; i++) {
-    if (rr->my_links & 1 << i) printf("U");
-    else printf("-");
-  }
-  printf("], rem_links[");
-  for (int i = 0; i < 8; i++) {
-    if (rr->remote_links & 1 << i) printf("U");
-    else printf("-");
-  }
-  printf("]\n");
-}
-
-void blacklist_link(struct rr_ctx* rr, int sel_link) {
-  printf("Blacklist link=%d | ", sel_link);
-  rr->remote_links &= 0xff ^ 1 << sel_link;
-  show_link_availability (rr);
-}
-
-void on_timeout_health (struct evt_core_ctx* ctx, void* user);
-
-void rr_pkt_register(struct evt_core_ctx* ctx,  struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
-  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
-  struct rr_ctx* rr = app_ctx->misc;
-  uint16_t real_idx = bp->ip.ap.fmt.content.clear.id % PACKET_BUFFER_SIZE;
-
-  assert(bp->ip.ap.fmt.headers.cmd == CMD_CLEAR);
-
-  // 1. We queue the packet to keep it
-  if (rr->real[real_idx].on && ring_lt(rr->real[real_idx].id, bp->ip.ap.fmt.content.clear.id)) {
-    fprintf(stderr, "Real array is full for packet_id=%d, idx=%d, last_delivered_content_id=%d BUG: [\n",
-            bp->ip.ap.fmt.content.clear.id,
-            real_idx,
-            rr->content_id);
-    for (int i = 0; i < PACKET_BUFFER_SIZE; i++) {
-      fprintf(stderr, "\t%d => %d\n", rr->real[i].id, rr->real[i].on);
-    }
-    fprintf(stderr, "] - could be replaced by drop\n");
-    exit(EXIT_FAILURE);
-  } else if (!rr->real[real_idx].on && ring_gt(bp->ip.ap.fmt.content.clear.id, rr->content_id)) {
-    rr->real[real_idx].on = 1;
-    rr->real[real_idx].id = bp->ip.ap.fmt.content.clear.id;
-    rr->real[real_idx].idx = real_idx;
-    rr->real[real_idx].link_fd = fdinfo->fd;
-    rr->real[real_idx].algo = app_ctx;
-    mv_buffer_rtoa(&app_ctx->br, fdinfo, &rr->real[real_idx].idx);
-  } else {
-    if (ctx->verbose) fprintf(stdout, "Packet %d already received (current: %d)\n", bp->ip.ap.fmt.content.clear.id, rr->content_id);
-    mv_buffer_rtof (&app_ctx->br, fdinfo);
-  }
-}
-
-void rr_deliver(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct queued_pkt* dp) {
-  struct evt_core_fdinfo *to_fdinfo = NULL;
-  struct rr_ctx* rr = app_ctx->misc;
-  char url[255];
-
-  // 1. Marked the packet as handled
-  dp->on = 0;
-
-  // 2. Get the buffer
-  struct buffer_packet* bp = get_app_buffer (&app_ctx->br, &dp->idx);
-  assert(bp->ip.ap.fmt.headers.cmd == CMD_CLEAR);
-
-  // 3. We update our cursor
-  rr->content_id = bp->ip.ap.fmt.content.clear.id;
-
-  // 4. Find its target
-  sprintf(url, "udp:write:127.0.0.1:%d", bp->ip.ap.fmt.content.clear.port);
-  to_fdinfo = evt_core_get_from_url (ctx, url);
-  if (to_fdinfo == NULL) {
-    fprintf(stderr, "No fd for URL %s in udp:write for tcp-read. Dropping packet %d :( \n", url, dp->idx);
-    //mv_buffer_wtor (app_ctx, fdinfo, bp);
-    mv_buffer_atof (&app_ctx->br, &dp->idx);
-  }
-
-  // 5. We move the buffer and notify the target
-  //mv_buffer_rtow (app_ctx, fdinfo, to_fdinfo, bp);
-  mv_buffer_atow (&app_ctx->br, &dp->idx, to_fdinfo);
-  main_on_udp_write(ctx, to_fdinfo);
-}
-
-void rr_pkt_manage_links(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
-  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
-  struct rr_ctx* rr = app_ctx->misc;
-
-  assert(bp->ip.ap.fmt.headers.cmd == CMD_HEALTH);
-
-  // 1. Health packet was received too late, dropping it
-  if (ring_le(bp->ip.ap.fmt.content.health.id, rr->health_id_late)) goto release;
-
-  // 2. Reactivate link if deactivated
-  char buffer[16];
-  url_get_port (buffer, fdinfo->url);
-  int link_num = atoi(buffer) - 7500; // @FIXME Hardcoded
-  if (!(rr->remote_links & (1 << link_num))) {
-    printf("Activate link=%d | ", link_num);
-    rr->remote_links |= 1 << link_num; // Make sure that the link is marked as working
-    show_link_availability (rr);
-  }
-
-  // 3. Update RR structure if its the greatest health_id we received
-  if (ring_gt(bp->ip.ap.fmt.content.health.id, rr->health_id)) {
-    // 3.1. Update current health id
-    rr->health_id = bp->ip.ap.fmt.content.health.id;
-
-    // 3.2. Update my links I can use thanks to target feedback
-    if (bp->ip.ap.fmt.content.health.bitfield != rr->my_links) {
-      rr->my_links = bp->ip.ap.fmt.content.health.bitfield;
-      printf("Update my links | ");
-      show_link_availability (rr);
-    }
-  }
-
-  // 4. Set callback to close this health packet window
-  int64_t timeout = rr->mjit - (int64_t) bp->ip.ap.fmt.content.health.deltat;
-  if (timeout <= 0) timeout = 0;
-  uint64_t idx = bp->ip.ap.fmt.content.health.id % PACKET_BUFFER_SIZE;
-
-  rr->wait[idx].health_id = bp->ip.ap.fmt.content.health.id;
-  rr->wait[idx].prevlink = bp->ip.ap.fmt.content.health.prevlink;
-  rr->wait[idx].min_blocked_pkt = bp->ip.ap.fmt.content.health.min_blocked_pkt;
-  rr->wait[idx].algo = app_ctx;
-
-  set_timeout (ctx, timeout, &rr->wait[idx], on_timeout_health);
-
-release:
-  mv_buffer_rtof(&app_ctx->br, fdinfo);
-}
-
-uint64_t rr_pkt_unroll(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx) {
-  struct rr_ctx* rr = app_ctx->misc;
-  struct evt_core_fdinfo* fdinfo = NULL;
-  struct buffer_packet* bp = NULL;
-  uint64_t delivered = 0;
-
-  while(1) {
-    //printf("Trying to deliver %d\n", rr->recv_id+1);
-    struct queued_pkt* def = &rr->real[(rr->content_id+1) % PACKET_BUFFER_SIZE];
-    if (!def->on) break;
-    rr_deliver(ctx, app_ctx, def);
-    delivered++;
-    //printf("Delivered %d\n", rr->recv_id);
-  }
-  return delivered;
-}
-
-//------
-
-int algo_rr_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
-  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
-  struct rr_ctx* rr = app_ctx->misc;
-
-  if (bp->ip.ap.fmt.headers.cmd == CMD_CLEAR) {
-    if (ctx->verbose > 1) fprintf(stderr, "    [algo/rr] Received a CLEAR packet of size %d on URL %s\n", bp->ip.ap.fmt.headers.size, fdinfo->url);
-    // 1. Register packet in our queue
-    rr_pkt_register(ctx, fdinfo, bp);
-
-    // 2. Process queue
-    rr_pkt_unroll (ctx, app_ctx);
-  } else if (bp->ip.ap.fmt.headers.cmd == CMD_HEALTH) {
-    if (ctx->verbose > 1) fprintf(stderr, "    [algo/rr] Received a HEALTH packet of size %d on URL %s\n", bp->ip.ap.fmt.headers.size, fdinfo->url);
-    rr_pkt_manage_links(ctx, fdinfo, bp);
-  } else {
-    fprintf(stderr, "    [algo/rr] Packet CMD unrecognized (%d)\n", bp->ip.ap.fmt.headers.cmd);
-    mv_buffer_rtof(&app_ctx->br, fdinfo);
-  }
-
-  return 0;
-}
-
-int algo_rr_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
-  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
-  struct rr_ctx* rr = app_ctx->misc;
-  struct evt_core_fdinfo *to_fdinfo = NULL;
-  uint16_t min_pkt;
-  guint len;
-  char url[255];
-  size_t health_packet_size = sizeof(bp->ip.ap.fmt.headers) + sizeof(bp->ip.ap.fmt.content.health);
-  size_t max_size = sizeof(struct internet_packet) - health_packet_size;
-
-  if (ctx->verbose > 1) fprintf(stderr, "    [algo/rr] Read a UDP packet on URL %s\n", fdinfo->url);
-
-  // 1. Prepare RR state and packet values
-  struct timespec curr;
-  int secs, nsecs;
-  uint64_t mili_sec;
-
-  if (clock_gettime(CLOCK_MONOTONIC, &curr) == -1){
-    perror("clock_gettime error");
-    exit(EXIT_FAILURE);
-  }
-
-  // 2. Compute delta t
-  secs = curr.tv_sec - rr->emit_time.tv_sec;
-  nsecs = curr.tv_nsec - rr->emit_time.tv_nsec;
-  rr->emit_time = curr;
-  mili_sec = secs * 1000 + nsecs / 1000000;
-  if (mili_sec > rr->mjit) mili_sec = rr->mjit;
-
-  // 3. Prepare fresh packet
-  assert(bp->ip.ap.fmt.headers.cmd == CMD_CLEAR);
-  bp->ip.ap.fmt.content.clear.id = rr->sent_content_id;
-  min_pkt = rr->sent_content_id;
-  rr->sent_content_id++;
-  if (bp->ip.ap.fmt.headers.size > max_size) {
-    fprintf(stderr, "Packet is too big to be relayed. Oops...\n");
-    exit(EXIT_FAILURE);
-  }
-
-  // 4. Append redundancy if needed
-  if (app_ctx->ap.redundant_data == 1) {
-    size_t current_size = get_full_size (bp);
-    size_t final_size = current_size + rr->prev_packet.ap.fmt.headers.size;
-    if (final_size <= max_size) {
-      min_pkt = rr->prev_packet.ap.fmt.content.clear.id;
-      append_buffer(&bp->ip.ap, bp->ap_count, &rr->prev_packet.ap); // We append previous packet
-      bp->ap_count++;
-    } else if (ctx->verbose) {
-      fprintf(stderr, "    [algo/rr] Can't append redundancy (current=%ld, after=%ld, max=%ld)\n", current_size, final_size, max_size);
-    }
-
-    append_buffer(&rr->prev_packet.ap, 0, &bp->ip.ap); // We store current packet for next time
-  }
-
-  // 5. Append health packet
-  struct buffer_packet hp;
-  hp.ip.ap.fmt.headers.cmd = CMD_HEALTH;
-  hp.ip.ap.fmt.headers.size = sizeof(bp->ip.ap.fmt.headers) + sizeof(bp->ip.ap.fmt.content.health);
-  hp.ip.ap.fmt.content.health.id = rr->sent_health_id;
-  hp.ip.ap.fmt.content.health.deltat = mili_sec;
-  hp.ip.ap.fmt.content.health.prevlink = rr->current_link;
-  hp.ip.ap.fmt.content.health.bitfield = rr->remote_links;
-  hp.ip.ap.fmt.content.health.min_blocked_pkt = min_pkt;
-  rr->sent_health_id++;
-  append_buffer(&bp->ip.ap, bp->ap_count, &hp.ip.ap);
-  bp->ap_count++;
-
-  // 6. Try to find someone to send it
-  int max = 16;
-  uint8_t sel_link = rr->current_link;
-  while(max-- >= 0) {
-    if (app_ctx->ap.is_waiting_bootstrap && !app_ctx->is_rdy) goto not_ready; // Still bootstrapping
-    sel_link = (sel_link + 1) % app_ctx->ap.links;
-    sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + sel_link); //@FIXME Hardcoded
-    to_fdinfo = evt_core_get_from_url (ctx, url);
-    if (to_fdinfo == NULL) {
-      if (ctx->verbose) fprintf(stderr, "    [algo/rr] write fd %s has not been found, skipping\n", url);
-      continue;
-    }
-
-    if ((len = write_queue_len (&app_ctx->br, to_fdinfo)) > 0) {
-      if (ctx->verbose) fprintf(stderr, "    [algo/rr] write queue of %s is not empty (%d), skipping and deactivating\n", to_fdinfo->url, len);
-      blacklist_link (rr, sel_link);
-      continue;
-    }
-
-    if (!app_ctx->ap.is_healing /* if healing deactivated */|| rr->my_links & (1 << sel_link) /* if link not down */ ) {
-      rr->current_link = sel_link;
-      mv_buffer_rtow (&app_ctx->br, fdinfo, to_fdinfo);
-      main_on_tcp_write(ctx, to_fdinfo);
-      return 0;
-    } else {
-      struct buffer_packet* dup_bp = dup_buffer_tow(&app_ctx->br, bp, to_fdinfo);
-      /*
-       * for later
-      dup_bp->ip.ap.fmt.content.health.min_blocked_pkt = 0;
-      dup_bp->ap_count = 1; // We want to send only health packet to help link recover... Bwarf same traffic on Tor...
-       */
-      main_on_tcp_write(ctx, to_fdinfo);
-    }
-  }
-
-not_ready:
-  // 3. We find no up target
-  fprintf(stderr, "Still bootstrapping or no link to forward data from %s in udp-read. Dropping packet :( \n", fdinfo->url);
-  mv_buffer_wtof (&app_ctx->br, fdinfo);
-  return 0;
-}
-
-void on_timeout_health (struct evt_core_ctx* ctx, void* raw) {
-  struct timer_info* t = raw;
-  struct algo_ctx* app_ctx = t->algo;
-  struct rr_ctx* rr = app_ctx->misc;
-
-  // 1. Update link recovery window if needed
-  if (ring_gt(t->health_id, rr->health_id_late)) rr->health_id_late = t->health_id;
-
-  // 2. Blacklist previous link if needed
-  uint16_t prev_health_id = (t->health_id - 1);
-  uint16_t prev_health_idx = prev_health_id % PACKET_BUFFER_SIZE;
-  struct timer_info* t_old = &rr->wait[prev_health_idx];
-  if (t_old->health_id != prev_health_id) blacklist_link (rr, t->prevlink);
-
-  // 3. Deliver blocked packets
-  // @FIXME CRAPPY CODE / CRAPPY LOGIC
-  //printf("t->min_blocked_pkt=%d, rr->content_id=%d\n", t->min_blocked_pkt, rr->content_id);
-  if (ring_gt(t->min_blocked_pkt, rr->content_id) && !rr->real[t->min_blocked_pkt % PACKET_BUFFER_SIZE].on) {
-    fprintf(stderr, "min_blocked_packet has not been received, t->min_blocked_pkt=%d, rr->content_id=%d\n", t->min_blocked_pkt, rr->content_id);
-    exit(EXIT_FAILURE);
-  }
-  while (ring_gt(t->min_blocked_pkt, rr->content_id - 1)) {
-    rr->content_id++;
-    rr_pkt_unroll (ctx, app_ctx);
-  }
-  rr_pkt_unroll (ctx, app_ctx);
-}
-
-int algo_rr_on_err(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
-  if (strstr(fdinfo->cat->name, "udp") != NULL) return 1;
-  return 0;
-}
-
-void algo_rr_free(void* v) {
-  struct rr_ctx* rr = v;
-  free(rr);
-}
-
-void algo_rr_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap) {
-  struct rr_ctx* rr = malloc(sizeof(struct rr_ctx));
-  if (rr == NULL) {
-    perror("malloc failed for rr_init.");
-    exit(EXIT_FAILURE);
-  }
-  memset(rr, 0, sizeof(struct rr_ctx));
-  rr->mjit = 200;
-  rr->my_links = 0xff;
-  rr->remote_links = 0xff;
-  rr->sent_health_id = 1;
-  rr->sent_content_id = 1;
-  rr->health_id = 0;
-  rr->health_id_late = 0;
-  rr->content_id = 0;
-  rr->current_link = 0;
-  app_ctx->misc = rr;
-  app_ctx->free_misc = algo_rr_free;
-
-  init_timer(ctx);
-}
-
diff --git a/src/algo_thunder.c b/src/algo_thunder.c
new file mode 100644
index 0000000..7956afc
--- /dev/null
+++ b/src/algo_thunder.c
@@ -0,0 +1,40 @@
+#include <sys/timerfd.h>
+#include "algo_utils.h"
+#include "utils.h"
+#include "url.h"
+#include "proxy.h"
+#include "timer.h"
+
+
+void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+}
+
+void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+
+}
+
+int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+
+
+  return 0;
+}
+
+void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap) {
+
+}
+
+int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+
+  return 0;
+}
+
+int algo_thunder_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  prepare(ctx, fdinfo, bp);
+  pad(ctx, fdinfo, bp);
+  return schedule(ctx, fdinfo, bp);
+}
+
+int algo_thunder_on_err(struct evt_core_ctx *ctx, struct evt_core_fdinfo *fdinfo) {
+  if (strstr(fdinfo->cat->name, "udp") != NULL) return 1;
+  return 0;
+}
diff --git a/src/packet.c b/src/packet.c
index 8af43f7..f49d6a2 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -1,28 +1,33 @@
 #include "packet.h"
 
 size_t get_full_size(struct buffer_packet* bp) {
-  union abstract_packet* ap = &bp->ip.ap;
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
   for (int i = 0; i < bp->ap_count; i++) {
     ap = (union abstract_packet*)(&ap->raw + ap->fmt.headers.size);
   }
-  return &ap->raw - &bp->ip.ap.raw;
+  return &ap->raw - &bp->ip[0];
 }
 
-enum FD_STATE read_packet_from_tcp(int fd, struct buffer_packet* bp) {
+void append_data(struct buffer_packet* bp, char* data, size_t size) {
+
+}
+
+enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   ssize_t nread;
-  size_t pkt_size_size = sizeof(bp->ip.ap.fmt.headers.size);
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
+  size_t pkt_size_size = sizeof(ap->fmt.headers.size);
   if (bp->mode != BP_READING) return FDS_ERR;
 
   while (bp->aread < pkt_size_size) {
-    nread = read(fd, &(bp->ip.ap.raw) + bp->aread, pkt_size_size - bp->aread);
+    nread = read(fdinfo->fd, &(ap->raw) + bp->aread, pkt_size_size - bp->aread);
     if (nread == 0) return FDS_AGAIN;
     if (nread == -1 && errno == EAGAIN) return FDS_AGAIN;
     if (nread == -1) return FDS_ERR;
     bp->aread += nread;
   }
 
-  while (bp->aread < bp->ip.ap.fmt.headers.size) {
-    nread = read(fd, &(bp->ip.ap.raw) + bp->aread, bp->ip.ap.fmt.headers.size - bp->aread);
+  while (bp->aread < ap->fmt.headers.size) {
+    nread = read(fdinfo->fd, &(ap->raw) + bp->aread, ap->fmt.headers.size - bp->aread);
     if (nread == 0) return FDS_AGAIN;
     if (nread == -1 && errno == EAGAIN) return FDS_AGAIN;
     if (nread == -1) return FDS_ERR;
@@ -36,13 +41,14 @@ enum FD_STATE read_packet_from_tcp(int fd, struct buffer_packet* bp) {
   return FDS_READY;
 }
 
-enum FD_STATE write_packet_to_tcp(int fd, struct buffer_packet* bp) {
+enum FD_STATE write_packet_to_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   ssize_t nwrite;
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
 
   //dump_buffer_packet (bp);
   if (bp->mode != BP_WRITING) return FDS_ERR;
   while (bp->awrite < get_full_size(bp)) {
-    nwrite = send(fd, &(bp->ip.ap.raw) + bp->awrite, get_full_size(bp) - bp->awrite, 0);
+    nwrite = send(fdinfo->fd, &(ap->raw) + bp->awrite, get_full_size(bp) - bp->awrite, 0);
     if (nwrite == -1 && errno == EAGAIN) return FDS_AGAIN;
     if (nwrite == -1) return FDS_ERR;
     bp->awrite += nwrite;
@@ -54,11 +60,13 @@ enum FD_STATE write_packet_to_tcp(int fd, struct buffer_packet* bp) {
   return FDS_READY;
 }
 
-enum FD_STATE write_packet_to_udp(int fd, struct buffer_packet* bp, struct udp_target* udp_t) {
+enum FD_STATE write_packet_to_udp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp, struct udp_target* udp_t) {
   ssize_t nwrite;
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
+
   size_t bytes_to_send;
   assert(bp->ip.ap.fmt.headers.cmd == CMD_CLEAR);
-  size_t pkt_header_size = sizeof(bp->ip.ap.fmt.headers) + sizeof(bp->ip.ap.fmt.content.clear) - sizeof(char);
+  size_t pkt_header_size = sizeof(ap->fmt.headers);
   struct sockaddr* addr = NULL;
   socklen_t addrlen = 0;
   if (udp_t->set) {
@@ -68,9 +76,9 @@ enum FD_STATE write_packet_to_udp(int fd, struct buffer_packet* bp, struct udp_t
 
   if (bp->mode != BP_WRITING) return FDS_ERR;
 
-  bytes_to_send = bp->ip.ap.fmt.headers.size - pkt_header_size;
-  nwrite = sendto(fd,
-                  &(bp->ip.ap.fmt.content.clear.payload),
+  bytes_to_send = ap->fmt.headers.size - pkt_header_size;
+  nwrite = sendto(fdinfo->fd,
+                  &(ap->fmt.content.udp_encapsulated),
                   bytes_to_send,
                   0,
                   addr,
@@ -86,19 +94,21 @@ enum FD_STATE write_packet_to_udp(int fd, struct buffer_packet* bp, struct udp_t
   return FDS_READY;
 }
 
-enum FD_STATE read_packet_from_udp (int fd, struct buffer_packet* bp, struct udp_target* udp_t) {
+enum FD_STATE read_packet_from_udp (struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp, struct udp_target* udp_t) {
   ssize_t nread;
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
+
   if (bp->mode != BP_READING) {
     fprintf(stderr, "Buffer packet is not in reading mode (mode: %d)\n", bp->mode);
     return FDS_ERR;
   }
 
-  size_t pkt_header_size = sizeof(bp->ip.ap.fmt.headers) + sizeof(bp->ip.ap.fmt.content.clear) - sizeof(char); // We remove the payload
-  size_t udp_packet_size = sizeof(struct internet_packet) - pkt_header_size;
+  size_t pkt_header_size = sizeof(ap->fmt.headers);
+  size_t udp_packet_size = sizeof(bp->ip) - pkt_header_size;
   socklen_t addrlen = sizeof(struct sockaddr_in);
 
-  nread = recvfrom(fd,
-                   &(bp->ip.ap.fmt.content.clear.payload),
+  nread = recvfrom(fdinfo->fd,
+                   &(ap->fmt.content.udp_encapsulated.payload),
                    udp_packet_size,
                    MSG_TRUNC,
                    (struct sockaddr*)&udp_t->addr,
@@ -117,8 +127,9 @@ enum FD_STATE read_packet_from_udp (int fd, struct buffer_packet* bp, struct udp
 
   udp_t->set = 1;
   udp_t->addrlen = addrlen;
-  bp->ip.ap.fmt.headers.size = nread + pkt_header_size;
-  bp->ip.ap.fmt.headers.cmd = CMD_CLEAR;
+  ap->fmt.headers.size = nread + pkt_header_size;
+  ap->fmt.headers.cmd = CMD_UDP_ENCAPSULATED;
+  ap->fmt.content.udp_encapsulated.port = url_get_port_int (fdinfo->url);
 
   bp->mode = BP_WRITING;
   bp->awrite = 0;
@@ -129,8 +140,8 @@ enum FD_STATE read_packet_from_udp (int fd, struct buffer_packet* bp, struct udp
 
 void dump_buffer_packet(struct buffer_packet* bp) {
   printf("<Buffer Packet>\n");
-  printf("  mode=%d, aread=%d, awrite=%d, ap_count=%d, usage=%ld/%ld\n", bp->mode, bp->aread, bp->awrite, bp->ap_count, get_full_size (bp), sizeof(struct internet_packet));
-  union abstract_packet* ap = &bp->ip.ap;
+  printf("  mode=%d, aread=%d, awrite=%d, ap_count=%d, usage=%ld/%ld\n", bp->mode, bp->aread, bp->awrite, bp->ap_count, get_full_size (bp), sizeof(bp->ip));
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
   for (int i = 0; i < bp->ap_count; i++) {
     dump_abstract_packet(ap);
     ap = (union abstract_packet*)(&ap->raw + ap->fmt.headers.size);
@@ -142,21 +153,20 @@ void dump_abstract_packet(union abstract_packet* ap) {
   printf("  <Abstract Packet>\n");
   printf("    size=%d, cmd=%d\n", ap->fmt.headers.size, ap->fmt.headers.cmd);
   switch (ap->fmt.headers.cmd) {
-  case CMD_HEALTH:
+  case CMD_LINK_MONITORING_THUNDER:
     printf("    <Health>id=%d, deltat=%d, prevlink=%d, min_blocked_pkt=%d, bitfield=%02x</Health>\n",
-           ap->fmt.content.health.id,
-           ap->fmt.content.health.deltat,
-           ap->fmt.content.health.prevlink,
-           ap->fmt.content.health.min_blocked_pkt,
-           ap->fmt.content.health.bitfield);
+           ap->fmt.content.link_monitoring_thunder.id,
+           ap->fmt.content.link_monitoring_thunder.deltat,
+           ap->fmt.content.link_monitoring_thunder.prevlink,
+           ap->fmt.content.link_monitoring_thunder.min_blocked_pkt,
+           ap->fmt.content.link_monitoring_thunder.bitfield);
     break;
-  case CMD_CLEAR:
-    printf("    <Clear>id=%d, port=%d</Clear>\n",
-           ap->fmt.content.clear.id,
-           ap->fmt.content.clear.port);
+  case CMD_UDP_METADATA_THUNDER:
+    printf("    <Clear>id=%d</Clear>\n",
+           ap->fmt.content.udp_metadata_thunder.id);
     break;
-  case CMD_XOR:
-    printf("    <Xor>Unimplemented</Xor>\n");
+  case CMD_UDP_ENCAPSULATED:
+    printf("    <Payload>port=%d</Payload>\n", ap->fmt.content.udp_encapsulated.port);
     break;
   default:
     printf("    <Unknown/>\n");
diff --git a/src/packet.h b/src/packet.h
index a8e78a8..30cb1d7 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -9,6 +9,8 @@
 #include <errno.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
+#include "evt_core.h"
+#include "url.h"
 
 /*
  * man 7 udp about receive operation on UDP sockets:
@@ -30,9 +32,9 @@ enum BP_MODE {
 };
 
 enum PKT_CMD {
-  CMD_HEALTH,
-  CMD_CLEAR,
-  CMD_XOR
+  CMD_UDP_ENCAPSULATED,
+  CMD_LINK_MONITORING_THUNDER,
+  CMD_UDP_METADATA_THUNDER,
 };
 
 union abstract_packet {
@@ -50,28 +52,27 @@ union abstract_packet {
         uint8_t prevlink;
         uint16_t deltat;
         uint16_t min_blocked_pkt;
-      } health;
+      } link_monitoring_thunder;
       struct {
         uint16_t id;
+        uint16_t deltat;
+
+      } udp_metadata_thunder;
+      struct {
         uint16_t port;
         char payload;
-      } clear;
+      } udp_encapsulated;
     } content;
   } fmt;
 };
 
-struct internet_packet {
-  union abstract_packet ap;
-  char rest[1499]; // MTU = 1500, 1 byte in the union as payload
-};
-
 struct buffer_packet {
   enum BP_MODE mode;
   uint8_t ap_count;
   uint16_t aread;
   uint16_t awrite;
   struct timespec seen;
-  struct internet_packet ip;
+  char ip[1500];
 };
 
 struct udp_target {
@@ -83,10 +84,10 @@ struct udp_target {
 
 size_t get_full_size(struct buffer_packet* bp);
 
-enum FD_STATE read_packet_from_tcp(int fd, struct buffer_packet* bp);
-enum FD_STATE write_packet_to_tcp(int fd, struct buffer_packet* bp);
-enum FD_STATE write_packet_to_udp(int fd, struct buffer_packet* bp, struct udp_target* udp_t);
-enum FD_STATE read_packet_from_udp (int fd, struct buffer_packet* bp, struct udp_target* udp_t);
+enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fd, struct buffer_packet* bp);
+enum FD_STATE write_packet_to_tcp(struct evt_core_fdinfo* fd, struct buffer_packet* bp);
+enum FD_STATE write_packet_to_udp(struct evt_core_fdinfo* fd, struct buffer_packet* bp, struct udp_target* udp_t);
+enum FD_STATE read_packet_from_udp (struct evt_core_fdinfo* fd, struct buffer_packet* bp, struct udp_target* udp_t);
 
 void dump_buffer_packet(struct buffer_packet* bp);
 void dump_abstract_packet(union abstract_packet* ap);
diff --git a/src/proxy.c b/src/proxy.c
index afe9772..39c4c44 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -48,7 +48,7 @@ int main_on_tcp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
 
   if (ctx->verbose > 1) fprintf(stderr, "  [proxy] Try to read a whole packet in the buffer\n");
   while (bp->mode == BP_READING) {
-    read_res = read_packet_from_tcp (fdinfo->fd, bp);
+    read_res = read_packet_from_tcp (fdinfo, bp);
     if (read_res == FDS_ERR) goto co_error;
     if (read_res == FDS_AGAIN) return 1;
   }
@@ -72,8 +72,7 @@ int main_on_udp_read(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo) {
   if ((bp = get_read_buffer(&app_ctx->br, fdinfo)) == NULL) return 1;
 
   // 2. Read packet from socket
-  bp->ip.ap.fmt.content.clear.port = url_get_port_int (fdinfo->url);
-  read_res = read_packet_from_udp (fdinfo->fd, bp, fdinfo->other);
+  read_res = read_packet_from_udp (fdinfo, bp, fdinfo->other);
   if (read_res == FDS_ERR) goto co_error;
   if (read_res == FDS_AGAIN) return 1;
 
@@ -105,7 +104,7 @@ int main_on_tcp_write(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo)
 
   // 2. Write data from the buffer to the socket
   while (bp->mode == BP_WRITING) {
-    write_res = write_packet_to_tcp(fdinfo->fd, bp);
+    write_res = write_packet_to_tcp(fdinfo, bp);
     if (write_res == FDS_ERR) goto co_error;
     if (write_res == FDS_AGAIN) return 1;
   }
@@ -131,7 +130,7 @@ int main_on_udp_write (struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo)
   if ((bp = get_write_buffer(&app_ctx->br, fdinfo)) == NULL) return 1;
 
   // 2. Write buffer
-  write_res = write_packet_to_udp(fdinfo->fd, bp, fdinfo->other);
+  write_res = write_packet_to_udp(fdinfo, bp, fdinfo->other);
   if (write_res == FDS_ERR) goto co_error;
   if (write_res == FDS_AGAIN) return 1;
 
diff --git a/src/proxy.h b/src/proxy.h
index fc9ab61..aee276e 100644
--- a/src/proxy.h
+++ b/src/proxy.h
@@ -50,16 +50,16 @@ int  algo_naive_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdin
 int  algo_naive_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp);
 int  algo_naive_on_err(struct evt_core_ctx *ctx, struct evt_core_fdinfo *fdinfo);
 
-void algo_rr_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap);
-int algo_rr_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp);
-int algo_rr_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp);
-int algo_rr_on_err(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo);
-
 void algo_dup2_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap);
 int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp);
 int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp);
 int algo_dup2_on_err(struct evt_core_ctx *ctx, struct evt_core_fdinfo *fdinfo);
 
+void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap);
+int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp);
+int algo_thunder_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp);
+int algo_thunder_on_err(struct evt_core_ctx *ctx, struct evt_core_fdinfo *fdinfo);
+
 static struct algo_desc available_algo[] = {
   {
     .name = "naive",
@@ -68,19 +68,19 @@ static struct algo_desc available_algo[] = {
     .on_datagram = algo_naive_on_datagram,
     .on_err = algo_naive_on_err
   },
-  {
-    .name = "rr",
-    .init = algo_rr_init,
-    .on_stream = algo_rr_on_stream,
-    .on_datagram =  algo_rr_on_datagram,
-    .on_err = algo_rr_on_err
-  },
   {
     .name = "dup2",
     .init = algo_dup2_init,
     .on_stream = algo_dup2_on_stream,
     .on_datagram = algo_dup2_on_datagram,
     .on_err = algo_dup2_on_err
+  },
+  {
+    .name = "thunder",
+    .init = algo_thunder_init,
+    .on_stream = algo_thunder_on_stream,
+    .on_datagram = algo_thunder_on_datagram,
+    .on_err = algo_thunder_on_err
   }
 };
 

From 577408a050cd9e76aa9dc544e1702a25558384ee Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Fri, 9 Aug 2019 18:09:15 +0200
Subject: [PATCH 02/35] WIP algo

---
 src/algo_dup2.c | 61 +++++++++++++++++++++++++++++++++----------------
 1 file changed, 41 insertions(+), 20 deletions(-)

diff --git a/src/algo_dup2.c b/src/algo_dup2.c
index e179f79..647a978 100644
--- a/src/algo_dup2.c
+++ b/src/algo_dup2.c
@@ -1,5 +1,6 @@
 #include "proxy.h"
 #include "algo_utils.h"
+#include "packet.h"
 
 struct dup2_ctx {
   uint16_t recv_id;
@@ -19,30 +20,50 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
   char url[256];
   struct evt_core_fdinfo *to_fdinfo = NULL;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
-  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
-
-  // Check that we didn't already received the packet
+  union abstract_packet *ap = (union abstract_packet*) &bp->ip, *ap_prev = NULL;
   struct dup2_ctx* dup2c = app_ctx->misc;
-  if (ring_ge(dup2c->recv_id, ap->fmt.content.clear.id)) {
+  uint16_t id = 0;
+
+  switch (ap->fmt.headers.cmd) {
+  case CMD_UDP_METADATA_THUNDER:
+    id = ap->fmt.content.udp_metadata_thunder.id;
     mv_buffer_rtof(&app_ctx->br, fdinfo);
-    return 0;
+
+    // Check that received identifier has not been delivered
+    if (ring_ge(dup2c->recv_id, id)) {
+      mv_buffer_atof(&app_ctx->br, fdinfo);
+      return 0;
+    }
+
+    // Update delivered identifier
+    dup2c->recv_id = id;
+
+    // 1. Find destination
+    ap_prev = (union abstract_packet*) get_app_buffer(&app_ctx->br, fdinfo)->ip;
+    sprintf(url, "udp:write:127.0.0.1:%d", ap_prev->fmt.content.udp_encapsulated.port);
+    to_fdinfo = evt_core_get_from_url (ctx, url);
+    if (to_fdinfo == NULL) {
+      fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
+      mv_buffer_atof (&app_ctx->br, fdinfo);
+      return 1;
+    }
+
+    // 2. Move buffer
+    mv_buffer_atow (&app_ctx->br, fdinfo, to_fdinfo);
+    main_on_udp_write(ctx, to_fdinfo);
+
+    break;
+
+  case CMD_UDP_ENCAPSULATED:
+    mv_buffer_rtoa(&app_ctx->br, fdinfo, fdinfo);
+    break;
+
+  default:
+    fprintf(stderr, "Unknown packet type\n");
+    break;
   }
 
-  dup2c->recv_id = bp->ip.ap.fmt.content.clear.id;
-
-  // 1. Find destination
-  sprintf(url, "udp:write:127.0.0.1:%d", bp->ip.ap.fmt.content.clear.port);
-  to_fdinfo = evt_core_get_from_url (ctx, url);
-  if (to_fdinfo == NULL) {
-    fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
-    mv_buffer_wtof (&app_ctx->br, fdinfo);
-    return 1;
-  }
-
-  // 2. Move buffer
-  mv_buffer_rtow (&app_ctx->br, fdinfo, to_fdinfo);
-  main_on_udp_write(ctx, to_fdinfo);
-  return 0;
+  return 1;
 }
 
 int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {

From b985e0041a1dfa029e27fea0634b483a9dc8e622 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Mon, 12 Aug 2019 15:28:22 +0200
Subject: [PATCH 03/35] Introduce a new packet format

---
 src/algo_dup2.c  | 87 ++++++++++++++++++++++++++----------------------
 src/algo_utils.c | 24 ++++---------
 src/algo_utils.h |  2 --
 src/capdiff.c    | 14 +++++---
 src/capreplay.c  |  4 ++-
 src/packet.c     | 72 ++++++++++++++++++++++++++-------------
 src/packet.h     | 13 ++++++--
 7 files changed, 124 insertions(+), 92 deletions(-)

diff --git a/src/algo_dup2.c b/src/algo_dup2.c
index 647a978..7750df0 100644
--- a/src/algo_dup2.c
+++ b/src/algo_dup2.c
@@ -20,50 +20,51 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
   char url[256];
   struct evt_core_fdinfo *to_fdinfo = NULL;
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
-  union abstract_packet *ap = (union abstract_packet*) &bp->ip, *ap_prev = NULL;
+  union abstract_packet *ap = (union abstract_packet*) &bp->ip;
   struct dup2_ctx* dup2c = app_ctx->misc;
-  uint16_t id = 0;
+  int32_t id = -1, port = -1;
 
-  switch (ap->fmt.headers.cmd) {
-  case CMD_UDP_METADATA_THUNDER:
-    id = ap->fmt.content.udp_metadata_thunder.id;
-    mv_buffer_rtof(&app_ctx->br, fdinfo);
-
-    // Check that received identifier has not been delivered
-    if (ring_ge(dup2c->recv_id, id)) {
-      mv_buffer_atof(&app_ctx->br, fdinfo);
-      return 0;
+  do {
+    switch (ap->fmt.headers.cmd) {
+    case CMD_UDP_METADATA_THUNDER:
+      id = ap->fmt.content.udp_metadata_thunder.id;
+      break;
+    case CMD_UDP_ENCAPSULATED:
+      port = ap->fmt.content.udp_encapsulated.port;
+      break;
+    default:
+      break;
     }
+  } while ((ap = ap_next(ap)) != NULL);
 
-    // Update delivered identifier
-    dup2c->recv_id = id;
-
-    // 1. Find destination
-    ap_prev = (union abstract_packet*) get_app_buffer(&app_ctx->br, fdinfo)->ip;
-    sprintf(url, "udp:write:127.0.0.1:%d", ap_prev->fmt.content.udp_encapsulated.port);
-    to_fdinfo = evt_core_get_from_url (ctx, url);
-    if (to_fdinfo == NULL) {
-      fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
-      mv_buffer_atof (&app_ctx->br, fdinfo);
-      return 1;
-    }
-
-    // 2. Move buffer
-    mv_buffer_atow (&app_ctx->br, fdinfo, to_fdinfo);
-    main_on_udp_write(ctx, to_fdinfo);
-
-    break;
-
-  case CMD_UDP_ENCAPSULATED:
-    mv_buffer_rtoa(&app_ctx->br, fdinfo, fdinfo);
-    break;
-
-  default:
-    fprintf(stderr, "Unknown packet type\n");
-    break;
+  if (port == -1 || id == -1) {
+    fprintf(stderr, "Missing data...");
+    exit(EXIT_FAILURE);
   }
 
-  return 1;
+  // Check that received identifier has not been delivered
+  if (ring_ge(dup2c->recv_id, id)) {
+    mv_buffer_atof(&app_ctx->br, fdinfo);
+    return 0;
+  }
+
+  // Update delivered identifier
+  dup2c->recv_id = id;
+
+  // 1. Find destination
+  sprintf(url, "udp:write:127.0.0.1:%d", port);
+  to_fdinfo = evt_core_get_from_url (ctx, url);
+  if (to_fdinfo == NULL) {
+    fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
+    mv_buffer_atof (&app_ctx->br, fdinfo);
+    return 1;
+  }
+
+  // 2. Move buffer
+  mv_buffer_atow (&app_ctx->br, fdinfo, to_fdinfo);
+  main_on_udp_write(ctx, to_fdinfo);
+
+  return 0;
 }
 
 int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -71,9 +72,15 @@ int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdin
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
 
   struct dup2_ctx* dup2c = app_ctx->misc;
-  bp->ip.ap.fmt.content.clear.id = dup2c->emit_id;
-  dup2c->emit_id = dup2c->emit_id + 1;
+  union abstract_packet metadata = {
+    .fmt.headers.cmd = CMD_UDP_METADATA_THUNDER,
+    .fmt.headers.size = sizeof(metadata.fmt.headers) + sizeof(metadata.fmt.content.udp_metadata_thunder),
+    .fmt.headers.flags = 0,
+    .fmt.content.udp_metadata_thunder.id = dup2c->emit_id
+  };
+  buffer_append_ap (bp, &metadata);
 
+  dup2c->emit_id = dup2c->emit_id + 1;
   struct evt_core_cat* cat = evt_core_get_from_cat (ctx, "tcp-write");
 
   for (int i = 0; i < app_ctx->ap.links; i++) {
diff --git a/src/algo_utils.c b/src/algo_utils.c
index be059b0..c17b60a 100644
--- a/src/algo_utils.c
+++ b/src/algo_utils.c
@@ -14,6 +14,11 @@ void naive_free_simple(void* v) {
   g_queue_free (g);
 }
 
+void __push_to_free(struct buffer_resources *app_ctx, struct buffer_packet* bp) {
+  memset(bp, 0, sizeof(struct buffer_packet));
+  g_queue_push_tail (app_ctx->free_buffer, bp);
+}
+
 void debug_buffer(struct buffer_resources *app_ctx, struct evt_core_fdinfo *fdinfo) {
   fprintf(stderr, "No more free buffer for fd=%d.\n", fdinfo->fd);
   int waiting_count = 0;
@@ -34,8 +39,7 @@ void init_buffer_management(struct buffer_resources* ctx) {
   ctx->used_buffer = g_hash_table_new(g_int_hash, g_int_equal);
   ctx->write_waiting = g_hash_table_new_full (g_int_hash, g_int_equal, NULL, naive_free_simple);
   for (int i = 0; i < sizeof(ctx->bps) / sizeof(ctx->bps[0]); i++) {
-    memset(&(ctx->bps[i]), 0, sizeof(struct buffer_packet));
-    g_queue_push_tail(ctx->free_buffer, &(ctx->bps[i]));
+    __push_to_free (ctx, &(ctx->bps[i]));
   }
 }
 
@@ -73,11 +77,6 @@ struct buffer_packet* get_read_buffer(struct buffer_resources *app_ctx, struct e
   return bp;
 }
 
-void __push_to_free(struct buffer_resources *app_ctx, struct buffer_packet* bp) {
-  memset(bp, 0, sizeof(struct buffer_packet));
-  g_queue_push_tail (app_ctx->free_buffer, bp);
-}
-
 guint write_queue_len(struct buffer_resources *app_ctx, struct evt_core_fdinfo *fdinfo) {
   GQueue* q;
 
@@ -104,8 +103,6 @@ struct buffer_packet* get_write_buffer(struct buffer_resources *app_ctx, struct
   // 3. Update state
   g_hash_table_insert(app_ctx->used_buffer, &(fdinfo->fd), bp);
 
-  bp->mode = BP_WRITING;
-  bp->awrite = 0;
   return bp;
 }
 
@@ -256,12 +253,3 @@ void notify_read(struct evt_core_ctx* ctx, struct buffer_resources* app_ctx) {
     }
   }
 }
-
-int append_buffer(union abstract_packet* dest, int pos, union abstract_packet* src) {
-  char* target = &(dest->raw);
-  while (pos-- > 0) {
-    target += ((union abstract_packet*) target)->fmt.headers.size;
-  }
-  memcpy(target, src, src->fmt.headers.size);
-  return 0;
-}
diff --git a/src/algo_utils.h b/src/algo_utils.h
index f2845d1..275923d 100644
--- a/src/algo_utils.h
+++ b/src/algo_utils.h
@@ -26,8 +26,6 @@ void mv_buffer_atof(struct buffer_resources* app_ctx, void* from);
 struct buffer_packet* dup_buffer_tow(struct buffer_resources* app_ctx, struct buffer_packet* bp, struct evt_core_fdinfo* to);
 guint write_queue_len(struct buffer_resources *app_ctx, struct evt_core_fdinfo *fdinfo);
 
-int append_buffer(union abstract_packet* dest, int pos, union abstract_packet* src);
-
 struct buffer_packet* get_write_buffer(struct buffer_resources *app_ctx, struct evt_core_fdinfo *fdinfo);
 struct buffer_packet* get_read_buffer(struct buffer_resources *app_ctx, struct evt_core_fdinfo *fdinfo);
 struct buffer_packet* get_app_buffer(struct buffer_resources *app_ctx, void* idx);
diff --git a/src/capdiff.c b/src/capdiff.c
index 96f73d8..143fc90 100644
--- a/src/capdiff.c
+++ b/src/capdiff.c
@@ -9,11 +9,12 @@
 #define MAX_PKTS_TO_CHECK_FOR_DROP 10
 
 uint8_t are_packets_equal(struct buffer_packet bpread[]) {
-  size_t s1 = bpread[0].ip.ap.fmt.headers.size, s2 = bpread[1].ip.ap.fmt.headers.size;
+  union abstract_packet *ap1 = (union abstract_packet*)&bpread[0].ip, *ap2 = (union abstract_packet*) bpread[1].ip;
+  size_t s1 = ap1->fmt.headers.size, s2 = ap2->fmt.headers.size;
   if (s1 != s2) return 0;
 
-  for (size_t idx = sizeof(bpread[0].ip.ap.fmt.headers) + sizeof(bpread[0].ip.ap.fmt.content.clear) - sizeof(char); idx < s1; idx++) {
-    char e1 = (&bpread[0].ip.ap.raw)[idx], e2 = (&bpread[1].ip.ap.raw)[idx];
+  for (size_t idx = &ap1->fmt.content.udp_encapsulated.payload - (char*)ap1; idx < s1; idx++) {
+    char e1 = (&ap1->raw)[idx], e2 = (&ap2->raw)[idx];
     if (e1 != e2) return 0;
   }
 
@@ -41,7 +42,10 @@ void destroy_pkt_stats(gpointer data) {
 }
 
 void update_stats(struct buffer_packet *bp, GHashTable* stat_elem) {
-  gint port = bp->ip.ap.fmt.content.clear.port;
+  union abstract_packet *ap = (union abstract_packet*)&bp->ip;
+  if (ap->fmt.headers.cmd != CMD_UDP_ENCAPSULATED) return;
+
+  gint port = ap->fmt.content.udp_encapsulated.port;
   struct pkt_stats *ps = g_hash_table_lookup(stat_elem, &port);
   if (ps == NULL) {
     ps = malloc(sizeof(struct pkt_stats));
@@ -57,7 +61,7 @@ void update_stats(struct buffer_packet *bp, GHashTable* stat_elem) {
   }
   ps->last = bp->seen;
   ps->count++;
-  ps->cumulated_size += bp->ip.ap.fmt.headers.size;
+  ps->cumulated_size += ap->fmt.headers.size;
 }
 
 void unroll_packets(struct cap_file cf[], struct buffer_packet bpread[], GHashTable* stats[], struct pkt_reconstruct *pr, int m, int i) {
diff --git a/src/capreplay.c b/src/capreplay.c
index d84873f..dbf5537 100644
--- a/src/capreplay.c
+++ b/src/capreplay.c
@@ -11,7 +11,9 @@ void get_ports(struct cap_file *cf) {
   size_t entry_count = cap_count_bp (cf);
   for (int c = 0; c < entry_count; c++) {
     cap_next_bp (cf, &bp);
-    int a = bp.ip.ap.fmt.content.clear.port;
+    union abstract_packet* ap = (union abstract_packet*) &bp.ip;
+    if (ap->fmt.headers.cmd != CMD_UDP_ENCAPSULATED) continue;
+    int a = ap->fmt.content.udp_encapsulated.port;
   }
   cap_begin(cf);
 }
diff --git a/src/packet.c b/src/packet.c
index f49d6a2..95dcd18 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -1,15 +1,33 @@
 #include "packet.h"
 
-size_t get_full_size(struct buffer_packet* bp) {
-  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
-  for (int i = 0; i < bp->ap_count; i++) {
-    ap = (union abstract_packet*)(&ap->raw + ap->fmt.headers.size);
-  }
-  return &ap->raw - &bp->ip[0];
+union abstract_packet* ap_next(union abstract_packet* ap) {
+  if (ap->fmt.headers.flags & FLAG_READ_NEXT)
+    return (union abstract_packet*)(&ap->raw + ap->fmt.headers.size);
+
+  return NULL;
 }
 
-void append_data(struct buffer_packet* bp, char* data, size_t size) {
+union abstract_packet* buffer_last_ptr(struct buffer_packet* bp) {
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip, *apn = NULL;
+  while ((apn = ap_next(ap)) != NULL) ap = apn;
 
+  return ap;
+}
+
+union abstract_packet* buffer_free_ptr(struct buffer_packet* bp) {
+  union abstract_packet* ap = buffer_last_ptr (bp);
+  ap = (union abstract_packet*)(&ap->raw + ap->fmt.headers.size);
+
+  return ap;
+}
+
+size_t get_full_size(struct buffer_packet* bp) {
+  return &(buffer_free_ptr (bp))->raw - &bp->ip[0];
+}
+
+void buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap) {
+  buffer_last_ptr(bp)->fmt.headers.flags |= FLAG_READ_NEXT;
+  memcpy(buffer_last_ptr(bp), ap, ap->fmt.headers.size);
 }
 
 enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -34,9 +52,11 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
     bp->aread += nread;
   }
 
+  bp->ap_count++;
+  if (bp->ap_count != get_full_size (bp)) return FDS_AGAIN;
+
   bp->mode = BP_WRITING;
   bp->awrite = 0;
-  bp->ap_count = 1;
 
   return FDS_READY;
 }
@@ -62,30 +82,34 @@ enum FD_STATE write_packet_to_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_
 
 enum FD_STATE write_packet_to_udp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp, struct udp_target* udp_t) {
   ssize_t nwrite;
-  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
-
-  size_t bytes_to_send;
-  assert(bp->ip.ap.fmt.headers.cmd == CMD_CLEAR);
-  size_t pkt_header_size = sizeof(ap->fmt.headers);
-  struct sockaddr* addr = NULL;
-  socklen_t addrlen = 0;
-  if (udp_t->set) {
-    addr = (struct sockaddr*) &udp_t->addr;
-    addrlen = sizeof(struct sockaddr_in);
-  }
-
+  union abstract_packet* ap = (union abstract_packet*) (&bp->ip + bp->awrite);
+  union abstract_packet* end = buffer_free_ptr (bp);
   if (bp->mode != BP_WRITING) return FDS_ERR;
 
-  bytes_to_send = ap->fmt.headers.size - pkt_header_size;
-  nwrite = sendto(fdinfo->fd,
+  while (ap != end) {
+    if (ap->fmt.headers.cmd != CMD_UDP_ENCAPSULATED) continue;
+
+    size_t bytes_to_send;
+    size_t pkt_header_size = sizeof(ap->fmt.headers);
+    struct sockaddr* addr = NULL;
+    socklen_t addrlen = 0;
+    if (udp_t->set) {
+      addr = (struct sockaddr*) &udp_t->addr;
+      addrlen = sizeof(struct sockaddr_in);
+    }
+
+    bytes_to_send = ap->fmt.headers.size - pkt_header_size;
+    nwrite = sendto(fdinfo->fd,
                   &(ap->fmt.content.udp_encapsulated),
                   bytes_to_send,
                   0,
                   addr,
                   addrlen);
 
-  if (nwrite == -1 && errno == EAGAIN) return FDS_AGAIN;
-  if (nwrite != bytes_to_send) return FDS_ERR;
+    if (nwrite == -1 && errno == EAGAIN) return FDS_AGAIN;
+    if (nwrite != bytes_to_send) return FDS_ERR;
+    bp->awrite += nwrite;
+  }
 
   bp->mode = BP_READING;
   bp->aread = 0;
diff --git a/src/packet.h b/src/packet.h
index 30cb1d7..5377cd4 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -37,12 +37,17 @@ enum PKT_CMD {
   CMD_UDP_METADATA_THUNDER,
 };
 
+enum PKT_FLAGS {
+  FLAG_READ_NEXT = 1 << 0,
+};
+
 union abstract_packet {
   char raw;
   struct {
     struct {
       uint16_t size;
-      enum PKT_CMD cmd;
+      uint8_t cmd;
+      uint8_t flags;
     } headers;
 
     union {
@@ -56,7 +61,6 @@ union abstract_packet {
       struct {
         uint16_t id;
         uint16_t deltat;
-
       } udp_metadata_thunder;
       struct {
         uint16_t port;
@@ -84,6 +88,11 @@ struct udp_target {
 
 size_t get_full_size(struct buffer_packet* bp);
 
+void buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap);
+union abstract_packet* buffer_free_ptr(struct buffer_packet* bp);
+union abstract_packet* buffer_last_ptr(struct buffer_packet* bp);
+union abstract_packet* ap_next(union abstract_packet* ap);
+
 enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fd, struct buffer_packet* bp);
 enum FD_STATE write_packet_to_tcp(struct evt_core_fdinfo* fd, struct buffer_packet* bp);
 enum FD_STATE write_packet_to_udp(struct evt_core_fdinfo* fd, struct buffer_packet* bp, struct udp_target* udp_t);

From cf027261c8da69ef837affdc06f9d392d574e571 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Mon, 12 Aug 2019 16:24:20 +0200
Subject: [PATCH 04/35] Fix packet handling bug

---
 src/algo_naive.c |  6 +++---
 src/packet.c     | 26 ++++++++++++++++++--------
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/src/algo_naive.c b/src/algo_naive.c
index fb85873..8b7a51e 100644
--- a/src/algo_naive.c
+++ b/src/algo_naive.c
@@ -11,7 +11,7 @@ int algo_naive_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinf
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   union abstract_packet* ap = (union abstract_packet*) &bp->ip;
 
-  // 1. Find destination
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_naive] 1/2 Find destination\n");
   sprintf(url, "udp:write:127.0.0.1:%d", ap->fmt.content.udp_encapsulated.port);
   to_fdinfo = evt_core_get_from_url (ctx, url);
   if (to_fdinfo == NULL) {
@@ -20,7 +20,7 @@ int algo_naive_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinf
     return 1;
   }
 
-  // 2. Move buffer
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_naive] 2/2 Move buffer\n");
   mv_buffer_rtow (&app_ctx->br, fdinfo, to_fdinfo);
   main_on_udp_write(ctx, to_fdinfo);
 
@@ -36,7 +36,7 @@ int algo_naive_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdi
   to_fdinfo = cat->socklist->len > 0 ? g_array_index(cat->socklist, struct evt_core_fdinfo*, 0) : NULL;
   if (to_fdinfo == NULL) {
     fprintf(stderr, "No fd for cat %s in udp-read. Dropping packet :( \n", cat->name);
-    mv_buffer_wtof (&app_ctx->br, fdinfo);
+    mv_buffer_wtof(&app_ctx->br, fdinfo);
     return 1;
   }
   //printf("Pass packet from %s to %s\n", fdinfo->url, url);
diff --git a/src/packet.c b/src/packet.c
index 95dcd18..e96fe66 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -7,27 +7,34 @@ union abstract_packet* ap_next(union abstract_packet* ap) {
   return NULL;
 }
 
-union abstract_packet* buffer_last_ptr(struct buffer_packet* bp) {
+union abstract_packet* buffer_last_ap(struct buffer_packet* bp) {
   union abstract_packet* ap = (union abstract_packet*) &bp->ip, *apn = NULL;
   while ((apn = ap_next(ap)) != NULL) ap = apn;
 
   return ap;
 }
 
-union abstract_packet* buffer_free_ptr(struct buffer_packet* bp) {
-  union abstract_packet* ap = buffer_last_ptr (bp);
+union abstract_packet* buffer_free_ap(struct buffer_packet* bp) {
+  union abstract_packet* ap = buffer_last_ap (bp);
   ap = (union abstract_packet*)(&ap->raw + ap->fmt.headers.size);
 
   return ap;
 }
 
+size_t buffer_count_ap(struct buffer_packet* bp) {
+  size_t s = 1;
+  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
+  while ((ap = ap_next(ap)) != NULL) s++;
+  return s;
+}
+
 size_t get_full_size(struct buffer_packet* bp) {
-  return &(buffer_free_ptr (bp))->raw - &bp->ip[0];
+  return &(buffer_free_ap (bp))->raw - &bp->ip[0];
 }
 
 void buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap) {
-  buffer_last_ptr(bp)->fmt.headers.flags |= FLAG_READ_NEXT;
-  memcpy(buffer_last_ptr(bp), ap, ap->fmt.headers.size);
+  buffer_last_ap(bp)->fmt.headers.flags |= FLAG_READ_NEXT;
+  memcpy(buffer_last_ap(bp), ap, ap->fmt.headers.size);
 }
 
 enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -53,7 +60,10 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
   }
 
   bp->ap_count++;
-  if (bp->ap_count != get_full_size (bp)) return FDS_AGAIN;
+  if (bp->ap_count != buffer_count_ap (bp)) {
+    printf("  Expected %ld packets in buffer, received %d\n", get_full_size(bp), bp->ap_count);
+    return FDS_AGAIN;
+  }
 
   bp->mode = BP_WRITING;
   bp->awrite = 0;
@@ -83,7 +93,7 @@ enum FD_STATE write_packet_to_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_
 enum FD_STATE write_packet_to_udp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp, struct udp_target* udp_t) {
   ssize_t nwrite;
   union abstract_packet* ap = (union abstract_packet*) (&bp->ip + bp->awrite);
-  union abstract_packet* end = buffer_free_ptr (bp);
+  union abstract_packet* end = buffer_free_ap(bp);
   if (bp->mode != BP_WRITING) return FDS_ERR;
 
   while (ap != end) {

From bc202c07eb51ba96eb27aa1ddaf5db6d73448678 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Mon, 12 Aug 2019 16:38:19 +0200
Subject: [PATCH 05/35] Learn to count!

---
 src/packet.c | 12 ++++++------
 src/proxy.c  |  4 ++++
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/packet.c b/src/packet.c
index e96fe66..1042343 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -93,14 +93,13 @@ enum FD_STATE write_packet_to_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_
 enum FD_STATE write_packet_to_udp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp, struct udp_target* udp_t) {
   ssize_t nwrite;
   union abstract_packet* ap = (union abstract_packet*) (&bp->ip + bp->awrite);
-  union abstract_packet* end = buffer_free_ap(bp);
   if (bp->mode != BP_WRITING) return FDS_ERR;
 
-  while (ap != end) {
+  do {
     if (ap->fmt.headers.cmd != CMD_UDP_ENCAPSULATED) continue;
 
     size_t bytes_to_send;
-    size_t pkt_header_size = sizeof(ap->fmt.headers);
+    size_t pkt_header_size = sizeof(ap->fmt.headers) + sizeof(ap->fmt.content.udp_encapsulated) - sizeof(ap->fmt.content.udp_encapsulated.payload);
     struct sockaddr* addr = NULL;
     socklen_t addrlen = 0;
     if (udp_t->set) {
@@ -110,7 +109,7 @@ enum FD_STATE write_packet_to_udp(struct evt_core_fdinfo* fdinfo, struct buffer_
 
     bytes_to_send = ap->fmt.headers.size - pkt_header_size;
     nwrite = sendto(fdinfo->fd,
-                  &(ap->fmt.content.udp_encapsulated),
+                  &(ap->fmt.content.udp_encapsulated.payload),
                   bytes_to_send,
                   0,
                   addr,
@@ -119,7 +118,8 @@ enum FD_STATE write_packet_to_udp(struct evt_core_fdinfo* fdinfo, struct buffer_
     if (nwrite == -1 && errno == EAGAIN) return FDS_AGAIN;
     if (nwrite != bytes_to_send) return FDS_ERR;
     bp->awrite += nwrite;
-  }
+
+  } while((ap = ap_next(ap)) != NULL);
 
   bp->mode = BP_READING;
   bp->aread = 0;
@@ -137,7 +137,7 @@ enum FD_STATE read_packet_from_udp (struct evt_core_fdinfo* fdinfo, struct buffe
     return FDS_ERR;
   }
 
-  size_t pkt_header_size = sizeof(ap->fmt.headers);
+  size_t pkt_header_size = sizeof(ap->fmt.headers) + sizeof(ap->fmt.content.udp_encapsulated) - sizeof(ap->fmt.content.udp_encapsulated.payload);
   size_t udp_packet_size = sizeof(bp->ip) - pkt_header_size;
   socklen_t addrlen = sizeof(struct sockaddr_in);
 
diff --git a/src/proxy.c b/src/proxy.c
index 39c4c44..1d0e169 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -127,18 +127,22 @@ int main_on_udp_write (struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo)
   int write_res = FDS_READY;
 
   // 1. Get current write buffer OR a buffer from the waiting queue OR leave
+  if (ctx->verbose > 1) fprintf(stderr, "  [proxy] Find write buffer\n");
   if ((bp = get_write_buffer(&app_ctx->br, fdinfo)) == NULL) return 1;
 
   // 2. Write buffer
+  if (ctx->verbose > 1) fprintf(stderr, "  [proxy] Write UDP packet\n");
   write_res = write_packet_to_udp(fdinfo, bp, fdinfo->other);
   if (write_res == FDS_ERR) goto co_error;
   if (write_res == FDS_AGAIN) return 1;
 
   // 3. Notify helpers
+  if (ctx->verbose > 1) fprintf(stderr, "  [proxy] Notify traffic capture\n");
   traffic_capture_notify (&app_ctx->cap, bp, "out");
 
   // 4. A whole packet has been written
   // Release the buffer and notify
+  if (ctx->verbose > 1) fprintf(stderr, "  [proxy] Release buffer and notify\n");
   mv_buffer_wtof(&app_ctx->br, fdinfo);
   notify_read(ctx, &app_ctx->br);
 

From d515cbf912f0eb0f570a8ff025bbde5f5b5afd5b Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Tue, 13 Aug 2019 10:47:50 +0200
Subject: [PATCH 06/35] Fixed one bug

---
 src/algo_dup2.c | 12 +++++++++-
 src/packet.c    | 59 +++++++++++++++++++++++++++++++------------------
 2 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/src/algo_dup2.c b/src/algo_dup2.c
index 7750df0..c3f1847 100644
--- a/src/algo_dup2.c
+++ b/src/algo_dup2.c
@@ -23,6 +23,8 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
   union abstract_packet *ap = (union abstract_packet*) &bp->ip;
   struct dup2_ctx* dup2c = app_ctx->misc;
   int32_t id = -1, port = -1;
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Received a buffer\n");
+  dump_buffer_packet(bp);
 
   do {
     switch (ap->fmt.headers.cmd) {
@@ -36,14 +38,16 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
       break;
     }
   } while ((ap = ap_next(ap)) != NULL);
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Extracted port=%d and id=%d\n", port, id);
 
   if (port == -1 || id == -1) {
-    fprintf(stderr, "Missing data...");
+    fprintf(stderr, "Missing data port=%d and id=%d...\n", port, id);
     exit(EXIT_FAILURE);
   }
 
   // Check that received identifier has not been delivered
   if (ring_ge(dup2c->recv_id, id)) {
+    if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Packet already delivered, dropping\n");
     mv_buffer_atof(&app_ctx->br, fdinfo);
     return 0;
   }
@@ -61,6 +65,7 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
   }
 
   // 2. Move buffer
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Scheduling packet for write\n");
   mv_buffer_atow (&app_ctx->br, fdinfo, to_fdinfo);
   main_on_udp_write(ctx, to_fdinfo);
 
@@ -80,7 +85,11 @@ int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdin
   };
   buffer_append_ap (bp, &metadata);
 
+  dump_buffer_packet(bp);
+
   dup2c->emit_id = dup2c->emit_id + 1;
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Added metadata\n");
+
   struct evt_core_cat* cat = evt_core_get_from_cat (ctx, "tcp-write");
 
   for (int i = 0; i < app_ctx->ap.links; i++) {
@@ -95,6 +104,7 @@ int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdin
     dup_buffer_tow (&app_ctx->br, bp, to_fdinfo);
     main_on_tcp_write(ctx, to_fdinfo);
   }
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Packets sent\n");
 
   // 3. Release the buffer
   mv_buffer_rtof (&app_ctx->br, fdinfo);
diff --git a/src/packet.c b/src/packet.c
index 1042343..f5879c0 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -35,35 +35,50 @@ size_t get_full_size(struct buffer_packet* bp) {
 void buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap) {
   buffer_last_ap(bp)->fmt.headers.flags |= FLAG_READ_NEXT;
   memcpy(buffer_last_ap(bp), ap, ap->fmt.headers.size);
+  bp->ap_count++;
 }
 
 enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
-  ssize_t nread;
+  ssize_t nread = 0, ap_aread = 0, cur_ap_aread = 0;
   union abstract_packet* ap = (union abstract_packet*) &bp->ip;
   size_t pkt_size_size = sizeof(ap->fmt.headers.size);
   if (bp->mode != BP_READING) return FDS_ERR;
 
-  while (bp->aread < pkt_size_size) {
-    nread = read(fdinfo->fd, &(ap->raw) + bp->aread, pkt_size_size - bp->aread);
-    if (nread == 0) return FDS_AGAIN;
-    if (nread == -1 && errno == EAGAIN) return FDS_AGAIN;
-    if (nread == -1) return FDS_ERR;
-    bp->aread += nread;
-  }
+  fprintf(stderr, "Entering read_packet_from_tcp\n");
+  do {
 
-  while (bp->aread < ap->fmt.headers.size) {
-    nread = read(fdinfo->fd, &(ap->raw) + bp->aread, ap->fmt.headers.size - bp->aread);
-    if (nread == 0) return FDS_AGAIN;
-    if (nread == -1 && errno == EAGAIN) return FDS_AGAIN;
-    if (nread == -1) return FDS_ERR;
-    bp->aread += nread;
-  }
+    fprintf(stderr, "bp->ap_count=%d\n", bp->ap_count);
+    ap = (union abstract_packet*) &bp->ip;
+    for (int i = 0; i < bp->ap_count; i++) {
+      ap_aread += ap->fmt.headers.size;
+      ap = ap_next (ap);
+    }
+    cur_ap_aread = bp->aread - ap_aread;
 
-  bp->ap_count++;
-  if (bp->ap_count != buffer_count_ap (bp)) {
-    printf("  Expected %ld packets in buffer, received %d\n", get_full_size(bp), bp->ap_count);
-    return FDS_AGAIN;
-  }
+    fprintf(stderr, "[size] bp_aread=%d, prev_ap_aread=%ld, cur_ap_aread=%ld\n", bp->aread, ap_aread, cur_ap_aread);
+    while (cur_ap_aread < pkt_size_size) {
+      nread = read(fdinfo->fd, &(ap->raw) + cur_ap_aread, pkt_size_size - cur_ap_aread);
+      if (nread == 0) return FDS_AGAIN;
+      if (nread == -1 && errno == EAGAIN) return FDS_AGAIN;
+      if (nread == -1) return FDS_ERR;
+      bp->aread += nread;
+      cur_ap_aread += nread;
+    }
+
+    fprintf(stderr, "[content] bp_aread=%d, prev_ap_aread=%ld, cur_ap_aread=%ld\n", bp->aread, ap_aread, cur_ap_aread);
+    while (cur_ap_aread < ap->fmt.headers.size) {
+      nread = read(fdinfo->fd, &(ap->raw) + cur_ap_aread, ap->fmt.headers.size - cur_ap_aread);
+      perror("read ap");
+      if (nread == 0) return FDS_AGAIN;
+      if (nread == -1 && errno == EAGAIN) return FDS_AGAIN;
+      if (nread == -1) return FDS_ERR;
+      bp->aread += nread;
+      cur_ap_aread += nread;
+    }
+
+    bp->ap_count++;
+    fprintf(stderr, "bp->ap_count=%d, buffer_count_ap(bp)=%ld\n", bp->ap_count, buffer_count_ap (bp));
+  } while (bp->ap_count != buffer_count_ap (bp));
 
   bp->mode = BP_WRITING;
   bp->awrite = 0;
@@ -188,7 +203,7 @@ void dump_abstract_packet(union abstract_packet* ap) {
   printf("    size=%d, cmd=%d\n", ap->fmt.headers.size, ap->fmt.headers.cmd);
   switch (ap->fmt.headers.cmd) {
   case CMD_LINK_MONITORING_THUNDER:
-    printf("    <Health>id=%d, deltat=%d, prevlink=%d, min_blocked_pkt=%d, bitfield=%02x</Health>\n",
+    printf("    <LinkMonitoringThunder>id=%d, deltat=%d, prevlink=%d, min_blocked_pkt=%d, bitfield=%02x</LinkMonitoringThunder>\n",
            ap->fmt.content.link_monitoring_thunder.id,
            ap->fmt.content.link_monitoring_thunder.deltat,
            ap->fmt.content.link_monitoring_thunder.prevlink,
@@ -196,7 +211,7 @@ void dump_abstract_packet(union abstract_packet* ap) {
            ap->fmt.content.link_monitoring_thunder.bitfield);
     break;
   case CMD_UDP_METADATA_THUNDER:
-    printf("    <Clear>id=%d</Clear>\n",
+    printf("    <UdpMetadataThunder>id=%d</UdpMetadataThunder>\n",
            ap->fmt.content.udp_metadata_thunder.id);
     break;
   case CMD_UDP_ENCAPSULATED:

From 0c50ca5a428e9e6072e2cc8c1d5d677b2f769577 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Tue, 13 Aug 2019 10:53:11 +0200
Subject: [PATCH 07/35] Fix another bug!

---
 src/algo_dup2.c | 11 ++++-------
 src/packet.c    | 10 +++++-----
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/src/algo_dup2.c b/src/algo_dup2.c
index c3f1847..726821f 100644
--- a/src/algo_dup2.c
+++ b/src/algo_dup2.c
@@ -48,7 +48,7 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
   // Check that received identifier has not been delivered
   if (ring_ge(dup2c->recv_id, id)) {
     if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Packet already delivered, dropping\n");
-    mv_buffer_atof(&app_ctx->br, fdinfo);
+    mv_buffer_rtof(&app_ctx->br, fdinfo);
     return 0;
   }
 
@@ -60,13 +60,13 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
   to_fdinfo = evt_core_get_from_url (ctx, url);
   if (to_fdinfo == NULL) {
     fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
-    mv_buffer_atof (&app_ctx->br, fdinfo);
+    mv_buffer_rtof (&app_ctx->br, fdinfo);
     return 1;
   }
 
   // 2. Move buffer
   if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Scheduling packet for write\n");
-  mv_buffer_atow (&app_ctx->br, fdinfo, to_fdinfo);
+  mv_buffer_rtow (&app_ctx->br, fdinfo, to_fdinfo);
   main_on_udp_write(ctx, to_fdinfo);
 
   return 0;
@@ -77,6 +77,7 @@ int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdin
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
 
   struct dup2_ctx* dup2c = app_ctx->misc;
+  dup2c->emit_id = dup2c->emit_id + 1;
   union abstract_packet metadata = {
     .fmt.headers.cmd = CMD_UDP_METADATA_THUNDER,
     .fmt.headers.size = sizeof(metadata.fmt.headers) + sizeof(metadata.fmt.content.udp_metadata_thunder),
@@ -84,14 +85,10 @@ int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdin
     .fmt.content.udp_metadata_thunder.id = dup2c->emit_id
   };
   buffer_append_ap (bp, &metadata);
-
   dump_buffer_packet(bp);
-
-  dup2c->emit_id = dup2c->emit_id + 1;
   if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Added metadata\n");
 
   struct evt_core_cat* cat = evt_core_get_from_cat (ctx, "tcp-write");
-
   for (int i = 0; i < app_ctx->ap.links; i++) {
     // 1. A whole packet has been read, we will find someone to write it
     to_fdinfo = cat->socklist->len > i ? g_array_index(cat->socklist, struct evt_core_fdinfo*, i) : NULL;
diff --git a/src/packet.c b/src/packet.c
index f5879c0..226fbe5 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -44,10 +44,10 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
   size_t pkt_size_size = sizeof(ap->fmt.headers.size);
   if (bp->mode != BP_READING) return FDS_ERR;
 
-  fprintf(stderr, "Entering read_packet_from_tcp\n");
+  //fprintf(stderr, "Entering read_packet_from_tcp\n");
   do {
 
-    fprintf(stderr, "bp->ap_count=%d\n", bp->ap_count);
+    //fprintf(stderr, "bp->ap_count=%d\n", bp->ap_count);
     ap = (union abstract_packet*) &bp->ip;
     for (int i = 0; i < bp->ap_count; i++) {
       ap_aread += ap->fmt.headers.size;
@@ -55,7 +55,7 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
     }
     cur_ap_aread = bp->aread - ap_aread;
 
-    fprintf(stderr, "[size] bp_aread=%d, prev_ap_aread=%ld, cur_ap_aread=%ld\n", bp->aread, ap_aread, cur_ap_aread);
+    //fprintf(stderr, "[size] bp_aread=%d, prev_ap_aread=%ld, cur_ap_aread=%ld\n", bp->aread, ap_aread, cur_ap_aread);
     while (cur_ap_aread < pkt_size_size) {
       nread = read(fdinfo->fd, &(ap->raw) + cur_ap_aread, pkt_size_size - cur_ap_aread);
       if (nread == 0) return FDS_AGAIN;
@@ -65,7 +65,7 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
       cur_ap_aread += nread;
     }
 
-    fprintf(stderr, "[content] bp_aread=%d, prev_ap_aread=%ld, cur_ap_aread=%ld\n", bp->aread, ap_aread, cur_ap_aread);
+    //fprintf(stderr, "[content] bp_aread=%d, prev_ap_aread=%ld, cur_ap_aread=%ld\n", bp->aread, ap_aread, cur_ap_aread);
     while (cur_ap_aread < ap->fmt.headers.size) {
       nread = read(fdinfo->fd, &(ap->raw) + cur_ap_aread, ap->fmt.headers.size - cur_ap_aread);
       perror("read ap");
@@ -77,7 +77,7 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
     }
 
     bp->ap_count++;
-    fprintf(stderr, "bp->ap_count=%d, buffer_count_ap(bp)=%ld\n", bp->ap_count, buffer_count_ap (bp));
+    //fprintf(stderr, "bp->ap_count=%d, buffer_count_ap(bp)=%ld\n", bp->ap_count, buffer_count_ap (bp));
   } while (bp->ap_count != buffer_count_ap (bp));
 
   bp->mode = BP_WRITING;

From 5fdfd9aa9afe10a054acefc68fb124614051b395 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Tue, 13 Aug 2019 10:58:20 +0200
Subject: [PATCH 08/35] Clean code

---
 src/algo_dup2.c | 12 ++++++++----
 src/packet.c    |  1 -
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/algo_dup2.c b/src/algo_dup2.c
index 726821f..b578749 100644
--- a/src/algo_dup2.c
+++ b/src/algo_dup2.c
@@ -23,8 +23,10 @@ int algo_dup2_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo
   union abstract_packet *ap = (union abstract_packet*) &bp->ip;
   struct dup2_ctx* dup2c = app_ctx->misc;
   int32_t id = -1, port = -1;
-  if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Received a buffer\n");
-  dump_buffer_packet(bp);
+  if (ctx->verbose > 1) {
+    fprintf(stderr, "    [algo_dup2] Received a buffer\n");
+    dump_buffer_packet(bp);
+  }
 
   do {
     switch (ap->fmt.headers.cmd) {
@@ -85,8 +87,10 @@ int algo_dup2_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdin
     .fmt.content.udp_metadata_thunder.id = dup2c->emit_id
   };
   buffer_append_ap (bp, &metadata);
-  dump_buffer_packet(bp);
-  if (ctx->verbose > 1) fprintf(stderr, "    [algo_dup2] Added metadata\n");
+  if (ctx->verbose > 1) {
+    dump_buffer_packet(bp);
+    fprintf(stderr, "    [algo_dup2] Added metadata\n");
+  }
 
   struct evt_core_cat* cat = evt_core_get_from_cat (ctx, "tcp-write");
   for (int i = 0; i < app_ctx->ap.links; i++) {
diff --git a/src/packet.c b/src/packet.c
index 226fbe5..4e57bfc 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -68,7 +68,6 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
     //fprintf(stderr, "[content] bp_aread=%d, prev_ap_aread=%ld, cur_ap_aread=%ld\n", bp->aread, ap_aread, cur_ap_aread);
     while (cur_ap_aread < ap->fmt.headers.size) {
       nread = read(fdinfo->fd, &(ap->raw) + cur_ap_aread, ap->fmt.headers.size - cur_ap_aread);
-      perror("read ap");
       if (nread == 0) return FDS_AGAIN;
       if (nread == -1 && errno == EAGAIN) return FDS_AGAIN;
       if (nread == -1) return FDS_ERR;

From e95a4caabe12640801d3c58a3f020ba37d26e95b Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Tue, 13 Aug 2019 17:07:52 +0200
Subject: [PATCH 09/35] WIP thunder algo

---
 src/algo_thunder.c | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 7956afc..ad94bac 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -5,8 +5,24 @@
 #include "proxy.h"
 #include "timer.h"
 
+struct thunder_ctx {
+  uint16_t recv_id;
+  uint16_t emit_id;
+};
+
 
 void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct thunder_ctx* thunderc = app_ctx->misc;
+
+  thunderc->emit_id++;
+  union abstract_packet metadata = {
+    .fmt.headers.cmd = CMD_UDP_METADATA_THUNDER,
+    .fmt.headers.size = sizeof(metadata.fmt.headers) + sizeof(metadata.fmt.content.udp_metadata_thunder),
+    .fmt.headers.flags = 0,
+    .fmt.content.udp_metadata_thunder.id = thunderc->emit_id,
+    .fmt.content.udp_metadata_thunder.deltat = 0
+  };
 }
 
 void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -20,6 +36,12 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
 }
 
 void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap) {
+  app_ctx->misc = malloc(sizeof(struct thunder_ctx));
+  if (app_ctx->misc == NULL) {
+    perror("malloc failed in algo thunder init");
+    exit(EXIT_FAILURE);
+  }
+  memset(app_ctx->misc, 0, sizeof(struct thunder_ctx));
 
 }
 

From 0b43f944a84fb701a6a0011b14846347ba8d261a Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Mon, 26 Aug 2019 12:08:31 +0200
Subject: [PATCH 10/35] WIP thunder

---
 src/algo_thunder.c | 43 ++++++++++++++++++++++++++++++++++++++++++-
 src/packet.c       |  7 +------
 src/packet.h       | 12 +++++++-----
 3 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index ad94bac..84d0c35 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -8,6 +8,11 @@
 struct thunder_ctx {
   uint16_t recv_id;
   uint16_t emit_id;
+  uint64_t lid_sent;
+  uint8_t selected_link;
+  uint8_t total_links;
+  uint64_t lid_sent_per_link[64];
+  uint8_t delta_t_per_link[64];
 };
 
 
@@ -21,8 +26,10 @@ void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     .fmt.headers.size = sizeof(metadata.fmt.headers) + sizeof(metadata.fmt.content.udp_metadata_thunder),
     .fmt.headers.flags = 0,
     .fmt.content.udp_metadata_thunder.id = thunderc->emit_id,
-    .fmt.content.udp_metadata_thunder.deltat = 0
+    .fmt.content.udp_metadata_thunder.deltat = 0 //@FIXME
   };
+  buffer_append_ap (bp, &metadata);
+
 }
 
 void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -30,7 +37,40 @@ void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer
 }
 
 int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct thunder_ctx* thunderc = app_ctx->misc;
+  struct evt_core_fdinfo *to_fdinfo = NULL;
+  struct evt_core_cat* cat = evt_core_get_from_cat (ctx, "tcp-write");
 
+  do {
+    thunderc->selected_link = thunderc->selected_link + 1 % cat->socklist->len;
+
+    if (thunderc->selected_link == 0) {
+      thunderc->lid_sent++;
+
+    }
+
+    union abstract_packet links = {
+      .fmt.headers.cmd = CMD_LINK_MONITORING_THUNDER,
+      .fmt.headers.size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder),
+      .fmt.headers.flags = 0,
+      .fmt.content.link_monitoring_thunder.lid = thunderc->lid_sent,
+      .fmt.content.link_monitoring_thunder.links_status = NULL
+    };
+
+    to_fdinfo = g_array_index(cat->socklist, struct evt_core_fdinfo*, thunderc->selected_link);
+
+    // We move the buffer and notify the target
+    struct buffer_packet* bp_dup = dup_buffer_tow (&app_ctx->br, bp, to_fdinfo);
+    buffer_append_ap (bp_dup, &links);
+
+    main_on_tcp_write(ctx, to_fdinfo);
+  } while (1);
+
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Packets sent\n");
+
+  // Release the buffer
+  mv_buffer_rtof (&app_ctx->br, fdinfo);
 
   return 0;
 }
@@ -42,6 +82,7 @@ void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struc
     exit(EXIT_FAILURE);
   }
   memset(app_ctx->misc, 0, sizeof(struct thunder_ctx));
+  struct thunder_ctx* thunderc = app_ctx->misc;
 
 }
 
diff --git a/src/packet.c b/src/packet.c
index 4e57bfc..c1e845b 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -202,12 +202,7 @@ void dump_abstract_packet(union abstract_packet* ap) {
   printf("    size=%d, cmd=%d\n", ap->fmt.headers.size, ap->fmt.headers.cmd);
   switch (ap->fmt.headers.cmd) {
   case CMD_LINK_MONITORING_THUNDER:
-    printf("    <LinkMonitoringThunder>id=%d, deltat=%d, prevlink=%d, min_blocked_pkt=%d, bitfield=%02x</LinkMonitoringThunder>\n",
-           ap->fmt.content.link_monitoring_thunder.id,
-           ap->fmt.content.link_monitoring_thunder.deltat,
-           ap->fmt.content.link_monitoring_thunder.prevlink,
-           ap->fmt.content.link_monitoring_thunder.min_blocked_pkt,
-           ap->fmt.content.link_monitoring_thunder.bitfield);
+    printf("    <LinkMonitoringThunder/>\n");
     break;
   case CMD_UDP_METADATA_THUNDER:
     printf("    <UdpMetadataThunder>id=%d</UdpMetadataThunder>\n",
diff --git a/src/packet.h b/src/packet.h
index 5377cd4..1c1ffdb 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -41,6 +41,11 @@ enum PKT_FLAGS {
   FLAG_READ_NEXT = 1 << 0,
 };
 
+struct link_info {
+  uint8_t delta_t;
+  uint8_t delta_lid;
+};
+
 union abstract_packet {
   char raw;
   struct {
@@ -52,11 +57,8 @@ union abstract_packet {
 
     union {
       struct {
-        uint16_t id;
-        uint8_t bitfield;
-        uint8_t prevlink;
-        uint16_t deltat;
-        uint16_t min_blocked_pkt;
+        uint16_t lid;
+        struct link_info *links_status;
       } link_monitoring_thunder;
       struct {
         uint16_t id;

From 82e1359f2890e0fb0d341a194c579e22137cb45a Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Mon, 26 Aug 2019 12:18:05 +0200
Subject: [PATCH 11/35] [algo_thunder] add lid update

---
 src/algo_thunder.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 84d0c35..0aa3e03 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -11,7 +11,7 @@ struct thunder_ctx {
   uint64_t lid_sent;
   uint8_t selected_link;
   uint8_t total_links;
-  uint64_t lid_sent_per_link[64];
+  uint8_t lid_sent_per_link[64];
   uint8_t delta_t_per_link[64];
 };
 
@@ -47,7 +47,9 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
 
     if (thunderc->selected_link == 0) {
       thunderc->lid_sent++;
-
+      for (int i = 0; i < thunderc->total_links; i++) {
+        thunderc->lid_sent_per_link[i]++;
+      }
     }
 
     union abstract_packet links = {

From a9b25c43db2f3fb23da09ce6170b7281e089a286 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Mon, 26 Aug 2019 17:35:23 +0200
Subject: [PATCH 12/35] Write scheduler

---
 src/algo_thunder.c | 54 ++++++++++++++++++++++++++++++----------------
 src/packet.c       |  6 ++++--
 src/packet.h       |  6 ++----
 3 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 0aa3e03..faaedb0 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -8,11 +8,11 @@
 struct thunder_ctx {
   uint16_t recv_id;
   uint16_t emit_id;
-  uint64_t lid_sent;
   uint8_t selected_link;
   uint8_t total_links;
-  uint8_t lid_sent_per_link[64];
   uint8_t delta_t_per_link[64];
+  uint8_t blacklisted[64];
+  struct timespec prev_link_time;
 };
 
 
@@ -26,7 +26,7 @@ void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     .fmt.headers.size = sizeof(metadata.fmt.headers) + sizeof(metadata.fmt.content.udp_metadata_thunder),
     .fmt.headers.flags = 0,
     .fmt.content.udp_metadata_thunder.id = thunderc->emit_id,
-    .fmt.content.udp_metadata_thunder.deltat = 0 //@FIXME
+    .fmt.content.udp_metadata_thunder.deltat = 0 //@FIXME delta t must be set
   };
   buffer_append_ap (bp, &metadata);
 
@@ -42,32 +42,50 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
   struct evt_core_fdinfo *to_fdinfo = NULL;
   struct evt_core_cat* cat = evt_core_get_from_cat (ctx, "tcp-write");
 
+  struct timespec curr;
+  int secs, nsecs;
+  uint64_t mili_sec;
+
+
   do {
-    thunderc->selected_link = thunderc->selected_link + 1 % cat->socklist->len;
-
-    if (thunderc->selected_link == 0) {
-      thunderc->lid_sent++;
-      for (int i = 0; i < thunderc->total_links; i++) {
-        thunderc->lid_sent_per_link[i]++;
-      }
-    }
+    // 1. We choose the link
+    thunderc->selected_link = (thunderc->selected_link + 1) % cat->socklist->len;
 
+    // 2. We create the packet template
     union abstract_packet links = {
       .fmt.headers.cmd = CMD_LINK_MONITORING_THUNDER,
-      .fmt.headers.size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder),
+      .fmt.headers.size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder) + sizeof(struct link_info) * (thunderc->total_links - 1),
       .fmt.headers.flags = 0,
-      .fmt.content.link_monitoring_thunder.lid = thunderc->lid_sent,
-      .fmt.content.link_monitoring_thunder.links_status = NULL
+      .fmt.content.link_monitoring_thunder.links_status = {}
     };
 
     to_fdinfo = g_array_index(cat->socklist, struct evt_core_fdinfo*, thunderc->selected_link);
 
-    // We move the buffer and notify the target
+    // 3. We append the template to the buffer
     struct buffer_packet* bp_dup = dup_buffer_tow (&app_ctx->br, bp, to_fdinfo);
-    buffer_append_ap (bp_dup, &links);
+    union abstract_packet *new_ap = buffer_append_ap (bp_dup, &links);
+
+    // 4. We compute the time difference
+    if (clock_gettime(CLOCK_MONOTONIC, &curr) == -1){
+      perror("clock_gettime error");
+      exit(EXIT_FAILURE);
+    }
+    secs = curr.tv_sec - thunderc->prev_link_time.tv_sec;
+    nsecs = curr.tv_nsec - thunderc->prev_link_time.tv_nsec;
+    thunderc->prev_link_time = curr;
+    mili_sec = secs * 1000 + nsecs / 1000000;
+    if (mili_sec > 200) mili_sec = 200;
+
+    // 5. We create the array
+    struct link_info *li = &new_ap->fmt.content.link_monitoring_thunder.links_status;
+    for (int i = 0; i < thunderc->total_links; i++) {
+      thunderc->delta_t_per_link[i] += mili_sec;
+      li[i].delta_t = thunderc->delta_t_per_link[i];
+    }
+    li[thunderc->selected_link].delta_t = 0;
 
     main_on_tcp_write(ctx, to_fdinfo);
-  } while (1);
+  } while (thunderc->blacklisted[thunderc->selected_link]);
 
   if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Packets sent\n");
 
@@ -85,7 +103,7 @@ void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struc
   }
   memset(app_ctx->misc, 0, sizeof(struct thunder_ctx));
   struct thunder_ctx* thunderc = app_ctx->misc;
-
+  thunderc->selected_link = UINT8_MAX - 1;
 }
 
 int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
diff --git a/src/packet.c b/src/packet.c
index c1e845b..85989a7 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -32,10 +32,12 @@ size_t get_full_size(struct buffer_packet* bp) {
   return &(buffer_free_ap (bp))->raw - &bp->ip[0];
 }
 
-void buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap) {
+union abstract_packet* buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap) {
   buffer_last_ap(bp)->fmt.headers.flags |= FLAG_READ_NEXT;
-  memcpy(buffer_last_ap(bp), ap, ap->fmt.headers.size);
+  union abstract_packet *new_ap = buffer_last_ap(bp);
+  memcpy(new_ap, ap, ap->fmt.headers.size);
   bp->ap_count++;
+  return new_ap;
 }
 
 enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
diff --git a/src/packet.h b/src/packet.h
index 1c1ffdb..6b1530f 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -43,7 +43,6 @@ enum PKT_FLAGS {
 
 struct link_info {
   uint8_t delta_t;
-  uint8_t delta_lid;
 };
 
 union abstract_packet {
@@ -57,8 +56,7 @@ union abstract_packet {
 
     union {
       struct {
-        uint16_t lid;
-        struct link_info *links_status;
+        struct link_info links_status;
       } link_monitoring_thunder;
       struct {
         uint16_t id;
@@ -90,7 +88,7 @@ struct udp_target {
 
 size_t get_full_size(struct buffer_packet* bp);
 
-void buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap);
+union abstract_packet* buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap);
 union abstract_packet* buffer_free_ptr(struct buffer_packet* bp);
 union abstract_packet* buffer_last_ptr(struct buffer_packet* bp);
 union abstract_packet* ap_next(union abstract_packet* ap);

From 4cb27b14209abca2804a58339656e520b567a986 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Mon, 26 Aug 2019 17:48:22 +0200
Subject: [PATCH 13/35] Add application duplication

---
 src/algo_thunder.c | 19 ++++++++++++++++---
 src/algo_utils.c   | 22 ++++++++++++++++++++++
 src/algo_utils.h   |  1 +
 3 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index faaedb0..315a4cb 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -15,7 +15,6 @@ struct thunder_ctx {
   struct timespec prev_link_time;
 };
 
-
 void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct thunder_ctx* thunderc = app_ctx->misc;
@@ -29,11 +28,11 @@ void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     .fmt.content.udp_metadata_thunder.deltat = 0 //@FIXME delta t must be set
   };
   buffer_append_ap (bp, &metadata);
-
 }
 
 void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
 
+
 }
 
 int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -106,8 +105,22 @@ void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struc
   thunderc->selected_link = UINT8_MAX - 1;
 }
 
-int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+void classify() {
 
+}
+
+void unpad() {
+
+}
+
+void adapt() {
+
+}
+
+int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  classify();
+  unpad();
+  adapt();
   return 0;
 }
 
diff --git a/src/algo_utils.c b/src/algo_utils.c
index c17b60a..6b5c301 100644
--- a/src/algo_utils.c
+++ b/src/algo_utils.c
@@ -232,6 +232,28 @@ struct buffer_packet* dup_buffer_tow(struct buffer_resources *app_ctx, struct bu
   return bp_dest;
 }
 
+struct buffer_packet* dup_buffer_toa(struct buffer_resources *app_ctx, struct buffer_packet* bp, void* to) {
+  GQueue* q;
+
+  // 1. We get a free buffer
+  struct buffer_packet* bp_dest = g_queue_pop_head(app_ctx->free_buffer);
+  if (bp_dest == NULL) {
+    debug_buffer(app_ctx, to);
+    return NULL;
+  }
+
+  // 2. We duplicate the data
+  memcpy(bp_dest, bp, sizeof(struct buffer_packet));
+
+  // 3. We put the data
+  if (g_hash_table_contains(app_ctx->application_waiting, to)) {
+    fprintf(stderr, "Data already exists for this entry\n");
+    exit(EXIT_FAILURE);
+  }
+  g_hash_table_insert(app_ctx->application_waiting, to, bp_dest);
+  return bp_dest;
+}
+
 struct buffer_packet* get_app_buffer(struct buffer_resources *app_ctx, void* idx) {
   return g_hash_table_lookup (app_ctx->application_waiting, idx);
 }
diff --git a/src/algo_utils.h b/src/algo_utils.h
index 275923d..ab53795 100644
--- a/src/algo_utils.h
+++ b/src/algo_utils.h
@@ -24,6 +24,7 @@ void mv_buffer_rtoa(struct buffer_resources* app_ctx, struct evt_core_fdinfo* fr
 void mv_buffer_atow(struct buffer_resources* app_ctx, void* from, struct evt_core_fdinfo* to);
 void mv_buffer_atof(struct buffer_resources* app_ctx, void* from);
 struct buffer_packet* dup_buffer_tow(struct buffer_resources* app_ctx, struct buffer_packet* bp, struct evt_core_fdinfo* to);
+struct buffer_packet* dup_buffer_toa(struct buffer_resources* app_ctx, struct buffer_packet* bp, void* to);
 guint write_queue_len(struct buffer_resources *app_ctx, struct evt_core_fdinfo *fdinfo);
 
 struct buffer_packet* get_write_buffer(struct buffer_resources *app_ctx, struct evt_core_fdinfo *fdinfo);

From 9c1971af34184304417b075ea6dd0b479c2aa73e Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Tue, 27 Aug 2019 10:10:17 +0200
Subject: [PATCH 14/35] Refactor delta in a function

---
 src/algo_thunder.c | 60 +++++++++++++++++++++++++++++++++-------------
 1 file changed, 43 insertions(+), 17 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 315a4cb..2facadf 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -12,26 +12,62 @@ struct thunder_ctx {
   uint8_t total_links;
   uint8_t delta_t_per_link[64];
   uint8_t blacklisted[64];
-  struct timespec prev_link_time;
+  size_t monit_pkt_size;
+  struct timespec prev_link_time, prev_packet_time;
 };
 
+uint64_t compute_delta(struct timespec* prev_time, uint64_t max) {
+  struct timespec curr;
+  int secs, nsecs;
+  uint64_t mili_sec;
+
+  // 4. We compute the time difference
+  if (clock_gettime(CLOCK_MONOTONIC, &curr) == -1){
+    perror("clock_gettime error");
+    exit(EXIT_FAILURE);
+  }
+  secs = curr.tv_sec - prev_time->tv_sec;
+  nsecs = curr.tv_nsec - prev_time->tv_nsec;
+  *prev_time = curr;
+  mili_sec = secs * 1000 + nsecs / 1000000;
+  if (mili_sec > max) mili_sec = max;
+
+  return mili_sec;
+}
+
 void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct thunder_ctx* thunderc = app_ctx->misc;
 
+  uint64_t delta_pkt = compute_delta(&thunderc->prev_packet_time, 200);
+
   thunderc->emit_id++;
   union abstract_packet metadata = {
     .fmt.headers.cmd = CMD_UDP_METADATA_THUNDER,
     .fmt.headers.size = sizeof(metadata.fmt.headers) + sizeof(metadata.fmt.content.udp_metadata_thunder),
     .fmt.headers.flags = 0,
     .fmt.content.udp_metadata_thunder.id = thunderc->emit_id,
-    .fmt.content.udp_metadata_thunder.deltat = 0 //@FIXME delta t must be set
+    .fmt.content.udp_metadata_thunder.deltat = delta_pkt
   };
   buffer_append_ap (bp, &metadata);
 }
 
 void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct thunder_ctx* thunderc = app_ctx->misc;
+  uint64_t ref = 0l + thunderc->emit_id;
 
+  dup_buffer_toa (&app_ctx->br, bp, (void *)ref);
+
+  if (ref > 10 && get_app_buffer (&app_ctx->br, (void *)ref - 10l)) {
+    mv_buffer_atof (&app_ctx->br, (void *)ref - 10l);
+  }
+
+  //@FIXME we must add the delta t of the current packet
+  for (uint64_t add_ref = ref; add_ref >= 0 && get_app_buffer (&app_ctx->br, (void *)add_ref); add_ref--) {
+    struct buffer_packet *bpa = get_app_buffer (&app_ctx->br, (void *)add_ref);
+
+  }
 
 }
 
@@ -41,11 +77,6 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
   struct evt_core_fdinfo *to_fdinfo = NULL;
   struct evt_core_cat* cat = evt_core_get_from_cat (ctx, "tcp-write");
 
-  struct timespec curr;
-  int secs, nsecs;
-  uint64_t mili_sec;
-
-
   do {
     // 1. We choose the link
     thunderc->selected_link = (thunderc->selected_link + 1) % cat->socklist->len;
@@ -53,7 +84,7 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     // 2. We create the packet template
     union abstract_packet links = {
       .fmt.headers.cmd = CMD_LINK_MONITORING_THUNDER,
-      .fmt.headers.size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder) + sizeof(struct link_info) * (thunderc->total_links - 1),
+      .fmt.headers.size = thunderc->monit_pkt_size,
       .fmt.headers.flags = 0,
       .fmt.content.link_monitoring_thunder.links_status = {}
     };
@@ -65,15 +96,7 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     union abstract_packet *new_ap = buffer_append_ap (bp_dup, &links);
 
     // 4. We compute the time difference
-    if (clock_gettime(CLOCK_MONOTONIC, &curr) == -1){
-      perror("clock_gettime error");
-      exit(EXIT_FAILURE);
-    }
-    secs = curr.tv_sec - thunderc->prev_link_time.tv_sec;
-    nsecs = curr.tv_nsec - thunderc->prev_link_time.tv_nsec;
-    thunderc->prev_link_time = curr;
-    mili_sec = secs * 1000 + nsecs / 1000000;
-    if (mili_sec > 200) mili_sec = 200;
+    uint64_t mili_sec = compute_delta (&thunderc->prev_link_time, 200);
 
     // 5. We create the array
     struct link_info *li = &new_ap->fmt.content.link_monitoring_thunder.links_status;
@@ -103,6 +126,9 @@ void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struc
   memset(app_ctx->misc, 0, sizeof(struct thunder_ctx));
   struct thunder_ctx* thunderc = app_ctx->misc;
   thunderc->selected_link = UINT8_MAX - 1;
+
+  union abstract_packet links = {};
+  thunderc->monit_pkt_size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder) + sizeof(struct link_info) * (thunderc->total_links - 1);
 }
 
 void classify() {

From 55f7bb3eef63a536f4b171e323a00db56efd4c40 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Tue, 27 Aug 2019 14:31:27 +0200
Subject: [PATCH 15/35] Implement pad

---
 src/algo_thunder.c | 33 +++++++++++++++++++++++++--------
 src/packet.c       | 14 +++++++++-----
 src/packet.h       | 12 +++++++-----
 3 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 2facadf..0573e9f 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -5,6 +5,9 @@
 #include "proxy.h"
 #include "timer.h"
 
+// A Tor cell size is 512 bytes but handle only 498 bytes of data
+#define TOR_CELL_SIZE 498
+
 struct thunder_ctx {
   uint16_t recv_id;
   uint16_t emit_id;
@@ -21,7 +24,7 @@ uint64_t compute_delta(struct timespec* prev_time, uint64_t max) {
   int secs, nsecs;
   uint64_t mili_sec;
 
-  // 4. We compute the time difference
+  // 1. We compute the time difference
   if (clock_gettime(CLOCK_MONOTONIC, &curr) == -1){
     perror("clock_gettime error");
     exit(EXIT_FAILURE);
@@ -39,9 +42,13 @@ void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct thunder_ctx* thunderc = app_ctx->misc;
 
+  // 1. Put the raw buffer in cache
+  thunderc->emit_id++;
+  uint64_t ref = 0l + thunderc->emit_id;
+  dup_buffer_toa (&app_ctx->br, bp, (void *)ref);
+
   uint64_t delta_pkt = compute_delta(&thunderc->prev_packet_time, 200);
 
-  thunderc->emit_id++;
   union abstract_packet metadata = {
     .fmt.headers.cmd = CMD_UDP_METADATA_THUNDER,
     .fmt.headers.size = sizeof(metadata.fmt.headers) + sizeof(metadata.fmt.content.udp_metadata_thunder),
@@ -57,18 +64,28 @@ void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer
   struct thunder_ctx* thunderc = app_ctx->misc;
   uint64_t ref = 0l + thunderc->emit_id;
 
-  dup_buffer_toa (&app_ctx->br, bp, (void *)ref);
-
+  // 1. Clean old buffers
   if (ref > 10 && get_app_buffer (&app_ctx->br, (void *)ref - 10l)) {
     mv_buffer_atof (&app_ctx->br, (void *)ref - 10l);
   }
 
-  //@FIXME we must add the delta t of the current packet
-  for (uint64_t add_ref = ref; add_ref >= 0 && get_app_buffer (&app_ctx->br, (void *)add_ref); add_ref--) {
-    struct buffer_packet *bpa = get_app_buffer (&app_ctx->br, (void *)add_ref);
+  // 2. Append abstract packets stored in our buffers
+  uint64_t add_ref = ref;
+  while(1) {
+    if (add_ref < 1) break;
+    add_ref--;
+    struct buffer_packet *bp_iter = get_app_buffer (&app_ctx->br, (void *)add_ref);
+    if (bp_iter == NULL) break;
+    union abstract_packet *ap = buffer_first_ap (bp_iter);
+    if (ap->fmt.headers.cmd != CMD_UDP_ENCAPSULATED || ap->fmt.headers.flags & FLAG_READ_NEXT) {
+      fprintf(stderr, "Invalid buffer!\n");
+      exit(EXIT_FAILURE);
+    }
 
+    if (buffer_full_size (bp) + ap->fmt.headers.size > TOR_CELL_SIZE - thunderc->monit_pkt_size) break;
+
+    buffer_append_ap (bp, ap);
   }
-
 }
 
 int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
diff --git a/src/packet.c b/src/packet.c
index 85989a7..1727fc8 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -7,8 +7,12 @@ union abstract_packet* ap_next(union abstract_packet* ap) {
   return NULL;
 }
 
+union abstract_packet* buffer_first_ap(struct buffer_packet* bp) {
+   return (union abstract_packet*) &bp->ip;
+}
+
 union abstract_packet* buffer_last_ap(struct buffer_packet* bp) {
-  union abstract_packet* ap = (union abstract_packet*) &bp->ip, *apn = NULL;
+  union abstract_packet* ap = buffer_first_ap (bp), *apn = NULL;
   while ((apn = ap_next(ap)) != NULL) ap = apn;
 
   return ap;
@@ -28,7 +32,7 @@ size_t buffer_count_ap(struct buffer_packet* bp) {
   return s;
 }
 
-size_t get_full_size(struct buffer_packet* bp) {
+size_t buffer_full_size(struct buffer_packet* bp) {
   return &(buffer_free_ap (bp))->raw - &bp->ip[0];
 }
 
@@ -93,8 +97,8 @@ enum FD_STATE write_packet_to_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_
 
   //dump_buffer_packet (bp);
   if (bp->mode != BP_WRITING) return FDS_ERR;
-  while (bp->awrite < get_full_size(bp)) {
-    nwrite = send(fdinfo->fd, &(ap->raw) + bp->awrite, get_full_size(bp) - bp->awrite, 0);
+  while (bp->awrite < buffer_full_size(bp)) {
+    nwrite = send(fdinfo->fd, &(ap->raw) + bp->awrite, buffer_full_size(bp) - bp->awrite, 0);
     if (nwrite == -1 && errno == EAGAIN) return FDS_AGAIN;
     if (nwrite == -1) return FDS_ERR;
     bp->awrite += nwrite;
@@ -190,7 +194,7 @@ enum FD_STATE read_packet_from_udp (struct evt_core_fdinfo* fdinfo, struct buffe
 
 void dump_buffer_packet(struct buffer_packet* bp) {
   printf("<Buffer Packet>\n");
-  printf("  mode=%d, aread=%d, awrite=%d, ap_count=%d, usage=%ld/%ld\n", bp->mode, bp->aread, bp->awrite, bp->ap_count, get_full_size (bp), sizeof(bp->ip));
+  printf("  mode=%d, aread=%d, awrite=%d, ap_count=%d, usage=%ld/%ld\n", bp->mode, bp->aread, bp->awrite, bp->ap_count, buffer_full_size (bp), sizeof(bp->ip));
   union abstract_packet* ap = (union abstract_packet*) &bp->ip;
   for (int i = 0; i < bp->ap_count; i++) {
     dump_abstract_packet(ap);
diff --git a/src/packet.h b/src/packet.h
index 6b1530f..94216d3 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -32,9 +32,9 @@ enum BP_MODE {
 };
 
 enum PKT_CMD {
-  CMD_UDP_ENCAPSULATED,
-  CMD_LINK_MONITORING_THUNDER,
-  CMD_UDP_METADATA_THUNDER,
+  CMD_UDP_ENCAPSULATED = 1,
+  CMD_LINK_MONITORING_THUNDER = 2,
+  CMD_UDP_METADATA_THUNDER = 3,
 };
 
 enum PKT_FLAGS {
@@ -89,8 +89,10 @@ struct udp_target {
 size_t get_full_size(struct buffer_packet* bp);
 
 union abstract_packet* buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap);
-union abstract_packet* buffer_free_ptr(struct buffer_packet* bp);
-union abstract_packet* buffer_last_ptr(struct buffer_packet* bp);
+union abstract_packet* buffer_free_ap(struct buffer_packet* bp);
+union abstract_packet* buffer_first_ap(struct buffer_packet* bp);
+union abstract_packet* buffer_last_ap(struct buffer_packet* bp);
+size_t buffer_full_size(struct buffer_packet* bp);
 union abstract_packet* ap_next(union abstract_packet* ap);
 
 enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fd, struct buffer_packet* bp);

From 6e3eb95778662340f45454b4edda4682d5f7f2d6 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Tue, 27 Aug 2019 17:28:14 +0200
Subject: [PATCH 16/35] WIP receive

---
 src/algo_thunder.c | 38 ++++++++++++++++++++++++++------------
 src/packet.h       |  1 -
 2 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 0573e9f..cce83e7 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -14,6 +14,7 @@ struct thunder_ctx {
   uint8_t selected_link;
   uint8_t total_links;
   uint8_t delta_t_per_link[64];
+  uint64_t received_pkts_on_link[64];
   uint8_t blacklisted[64];
   size_t monit_pkt_size;
   struct timespec prev_link_time, prev_packet_time;
@@ -42,19 +43,12 @@ void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct thunder_ctx* thunderc = app_ctx->misc;
 
-  // 1. Put the raw buffer in cache
   thunderc->emit_id++;
-  uint64_t ref = 0l + thunderc->emit_id;
-  dup_buffer_toa (&app_ctx->br, bp, (void *)ref);
-
-  uint64_t delta_pkt = compute_delta(&thunderc->prev_packet_time, 200);
-
   union abstract_packet metadata = {
     .fmt.headers.cmd = CMD_UDP_METADATA_THUNDER,
     .fmt.headers.size = sizeof(metadata.fmt.headers) + sizeof(metadata.fmt.content.udp_metadata_thunder),
     .fmt.headers.flags = 0,
     .fmt.content.udp_metadata_thunder.id = thunderc->emit_id,
-    .fmt.content.udp_metadata_thunder.deltat = delta_pkt
   };
   buffer_append_ap (bp, &metadata);
 }
@@ -64,9 +58,11 @@ void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer
   struct thunder_ctx* thunderc = app_ctx->misc;
   uint64_t ref = 0l + thunderc->emit_id;
 
-  // 1. Clean old buffers
-  if (ref > 10 && get_app_buffer (&app_ctx->br, (void *)ref - 10l)) {
-    mv_buffer_atof (&app_ctx->br, (void *)ref - 10l);
+  dup_buffer_toa (&app_ctx->br, bp, (void *)ref);
+
+  // 1. Clean old buffers (we keep only thunderc->total_links buffer, keeping more would be useless)
+  if (ref > thunderc->total_links && get_app_buffer (&app_ctx->br, (void *)(ref - thunderc->total_links))) {
+    mv_buffer_atof (&app_ctx->br, (void *)(ref - thunderc->total_links));
   }
 
   // 2. Append abstract packets stored in our buffers
@@ -148,8 +144,26 @@ void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struc
   thunderc->monit_pkt_size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder) + sizeof(struct link_info) * (thunderc->total_links - 1);
 }
 
-void classify() {
+void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct thunder_ctx* thunderc = app_ctx->misc;
 
+  union abstract_packet* ap = buffer_first_ap (bp);
+  while (ap != NULL && ap->fmt.headers.cmd != CMD_LINK_MONITORING_THUNDER) ap = ap_next(ap);
+  if (ap == NULL) {
+    fprintf(stderr, "Unable to find our packet\n");
+    exit(EXIT_FAILURE);
+  }
+
+  int link_id = url_get_port_int(fdinfo->url) - 7500;
+  thunderc->received_pkts_on_link[link_id]++;
+
+  struct link_info *li = &ap->fmt.content.link_monitoring_thunder.links_status;
+  for (int i = 0; i < thunderc->total_links; i++) {
+    uint64_t expected = i <= link_id ? thunderc->received_pkts_on_link[link_id] : thunderc->received_pkts_on_link[link_id] - 1;
+    if (thunderc->received_pkts_on_link[i] >= expected) continue; // Nothing to do, all packets have been received
+    set_timeout ()
+  }
 }
 
 void unpad() {
@@ -161,7 +175,7 @@ void adapt() {
 }
 
 int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
-  classify();
+  classify(ctx, fdinfo, bp);
   unpad();
   adapt();
   return 0;
diff --git a/src/packet.h b/src/packet.h
index 94216d3..81423df 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -60,7 +60,6 @@ union abstract_packet {
       } link_monitoring_thunder;
       struct {
         uint16_t id;
-        uint16_t deltat;
       } udp_metadata_thunder;
       struct {
         uint16_t port;

From 298d0f7f26a469d6299b17a731d4add03cc22591 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Wed, 28 Aug 2019 10:50:34 +0200
Subject: [PATCH 17/35] Add timeout for the classifier

---
 src/algo_thunder.c | 33 ++++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index cce83e7..9e7072c 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -7,6 +7,7 @@
 
 // A Tor cell size is 512 bytes but handle only 498 bytes of data
 #define TOR_CELL_SIZE 498
+#define ALLOWED_JITTER_MS 200
 
 struct thunder_ctx {
   uint16_t recv_id;
@@ -39,6 +40,10 @@ uint64_t compute_delta(struct timespec* prev_time, uint64_t max) {
   return mili_sec;
 }
 
+int is_blacklisted(struct thunder_ctx* thunderc, int link_id) {
+  return thunderc->blacklisted[link_id] >= thunderc->received_pkts_on_link[link_id];
+}
+
 void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct thunder_ctx* thunderc = app_ctx->misc;
@@ -120,7 +125,7 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     li[thunderc->selected_link].delta_t = 0;
 
     main_on_tcp_write(ctx, to_fdinfo);
-  } while (thunderc->blacklisted[thunderc->selected_link]);
+  } while (is_blacklisted (thunderc, thunderc->selected_link));
 
   if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Packets sent\n");
 
@@ -144,6 +149,21 @@ void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struc
   thunderc->monit_pkt_size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder) + sizeof(struct link_info) * (thunderc->total_links - 1);
 }
 
+struct block_info { uint8_t i; struct algo_ctx* app_ctx; uint64_t missing; };
+
+void on_block (struct evt_core_ctx* ctx, void* raw) {
+  struct block_info* bi = raw;
+  struct thunder_ctx* thunderc = bi->app_ctx->misc;
+
+  if (thunderc->received_pkts_on_link[bi->i] >= bi->missing) goto release;
+  if (thunderc->blacklisted[bi->i] >= bi->missing) goto release;
+
+  thunderc->blacklisted[bi->i] = bi->missing;
+
+release:
+  free(bi);
+}
+
 void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct thunder_ctx* thunderc = app_ctx->misc;
@@ -159,10 +179,17 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
   thunderc->received_pkts_on_link[link_id]++;
 
   struct link_info *li = &ap->fmt.content.link_monitoring_thunder.links_status;
-  for (int i = 0; i < thunderc->total_links; i++) {
+  for (uint8_t i = 0; i < thunderc->total_links; i++) {
     uint64_t expected = i <= link_id ? thunderc->received_pkts_on_link[link_id] : thunderc->received_pkts_on_link[link_id] - 1;
     if (thunderc->received_pkts_on_link[i] >= expected) continue; // Nothing to do, all packets have been received
-    set_timeout ()
+
+    int64_t timeout = ALLOWED_JITTER_MS - li[i].delta_t;
+    if (timeout < 0) timeout = 0;
+
+    struct block_info *bi = malloc(sizeof(struct block_info));
+    bi->i = i; bi->app_ctx = app_ctx; bi->missing = expected;
+
+    set_timeout (ctx, timeout, &bi, on_block);
   }
 }
 

From 4d5dd554c5e715e7355c3672ef9985c67ee53667 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Wed, 28 Aug 2019 11:35:43 +0200
Subject: [PATCH 18/35] Add unpad function

---
 src/algo_thunder.c | 34 +++++++++++++++++++++++++++++-----
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 9e7072c..20e854a 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -8,15 +8,16 @@
 // A Tor cell size is 512 bytes but handle only 498 bytes of data
 #define TOR_CELL_SIZE 498
 #define ALLOWED_JITTER_MS 200
+#define MAX_LINKS 64
 
 struct thunder_ctx {
   uint16_t recv_id;
   uint16_t emit_id;
   uint8_t selected_link;
   uint8_t total_links;
-  uint8_t delta_t_per_link[64];
-  uint64_t received_pkts_on_link[64];
-  uint8_t blacklisted[64];
+  uint8_t delta_t_per_link[MAX_LINKS];
+  uint64_t received_pkts_on_link[MAX_LINKS];
+  uint8_t blacklisted[MAX_LINKS];
   size_t monit_pkt_size;
   struct timespec prev_link_time, prev_packet_time;
 };
@@ -193,8 +194,29 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
   }
 }
 
-void unpad() {
+struct unpad_info {
+  union abstract_packet *ap_arr_pl[MAX_LINKS], *ap_arr_meta[MAX_LINKS];
+  uint8_t ap_arr_vals;
+};
 
+void unpad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp, struct unpad_info *ui) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct thunder_ctx* thunderc = app_ctx->misc;
+
+  for (union abstract_packet* ap = buffer_first_ap (bp); ap != NULL; ap = ap_next(ap)) {
+    if (ap->fmt.headers.cmd != CMD_UDP_ENCAPSULATED) continue;
+
+    union abstract_packet* ap_meta = ap_next(ap);
+    if (ap_meta == NULL || ap_meta->fmt.headers.cmd != CMD_UDP_METADATA_THUNDER) {
+      fprintf(stderr, "Unexpected packet, expecting udp metadata\n");
+    }
+
+    if (ap_meta->fmt.content.udp_metadata_thunder.id > thunderc->recv_id) {
+      ui->ap_arr_pl[ui->ap_arr_vals] = ap;
+      ui->ap_arr_meta[ui->ap_arr_vals] = ap_meta;
+      ui->ap_arr_vals++;
+    }
+  }
 }
 
 void adapt() {
@@ -202,8 +224,10 @@ void adapt() {
 }
 
 int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  struct unpad_info ui = {0};
+
   classify(ctx, fdinfo, bp);
-  unpad();
+  unpad(ctx, fdinfo, bp, &ui);
   adapt();
   return 0;
 }

From a1d58c120356405180eb8084200aaaf98f86e1c2 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Wed, 28 Aug 2019 14:57:20 +0200
Subject: [PATCH 19/35] Fix a first set of bugs

---
 src/algo_thunder.c | 67 +++++++++++++++++++++++++++++++++-------------
 src/algo_utils.c   | 19 ++++++++-----
 src/algo_utils.h   |  2 ++
 src/donar_init.c   |  2 +-
 src/packet.c       |  2 +-
 5 files changed, 66 insertions(+), 26 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 20e854a..3e66176 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -17,7 +17,7 @@ struct thunder_ctx {
   uint8_t total_links;
   uint8_t delta_t_per_link[MAX_LINKS];
   uint64_t received_pkts_on_link[MAX_LINKS];
-  uint8_t blacklisted[MAX_LINKS];
+  uint64_t blacklisted[MAX_LINKS];
   size_t monit_pkt_size;
   struct timespec prev_link_time, prev_packet_time;
 };
@@ -125,6 +125,10 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     }
     li[thunderc->selected_link].delta_t = 0;
 
+    if (ctx->verbose > 1) {
+      dump_buffer_packet(bp_dup);
+      fprintf(stderr, "    [algo_thunder] Will send this info\n");
+    }
     main_on_tcp_write(ctx, to_fdinfo);
   } while (is_blacklisted (thunderc, thunderc->selected_link));
 
@@ -136,20 +140,6 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
   return 0;
 }
 
-void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap) {
-  app_ctx->misc = malloc(sizeof(struct thunder_ctx));
-  if (app_ctx->misc == NULL) {
-    perror("malloc failed in algo thunder init");
-    exit(EXIT_FAILURE);
-  }
-  memset(app_ctx->misc, 0, sizeof(struct thunder_ctx));
-  struct thunder_ctx* thunderc = app_ctx->misc;
-  thunderc->selected_link = UINT8_MAX - 1;
-
-  union abstract_packet links = {};
-  thunderc->monit_pkt_size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder) + sizeof(struct link_info) * (thunderc->total_links - 1);
-}
-
 struct block_info { uint8_t i; struct algo_ctx* app_ctx; uint64_t missing; };
 
 void on_block (struct evt_core_ctx* ctx, void* raw) {
@@ -219,8 +209,29 @@ void unpad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
   }
 }
 
-void adapt() {
+void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp, struct unpad_info *ui) {
+  struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
+  struct thunder_ctx* thunderc = app_ctx->misc;
+  char url[256];
+  struct evt_core_fdinfo *to_fdinfo = NULL;
 
+  for (uint8_t i = ui->ap_arr_vals-1; i >= 0; i--) {
+    if (ui->ap_arr_meta[i]->fmt.content.udp_metadata_thunder.id <= thunderc->recv_id) continue;
+    thunderc->recv_id = ui->ap_arr_meta[i]->fmt.content.udp_metadata_thunder.id;
+
+    // Find destination
+    sprintf(url, "udp:write:127.0.0.1:%d", ui->ap_arr_pl[i]->fmt.content.udp_encapsulated.port);
+    to_fdinfo = evt_core_get_from_url (ctx, url);
+    if (to_fdinfo == NULL) {
+      fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
+    }
+
+    struct buffer_packet *bp_dest = inject_buffer_tow (&app_ctx->br, to_fdinfo);
+    buffer_append_ap (bp_dest, ui->ap_arr_pl[i]);
+    main_on_udp_write(ctx, to_fdinfo);
+  }
+
+  mv_buffer_rtof (&app_ctx->br, fdinfo);
 }
 
 int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -228,14 +239,34 @@ int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdi
 
   classify(ctx, fdinfo, bp);
   unpad(ctx, fdinfo, bp, &ui);
-  adapt();
+  adapt(ctx, fdinfo, bp, &ui);
   return 0;
 }
 
 int algo_thunder_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   prepare(ctx, fdinfo, bp);
   pad(ctx, fdinfo, bp);
-  return schedule(ctx, fdinfo, bp);
+  schedule(ctx, fdinfo, bp);
+  return 0;
+}
+
+void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap) {
+  app_ctx->misc = malloc(sizeof(struct thunder_ctx));
+  if (app_ctx->misc == NULL) {
+    perror("malloc failed in algo thunder init");
+    exit(EXIT_FAILURE);
+  }
+  memset(app_ctx->misc, 0, sizeof(struct thunder_ctx));
+  struct thunder_ctx* thunderc = app_ctx->misc;
+  thunderc->recv_id = 1;
+  thunderc->emit_id = 1;
+  thunderc->total_links = app_ctx->ap.links;
+  thunderc->selected_link = thunderc->total_links;
+  for (int i = 0; i < MAX_LINKS; i++) thunderc->received_pkts_on_link[i] = 1;
+
+  union abstract_packet links = {};
+  //fprintf(stderr, "Total links %d\n", thunderc->total_links);
+  thunderc->monit_pkt_size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder) + sizeof(struct link_info) * (thunderc->total_links - 1);
 }
 
 int algo_thunder_on_err(struct evt_core_ctx *ctx, struct evt_core_fdinfo *fdinfo) {
diff --git a/src/algo_utils.c b/src/algo_utils.c
index 6b5c301..68f862c 100644
--- a/src/algo_utils.c
+++ b/src/algo_utils.c
@@ -207,7 +207,7 @@ void mv_buffer_atof(struct buffer_resources *app_ctx, void* from) {
   __push_to_free (app_ctx, bp);
 }
 
-struct buffer_packet* dup_buffer_tow(struct buffer_resources *app_ctx, struct buffer_packet* bp, struct evt_core_fdinfo* to) {
+struct buffer_packet* inject_buffer_tow(struct buffer_resources *app_ctx, struct evt_core_fdinfo* to) {
   GQueue* q;
 
   // 1. We get a free buffer
@@ -217,21 +217,28 @@ struct buffer_packet* dup_buffer_tow(struct buffer_resources *app_ctx, struct bu
     return NULL;
   }
 
-  // 2. We duplicate the data
-  memcpy(bp_dest, bp, sizeof(struct buffer_packet));
-
-  // 3. We get the target writing queue
+  // 2. We get the target writing queue
   q = g_hash_table_lookup(app_ctx->write_waiting, &(to->fd));
   if (q == NULL) {
     q = g_queue_new ();
     g_hash_table_insert(app_ctx->write_waiting, &(to->fd), q);
   }
 
-  // 4. We push the content to the appropriate destination
+  // 3. We push the content to the appropriate destination
   g_queue_push_tail(q, bp_dest);
   return bp_dest;
 }
 
+struct buffer_packet* dup_buffer_tow(struct buffer_resources *app_ctx, struct buffer_packet* bp, struct evt_core_fdinfo* to) {
+  // 1. Inject a new buffer
+  struct buffer_packet* bp_dest = inject_buffer_tow (app_ctx, to);
+
+  // 2. We duplicate the data
+  memcpy(bp_dest, bp, sizeof(struct buffer_packet));
+
+  return bp_dest;
+}
+
 struct buffer_packet* dup_buffer_toa(struct buffer_resources *app_ctx, struct buffer_packet* bp, void* to) {
   GQueue* q;
 
diff --git a/src/algo_utils.h b/src/algo_utils.h
index ab53795..1a78950 100644
--- a/src/algo_utils.h
+++ b/src/algo_utils.h
@@ -23,6 +23,8 @@ void mv_buffer_wtof(struct buffer_resources* app_ctx, struct evt_core_fdinfo* fr
 void mv_buffer_rtoa(struct buffer_resources* app_ctx, struct evt_core_fdinfo* from, void* to);
 void mv_buffer_atow(struct buffer_resources* app_ctx, void* from, struct evt_core_fdinfo* to);
 void mv_buffer_atof(struct buffer_resources* app_ctx, void* from);
+
+struct buffer_packet* inject_buffer_tow(struct buffer_resources *app_ctx, struct evt_core_fdinfo* to);
 struct buffer_packet* dup_buffer_tow(struct buffer_resources* app_ctx, struct buffer_packet* bp, struct evt_core_fdinfo* to);
 struct buffer_packet* dup_buffer_toa(struct buffer_resources* app_ctx, struct buffer_packet* bp, void* to);
 guint write_queue_len(struct buffer_resources *app_ctx, struct evt_core_fdinfo *fdinfo);
diff --git a/src/donar_init.c b/src/donar_init.c
index 081a8fa..0975f47 100644
--- a/src/donar_init.c
+++ b/src/donar_init.c
@@ -28,7 +28,7 @@ int on_signal(struct evt_core_ctx* evts, struct evt_core_fdinfo* fdinfo) {
 }
 
 void signal_init(struct evt_core_ctx* evts) {
-  sigset_t mask;
+  sigset_t mask = {0};
 
   struct evt_core_cat signal_read = {
     .name = "signal-read",
diff --git a/src/packet.c b/src/packet.c
index 1727fc8..3422a5f 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -208,7 +208,7 @@ void dump_abstract_packet(union abstract_packet* ap) {
   printf("    size=%d, cmd=%d\n", ap->fmt.headers.size, ap->fmt.headers.cmd);
   switch (ap->fmt.headers.cmd) {
   case CMD_LINK_MONITORING_THUNDER:
-    printf("    <LinkMonitoringThunder/>\n");
+    printf("    <LinkMonitoringThunder></LinkMonitoringThunder>\n");
     break;
   case CMD_UDP_METADATA_THUNDER:
     printf("    <UdpMetadataThunder>id=%d</UdpMetadataThunder>\n",

From a9e5267495ea0c8f15e2b88d47515930d7232da8 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Wed, 28 Aug 2019 16:33:43 +0200
Subject: [PATCH 20/35] WIP debug algo thunder

---
 src/algo_thunder.c | 31 ++++++++++++++++++++++++++++---
 src/packet.c       |  1 +
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 3e66176..d7c78c0 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -57,6 +57,7 @@ void prepare(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     .fmt.content.udp_metadata_thunder.id = thunderc->emit_id,
   };
   buffer_append_ap (bp, &metadata);
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] UDP metadata added\n");
 }
 
 void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -79,14 +80,21 @@ void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer
     struct buffer_packet *bp_iter = get_app_buffer (&app_ctx->br, (void *)add_ref);
     if (bp_iter == NULL) break;
     union abstract_packet *ap = buffer_first_ap (bp_iter);
-    if (ap->fmt.headers.cmd != CMD_UDP_ENCAPSULATED || ap->fmt.headers.flags & FLAG_READ_NEXT) {
-      fprintf(stderr, "Invalid buffer!\n");
+    if (ap->fmt.headers.cmd != CMD_UDP_ENCAPSULATED) {
+      fprintf(stderr, "Invalid buffer payload!\n");
+      exit(EXIT_FAILURE);
+    }
+    union abstract_packet *ap_meta = ap_next (ap);
+    if (ap_meta->fmt.headers.cmd != CMD_UDP_METADATA_THUNDER) {
+      fprintf(stderr, "Invalid buffer metadata!\n");
       exit(EXIT_FAILURE);
     }
 
-    if (buffer_full_size (bp) + ap->fmt.headers.size > TOR_CELL_SIZE - thunderc->monit_pkt_size) break;
+    if (buffer_full_size (bp) + ap->fmt.headers.size + ap_meta->fmt.headers.size > TOR_CELL_SIZE - thunderc->monit_pkt_size) break;
 
     buffer_append_ap (bp, ap);
+    buffer_append_ap (bp, ap_meta);
+    if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Pad packet (now %ld bytes)\n", buffer_full_size (bp));
   }
 }
 
@@ -98,6 +106,10 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
 
   do {
     // 1. We choose the link
+    if (cat->socklist->len == 0) {
+      if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] No link available, packet will be dropped\n");
+      break;
+    }
     thunderc->selected_link = (thunderc->selected_link + 1) % cat->socklist->len;
 
     // 2. We create the packet template
@@ -181,7 +193,12 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
     bi->i = i; bi->app_ctx = app_ctx; bi->missing = expected;
 
     set_timeout (ctx, timeout, &bi, on_block);
+    if (ctx->verbose > 1) {
+      fprintf(stderr, "    [algo_thunder] Set timeout on link %d of %ld ms (packets expected: %ld, seen: %ld)\n",
+              i, timeout, expected, thunderc->received_pkts_on_link[i]);
+    }
   }
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Classify done\n");
 }
 
 struct unpad_info {
@@ -207,6 +224,7 @@ void unpad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
       ui->ap_arr_vals++;
     }
   }
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Unpad done\n");
 }
 
 void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp, struct unpad_info *ui) {
@@ -232,6 +250,7 @@ void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
   }
 
   mv_buffer_rtof (&app_ctx->br, fdinfo);
+  if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Adapt done\n");
 }
 
 int algo_thunder_on_stream(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
@@ -250,8 +269,14 @@ int algo_thunder_on_datagram(struct evt_core_ctx* ctx, struct evt_core_fdinfo* f
   return 0;
 }
 
+void algo_thunder_free(void* v) {
+  struct rr_ctx* rr = v;
+  free(rr);
+}
+
 void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struct algo_params* ap) {
   app_ctx->misc = malloc(sizeof(struct thunder_ctx));
+  app_ctx->free_misc = algo_thunder_free;
   if (app_ctx->misc == NULL) {
     perror("malloc failed in algo thunder init");
     exit(EXIT_FAILURE);
diff --git a/src/packet.c b/src/packet.c
index 3422a5f..898a734 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -41,6 +41,7 @@ union abstract_packet* buffer_append_ap(struct buffer_packet* bp, union abstract
   union abstract_packet *new_ap = buffer_last_ap(bp);
   memcpy(new_ap, ap, ap->fmt.headers.size);
   bp->ap_count++;
+  new_ap->fmt.headers.flags &= ~FLAG_READ_NEXT;
   return new_ap;
 }
 

From eb7b3a4712ee7e202af8b669c9e49ac2a5478542 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Wed, 28 Aug 2019 17:42:15 +0200
Subject: [PATCH 21/35] Fix another stupid bug

---
 src/packet.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/packet.c b/src/packet.c
index 898a734..b6c5a20 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -47,7 +47,7 @@ union abstract_packet* buffer_append_ap(struct buffer_packet* bp, union abstract
 
 enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   ssize_t nread = 0, ap_aread = 0, cur_ap_aread = 0;
-  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
+  union abstract_packet* ap = buffer_first_ap (bp);
   size_t pkt_size_size = sizeof(ap->fmt.headers.size);
   if (bp->mode != BP_READING) return FDS_ERR;
 
@@ -55,7 +55,8 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
   do {
 
     //fprintf(stderr, "bp->ap_count=%d\n", bp->ap_count);
-    ap = (union abstract_packet*) &bp->ip;
+    ap = buffer_first_ap (bp);
+    ap_aread = 0;
     for (int i = 0; i < bp->ap_count; i++) {
       ap_aread += ap->fmt.headers.size;
       ap = ap_next (ap);
@@ -84,6 +85,7 @@ enum FD_STATE read_packet_from_tcp(struct evt_core_fdinfo* fdinfo, struct buffer
 
     bp->ap_count++;
     //fprintf(stderr, "bp->ap_count=%d, buffer_count_ap(bp)=%ld\n", bp->ap_count, buffer_count_ap (bp));
+    //dump_buffer_packet (bp);
   } while (bp->ap_count != buffer_count_ap (bp));
 
   bp->mode = BP_WRITING;
@@ -196,10 +198,8 @@ enum FD_STATE read_packet_from_udp (struct evt_core_fdinfo* fdinfo, struct buffe
 void dump_buffer_packet(struct buffer_packet* bp) {
   printf("<Buffer Packet>\n");
   printf("  mode=%d, aread=%d, awrite=%d, ap_count=%d, usage=%ld/%ld\n", bp->mode, bp->aread, bp->awrite, bp->ap_count, buffer_full_size (bp), sizeof(bp->ip));
-  union abstract_packet* ap = (union abstract_packet*) &bp->ip;
-  for (int i = 0; i < bp->ap_count; i++) {
+  for (union abstract_packet* ap = buffer_first_ap (bp); ap != NULL; ap = ap_next (ap)) {
     dump_abstract_packet(ap);
-    ap = (union abstract_packet*)(&ap->raw + ap->fmt.headers.size);
   }
   printf("</Buffer Packet>\n");
 }

From 7fdd43658cd36653c99ad2402ee0c970530ed6bb Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Wed, 28 Aug 2019 17:49:01 +0200
Subject: [PATCH 22/35] Initialize timer

---
 src/algo_thunder.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index d7c78c0..469ce5c 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -292,6 +292,8 @@ void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struc
   union abstract_packet links = {};
   //fprintf(stderr, "Total links %d\n", thunderc->total_links);
   thunderc->monit_pkt_size = sizeof(links.fmt.headers) + sizeof(links.fmt.content.link_monitoring_thunder) + sizeof(struct link_info) * (thunderc->total_links - 1);
+
+  init_timer(ctx);
 }
 
 int algo_thunder_on_err(struct evt_core_ctx *ctx, struct evt_core_fdinfo *fdinfo) {

From f898521f15b09c8ba5dc7753ff29ca6eb3128a7d Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Wed, 28 Aug 2019 18:05:56 +0200
Subject: [PATCH 23/35] Fix timeout and append bug

---
 src/algo_thunder.c |  3 ++-
 src/packet.c       | 14 ++++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 469ce5c..0ac5269 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -192,7 +192,7 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
     struct block_info *bi = malloc(sizeof(struct block_info));
     bi->i = i; bi->app_ctx = app_ctx; bi->missing = expected;
 
-    set_timeout (ctx, timeout, &bi, on_block);
+    set_timeout (ctx, timeout, bi, on_block);
     if (ctx->verbose > 1) {
       fprintf(stderr, "    [algo_thunder] Set timeout on link %d of %ld ms (packets expected: %ld, seen: %ld)\n",
               i, timeout, expected, thunderc->received_pkts_on_link[i]);
@@ -245,6 +245,7 @@ void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
     }
 
     struct buffer_packet *bp_dest = inject_buffer_tow (&app_ctx->br, to_fdinfo);
+    dump_buffer_packet (bp_dest);
     buffer_append_ap (bp_dest, ui->ap_arr_pl[i]);
     main_on_udp_write(ctx, to_fdinfo);
   }
diff --git a/src/packet.c b/src/packet.c
index b6c5a20..c447f3d 100644
--- a/src/packet.c
+++ b/src/packet.c
@@ -1,7 +1,15 @@
 #include "packet.h"
 
+int ap_exists(union abstract_packet* ap) {
+  return ap->fmt.headers.cmd != 0;
+}
+
+int buffer_has_ap(struct buffer_packet* bp) {
+  return ap_exists(buffer_first_ap (bp));
+}
+
 union abstract_packet* ap_next(union abstract_packet* ap) {
-  if (ap->fmt.headers.flags & FLAG_READ_NEXT)
+  if (ap_exists (ap) && ap->fmt.headers.flags & FLAG_READ_NEXT)
     return (union abstract_packet*)(&ap->raw + ap->fmt.headers.size);
 
   return NULL;
@@ -37,7 +45,9 @@ size_t buffer_full_size(struct buffer_packet* bp) {
 }
 
 union abstract_packet* buffer_append_ap(struct buffer_packet* bp, union abstract_packet* ap) {
-  buffer_last_ap(bp)->fmt.headers.flags |= FLAG_READ_NEXT;
+  if (buffer_has_ap (bp))
+    buffer_last_ap(bp)->fmt.headers.flags |= FLAG_READ_NEXT;
+
   union abstract_packet *new_ap = buffer_last_ap(bp);
   memcpy(new_ap, ap, ap->fmt.headers.size);
   bp->ap_count++;

From 33b45583a23a1d47765af89a85d5cb2b068ca9f0 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Wed, 28 Aug 2019 18:27:06 +0200
Subject: [PATCH 24/35] Fix an unsigned int bug

---
 src/algo_thunder.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 0ac5269..8da5c8d 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -233,7 +233,8 @@ void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
   char url[256];
   struct evt_core_fdinfo *to_fdinfo = NULL;
 
-  for (uint8_t i = ui->ap_arr_vals-1; i >= 0; i--) {
+  for (int i = ui->ap_arr_vals-1; i >= 0; i--) {
+    fprintf(stderr, "i=%d, ui->ap_arr_vals=%d\n", i, ui->ap_arr_vals);
     if (ui->ap_arr_meta[i]->fmt.content.udp_metadata_thunder.id <= thunderc->recv_id) continue;
     thunderc->recv_id = ui->ap_arr_meta[i]->fmt.content.udp_metadata_thunder.id;
 

From e99c8a8e5a32a98f02ddf347a4a3e987cc9e6d8a Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Thu, 29 Aug 2019 11:27:34 +0200
Subject: [PATCH 25/35] First packets exchanged

---
 src/algo_thunder.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 8da5c8d..88f04cf 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -234,19 +234,21 @@ void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
   struct evt_core_fdinfo *to_fdinfo = NULL;
 
   for (int i = ui->ap_arr_vals-1; i >= 0; i--) {
-    fprintf(stderr, "i=%d, ui->ap_arr_vals=%d\n", i, ui->ap_arr_vals);
+    //fprintf(stderr, "i=%d, ui->ap_arr_vals=%d\n", i, ui->ap_arr_vals);
     if (ui->ap_arr_meta[i]->fmt.content.udp_metadata_thunder.id <= thunderc->recv_id) continue;
     thunderc->recv_id = ui->ap_arr_meta[i]->fmt.content.udp_metadata_thunder.id;
 
     // Find destination
     sprintf(url, "udp:write:127.0.0.1:%d", ui->ap_arr_pl[i]->fmt.content.udp_encapsulated.port);
     to_fdinfo = evt_core_get_from_url (ctx, url);
+    fprintf(stderr, "%s\n", url);
     if (to_fdinfo == NULL) {
       fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
     }
 
     struct buffer_packet *bp_dest = inject_buffer_tow (&app_ctx->br, to_fdinfo);
-    dump_buffer_packet (bp_dest);
+    bp_dest->mode = BP_WRITING;
+    //dump_buffer_packet (bp_dest);
     buffer_append_ap (bp_dest, ui->ap_arr_pl[i]);
     main_on_udp_write(ctx, to_fdinfo);
   }

From f28674a79dece0bd03f47781bebbdf01531bd826 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Thu, 29 Aug 2019 11:31:59 +0200
Subject: [PATCH 26/35] Remove log

---
 src/algo_thunder.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 88f04cf..679bd6a 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -241,7 +241,6 @@ void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
     // Find destination
     sprintf(url, "udp:write:127.0.0.1:%d", ui->ap_arr_pl[i]->fmt.content.udp_encapsulated.port);
     to_fdinfo = evt_core_get_from_url (ctx, url);
-    fprintf(stderr, "%s\n", url);
     if (to_fdinfo == NULL) {
       fprintf(stderr, "No fd for URL %s in tcp-read. Dropping packet :( \n", url);
     }

From 5ed70a08484ae0136bf4d9f00dc17c497f8224d8 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Thu, 29 Aug 2019 17:53:49 +0200
Subject: [PATCH 27/35] Fix bug

---
 src/algo_thunder.c | 32 +++++++++++++++++++++++++++++---
 src/packet.h       |  1 +
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 679bd6a..6151dc5 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -99,6 +99,7 @@ void pad(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer
 }
 
 int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
+  char url[256];
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct thunder_ctx* thunderc = app_ctx->misc;
   struct evt_core_fdinfo *to_fdinfo = NULL;
@@ -110,7 +111,13 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
       if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] No link available, packet will be dropped\n");
       break;
     }
-    thunderc->selected_link = (thunderc->selected_link + 1) % cat->socklist->len;
+
+    to_fdinfo = NULL;
+    do {
+      thunderc->selected_link = (thunderc->selected_link + 1) % thunderc->total_links;
+      sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + thunderc->selected_link);
+      to_fdinfo = evt_core_get_from_url (ctx, url);
+    } while (to_fdinfo == NULL);
 
     // 2. We create the packet template
     union abstract_packet links = {
@@ -120,8 +127,6 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
       .fmt.content.link_monitoring_thunder.links_status = {}
     };
 
-    to_fdinfo = g_array_index(cat->socklist, struct evt_core_fdinfo*, thunderc->selected_link);
-
     // 3. We append the template to the buffer
     struct buffer_packet* bp_dup = dup_buffer_tow (&app_ctx->br, bp, to_fdinfo);
     union abstract_packet *new_ap = buffer_append_ap (bp_dup, &links);
@@ -142,6 +147,7 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
       fprintf(stderr, "    [algo_thunder] Will send this info\n");
     }
     main_on_tcp_write(ctx, to_fdinfo);
+
   } while (is_blacklisted (thunderc, thunderc->selected_link));
 
   if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Packets sent\n");
@@ -161,6 +167,7 @@ void on_block (struct evt_core_ctx* ctx, void* raw) {
   if (thunderc->received_pkts_on_link[bi->i] >= bi->missing) goto release;
   if (thunderc->blacklisted[bi->i] >= bi->missing) goto release;
 
+  //printf("[algo_thunder] Blacklisting link %d\n", bi->i);
   thunderc->blacklisted[bi->i] = bi->missing;
 
 release:
@@ -178,8 +185,13 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
     exit(EXIT_FAILURE);
   }
 
+  if (ap->fmt.headers.flags & FLAG_RESET) {
+    for (int i = 0; i < MAX_LINKS; i++) thunderc->received_pkts_on_link[i] = 1;
+  }
+
   int link_id = url_get_port_int(fdinfo->url) - 7500;
   thunderc->received_pkts_on_link[link_id]++;
+  printf("Received %ld packets on link %d\n", thunderc->received_pkts_on_link[link_id], link_id);
 
   struct link_info *li = &ap->fmt.content.link_monitoring_thunder.links_status;
   for (uint8_t i = 0; i < thunderc->total_links; i++) {
@@ -193,12 +205,20 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
     bi->i = i; bi->app_ctx = app_ctx; bi->missing = expected;
 
     set_timeout (ctx, timeout, bi, on_block);
+    printf("  Triggered timeout for link %d in %ldms (expected: %ld, seen: %ld)\n", i, timeout, expected, thunderc->received_pkts_on_link[i]);
     if (ctx->verbose > 1) {
       fprintf(stderr, "    [algo_thunder] Set timeout on link %d of %ld ms (packets expected: %ld, seen: %ld)\n",
               i, timeout, expected, thunderc->received_pkts_on_link[i]);
     }
   }
   if (ctx->verbose > 1) fprintf(stderr, "    [algo_thunder] Classify done\n");
+
+  printf("Blacklisted links: ");
+  for (int i = 0; i < thunderc->total_links; i++) {
+    if (is_blacklisted (thunderc, i)) printf("_");
+    else printf("U");
+  }
+  printf("\n");
 }
 
 struct unpad_info {
@@ -232,6 +252,7 @@ void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
   struct thunder_ctx* thunderc = app_ctx->misc;
   char url[256];
   struct evt_core_fdinfo *to_fdinfo = NULL;
+  uint64_t delivered = 0;
 
   for (int i = ui->ap_arr_vals-1; i >= 0; i--) {
     //fprintf(stderr, "i=%d, ui->ap_arr_vals=%d\n", i, ui->ap_arr_vals);
@@ -250,6 +271,11 @@ void adapt(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buff
     //dump_buffer_packet (bp_dest);
     buffer_append_ap (bp_dest, ui->ap_arr_pl[i]);
     main_on_udp_write(ctx, to_fdinfo);
+    delivered++;
+  }
+
+  if (delivered != 1) {
+    //printf("[algo_thunder] Delivered %ld packets (now id=%d)\n", delivered, thunderc->recv_id);
   }
 
   mv_buffer_rtof (&app_ctx->br, fdinfo);
diff --git a/src/packet.h b/src/packet.h
index 81423df..c04376f 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -39,6 +39,7 @@ enum PKT_CMD {
 
 enum PKT_FLAGS {
   FLAG_READ_NEXT = 1 << 0,
+  FLAG_RESET = 1 << 1,
 };
 
 struct link_info {

From 124c0375c0755e98b5b3086124e0229b181c7207 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Thu, 29 Aug 2019 18:11:50 +0200
Subject: [PATCH 28/35] Better handle timeouts

---
 src/algo_thunder.c | 8 ++++----
 src/packet.h       | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 6151dc5..b856504 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -15,7 +15,7 @@ struct thunder_ctx {
   uint16_t emit_id;
   uint8_t selected_link;
   uint8_t total_links;
-  uint8_t delta_t_per_link[MAX_LINKS];
+  uint64_t delta_t_per_link[MAX_LINKS];
   uint64_t received_pkts_on_link[MAX_LINKS];
   uint64_t blacklisted[MAX_LINKS];
   size_t monit_pkt_size;
@@ -132,13 +132,13 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
     union abstract_packet *new_ap = buffer_append_ap (bp_dup, &links);
 
     // 4. We compute the time difference
-    uint64_t mili_sec = compute_delta (&thunderc->prev_link_time, 200);
+    uint64_t mili_sec = compute_delta (&thunderc->prev_link_time, UINT16_MAX);
 
     // 5. We create the array
     struct link_info *li = &new_ap->fmt.content.link_monitoring_thunder.links_status;
     for (int i = 0; i < thunderc->total_links; i++) {
       thunderc->delta_t_per_link[i] += mili_sec;
-      li[i].delta_t = thunderc->delta_t_per_link[i];
+      li[i].delta_t = thunderc->delta_t_per_link[i] > UINT16_MAX ? UINT16_MAX : thunderc->delta_t_per_link[i];
     }
     li[thunderc->selected_link].delta_t = 0;
 
@@ -315,7 +315,7 @@ void algo_thunder_init(struct evt_core_ctx* ctx, struct algo_ctx* app_ctx, struc
   thunderc->recv_id = 1;
   thunderc->emit_id = 1;
   thunderc->total_links = app_ctx->ap.links;
-  thunderc->selected_link = thunderc->total_links;
+  thunderc->selected_link = thunderc->total_links - 1;
   for (int i = 0; i < MAX_LINKS; i++) thunderc->received_pkts_on_link[i] = 1;
 
   union abstract_packet links = {};
diff --git a/src/packet.h b/src/packet.h
index c04376f..544e9cc 100644
--- a/src/packet.h
+++ b/src/packet.h
@@ -43,7 +43,7 @@ enum PKT_FLAGS {
 };
 
 struct link_info {
-  uint8_t delta_t;
+  uint16_t delta_t;
 };
 
 union abstract_packet {

From 2f73ea96e15ea1b0a8082aa373ec0092ef6c9e3b Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Thu, 29 Aug 2019 18:25:37 +0200
Subject: [PATCH 29/35] Fix scheduler

---
 src/algo_thunder.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index b856504..fc88556 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -118,6 +118,7 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
       sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + thunderc->selected_link);
       to_fdinfo = evt_core_get_from_url (ctx, url);
     } while (to_fdinfo == NULL);
+    printf("URL %s has been retained\n", url);
 
     // 2. We create the packet template
     union abstract_packet links = {
@@ -140,6 +141,7 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
       thunderc->delta_t_per_link[i] += mili_sec;
       li[i].delta_t = thunderc->delta_t_per_link[i] > UINT16_MAX ? UINT16_MAX : thunderc->delta_t_per_link[i];
     }
+    thunderc->delta_t_per_link[thunderc->selected_link] = 0;
     li[thunderc->selected_link].delta_t = 0;
 
     if (ctx->verbose > 1) {

From f4fa63fcef8126b6a61380dd477396164906eee2 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Thu, 29 Aug 2019 18:44:27 +0200
Subject: [PATCH 30/35] Remove logs

---
 src/algo_thunder.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index fc88556..d422b78 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -7,7 +7,7 @@
 
 // A Tor cell size is 512 bytes but handle only 498 bytes of data
 #define TOR_CELL_SIZE 498
-#define ALLOWED_JITTER_MS 200
+#define ALLOWED_JITTER_MS 100
 #define MAX_LINKS 64
 
 struct thunder_ctx {
@@ -118,7 +118,7 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
       sprintf(url, "tcp:write:127.0.0.1:%d", 7500 + thunderc->selected_link);
       to_fdinfo = evt_core_get_from_url (ctx, url);
     } while (to_fdinfo == NULL);
-    printf("URL %s has been retained\n", url);
+    //printf("URL %s has been retained\n", url);
 
     // 2. We create the packet template
     union abstract_packet links = {

From 3f40d205c9afbd177cc03fc3de5a873e475a5950 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Fri, 30 Aug 2019 10:39:00 +0200
Subject: [PATCH 31/35] Better classification

---
 src/algo_thunder.c | 63 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 57 insertions(+), 6 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index d422b78..ab30314 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -7,7 +7,7 @@
 
 // A Tor cell size is 512 bytes but handle only 498 bytes of data
 #define TOR_CELL_SIZE 498
-#define ALLOWED_JITTER_MS 100
+#define ALLOWED_JITTER_MS 200
 #define MAX_LINKS 64
 
 struct thunder_ctx {
@@ -16,10 +16,11 @@ struct thunder_ctx {
   uint8_t selected_link;
   uint8_t total_links;
   uint64_t delta_t_per_link[MAX_LINKS];
+  uint64_t rcv_delta_t_per_link[MAX_LINKS];
   uint64_t received_pkts_on_link[MAX_LINKS];
   uint64_t blacklisted[MAX_LINKS];
   size_t monit_pkt_size;
-  struct timespec prev_link_time, prev_packet_time;
+  struct timespec prev_link_time, prev_rcv_link_time;
 };
 
 uint64_t compute_delta(struct timespec* prev_time, uint64_t max) {
@@ -160,7 +161,7 @@ int schedule(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct bu
   return 0;
 }
 
-struct block_info { uint8_t i; struct algo_ctx* app_ctx; uint64_t missing; };
+struct block_info { uint8_t i; struct algo_ctx* app_ctx; uint64_t missing;};
 
 void on_block (struct evt_core_ctx* ctx, void* raw) {
   struct block_info* bi = raw;
@@ -176,6 +177,16 @@ release:
   free(bi);
 }
 
+int is_in_order(struct thunder_ctx* thunderc, uint8_t link_id) {
+  uint64_t ref = thunderc->received_pkts_on_link[link_id];
+  for (int i = 0; i < thunderc->total_links; i++) {
+    uint64_t expected = link_id > i ? ref - 1 : ref;
+    if (thunderc->received_pkts_on_link[i] > expected) return 0;
+  }
+
+  return 1;
+}
+
 void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct buffer_packet* bp) {
   struct algo_ctx* app_ctx = fdinfo->cat->app_ctx;
   struct thunder_ctx* thunderc = app_ctx->misc;
@@ -187,27 +198,67 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
     exit(EXIT_FAILURE);
   }
 
+  /*
   if (ap->fmt.headers.flags & FLAG_RESET) {
     for (int i = 0; i < MAX_LINKS; i++) thunderc->received_pkts_on_link[i] = 1;
   }
+  */
 
+  // 1. Update link info
   int link_id = url_get_port_int(fdinfo->url) - 7500;
   thunderc->received_pkts_on_link[link_id]++;
   printf("Received %ld packets on link %d\n", thunderc->received_pkts_on_link[link_id], link_id);
-
   struct link_info *li = &ap->fmt.content.link_monitoring_thunder.links_status;
+
+  uint64_t mili_sec = compute_delta (&thunderc->prev_rcv_link_time, UINT16_MAX);
+  for (int i = 0; i < thunderc->total_links; i++) {
+    thunderc->rcv_delta_t_per_link[i] += mili_sec;
+  }
+  thunderc->rcv_delta_t_per_link[link_id] = 0;
+
+  // 2. Disable links that have received packets too late
+  if (is_in_order (thunderc, link_id)) {
+    printf("Local: ");
+    for (int i = 0; i < thunderc->total_links; i++) {
+      printf("%ld ", thunderc->rcv_delta_t_per_link[i]);
+    }
+    printf("\n");
+    printf("Packet: ");
+    for (int i = 0; i < thunderc->total_links; i++) {
+      printf("%d ", li[i].delta_t);
+    }
+    printf("\n");
+
+    for (int i = 0; i < thunderc->total_links; i++) {
+      if (ALLOWED_JITTER_MS >= li[i].delta_t) continue;
+      if (li[i].delta_t - ALLOWED_JITTER_MS <= thunderc->rcv_delta_t_per_link[i]) continue;
+
+      struct block_info *bi = malloc(sizeof(struct block_info));
+      bi->i = i; bi->app_ctx = app_ctx; bi->missing = thunderc->received_pkts_on_link[i]+1;
+
+      printf("  Packet Too Late - Blocked link %d (expected: at least %dms ago, received: %ldms ago)\n", i, li[i].delta_t - ALLOWED_JITTER_MS, thunderc->rcv_delta_t_per_link[i]);
+      on_block(ctx, bi);
+    }
+  }
+
+  // 3. Disable links that miss packets
   for (uint8_t i = 0; i < thunderc->total_links; i++) {
     uint64_t expected = i <= link_id ? thunderc->received_pkts_on_link[link_id] : thunderc->received_pkts_on_link[link_id] - 1;
     if (thunderc->received_pkts_on_link[i] >= expected) continue; // Nothing to do, all packets have been received
 
     int64_t timeout = ALLOWED_JITTER_MS - li[i].delta_t;
-    if (timeout < 0) timeout = 0;
 
     struct block_info *bi = malloc(sizeof(struct block_info));
     bi->i = i; bi->app_ctx = app_ctx; bi->missing = expected;
 
+    if (timeout <= 0) {
+      on_block(ctx, bi);
+      printf("  Missing Packet - Blocked link %d (expected: %ld, seen: %ld, )\n", i, expected, thunderc->received_pkts_on_link[i]);
+      continue;
+    }
+
     set_timeout (ctx, timeout, bi, on_block);
-    printf("  Triggered timeout for link %d in %ldms (expected: %ld, seen: %ld)\n", i, timeout, expected, thunderc->received_pkts_on_link[i]);
+    printf("  Missing Packet - Triggered timeout for link %d in %ldms (expected: %ld, seen: %ld)\n", i, timeout, expected, thunderc->received_pkts_on_link[i]);
     if (ctx->verbose > 1) {
       fprintf(stderr, "    [algo_thunder] Set timeout on link %d of %ld ms (packets expected: %ld, seen: %ld)\n",
               i, timeout, expected, thunderc->received_pkts_on_link[i]);

From f61148345eaf0de53deb6f85aa464812a7114ea9 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Fri, 30 Aug 2019 10:48:23 +0200
Subject: [PATCH 32/35] Fix out of order

---
 src/algo_thunder.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index ab30314..40d273b 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -180,8 +180,11 @@ release:
 int is_in_order(struct thunder_ctx* thunderc, uint8_t link_id) {
   uint64_t ref = thunderc->received_pkts_on_link[link_id];
   for (int i = 0; i < thunderc->total_links; i++) {
-    uint64_t expected = link_id > i ? ref - 1 : ref;
-    if (thunderc->received_pkts_on_link[i] > expected) return 0;
+    uint64_t expected = link_id >= i ? ref : ref - 1;
+    if (thunderc->received_pkts_on_link[i] > expected) {
+      printf("link_id=%d, i=%d, pkt_i=%ld, pkt_i_expected=%ld, pkt_link_id=%ld\n", link_id, i, thunderc->received_pkts_on_link[i], expected, ref);
+      return 0;
+    }
   }
 
   return 1;
@@ -253,7 +256,7 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
 
     if (timeout <= 0) {
       on_block(ctx, bi);
-      printf("  Missing Packet - Blocked link %d (expected: %ld, seen: %ld, )\n", i, expected, thunderc->received_pkts_on_link[i]);
+      printf("  Missing Packet - Blocked link %d (expected: %ld, seen: %ld)\n", i, expected, thunderc->received_pkts_on_link[i]);
       continue;
     }
 

From 73c09bc4e9223208c89a337f8caa372b4a44d0a0 Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Fri, 30 Aug 2019 11:12:20 +0200
Subject: [PATCH 33/35] Comment debug logs

---
 src/algo_thunder.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index 40d273b..c483bb8 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -182,7 +182,7 @@ int is_in_order(struct thunder_ctx* thunderc, uint8_t link_id) {
   for (int i = 0; i < thunderc->total_links; i++) {
     uint64_t expected = link_id >= i ? ref : ref - 1;
     if (thunderc->received_pkts_on_link[i] > expected) {
-      printf("link_id=%d, i=%d, pkt_i=%ld, pkt_i_expected=%ld, pkt_link_id=%ld\n", link_id, i, thunderc->received_pkts_on_link[i], expected, ref);
+      //printf("link_id=%d, i=%d, pkt_i=%ld, pkt_i_expected=%ld, pkt_link_id=%ld\n", link_id, i, thunderc->received_pkts_on_link[i], expected, ref);
       return 0;
     }
   }
@@ -210,7 +210,7 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
   // 1. Update link info
   int link_id = url_get_port_int(fdinfo->url) - 7500;
   thunderc->received_pkts_on_link[link_id]++;
-  printf("Received %ld packets on link %d\n", thunderc->received_pkts_on_link[link_id], link_id);
+  //printf("Received %ld packets on link %d\n", thunderc->received_pkts_on_link[link_id], link_id);
   struct link_info *li = &ap->fmt.content.link_monitoring_thunder.links_status;
 
   uint64_t mili_sec = compute_delta (&thunderc->prev_rcv_link_time, UINT16_MAX);
@@ -221,7 +221,7 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
 
   // 2. Disable links that have received packets too late
   if (is_in_order (thunderc, link_id)) {
-    printf("Local: ");
+    /*printf("Local: ");
     for (int i = 0; i < thunderc->total_links; i++) {
       printf("%ld ", thunderc->rcv_delta_t_per_link[i]);
     }
@@ -230,7 +230,7 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
     for (int i = 0; i < thunderc->total_links; i++) {
       printf("%d ", li[i].delta_t);
     }
-    printf("\n");
+    printf("\n");*/
 
     for (int i = 0; i < thunderc->total_links; i++) {
       if (ALLOWED_JITTER_MS >= li[i].delta_t) continue;
@@ -239,7 +239,7 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
       struct block_info *bi = malloc(sizeof(struct block_info));
       bi->i = i; bi->app_ctx = app_ctx; bi->missing = thunderc->received_pkts_on_link[i]+1;
 
-      printf("  Packet Too Late - Blocked link %d (expected: at least %dms ago, received: %ldms ago)\n", i, li[i].delta_t - ALLOWED_JITTER_MS, thunderc->rcv_delta_t_per_link[i]);
+      //printf("  Packet Too Late - Blocked link %d (expected: at least %dms ago, received: %ldms ago)\n", i, li[i].delta_t - ALLOWED_JITTER_MS, thunderc->rcv_delta_t_per_link[i]);
       on_block(ctx, bi);
     }
   }
@@ -256,12 +256,12 @@ void classify(struct evt_core_ctx* ctx, struct evt_core_fdinfo* fdinfo, struct b
 
     if (timeout <= 0) {
       on_block(ctx, bi);
-      printf("  Missing Packet - Blocked link %d (expected: %ld, seen: %ld)\n", i, expected, thunderc->received_pkts_on_link[i]);
+      //printf("  Missing Packet - Blocked link %d (expected: %ld, seen: %ld)\n", i, expected, thunderc->received_pkts_on_link[i]);
       continue;
     }
 
     set_timeout (ctx, timeout, bi, on_block);
-    printf("  Missing Packet - Triggered timeout for link %d in %ldms (expected: %ld, seen: %ld)\n", i, timeout, expected, thunderc->received_pkts_on_link[i]);
+    //printf("  Missing Packet - Triggered timeout for link %d in %ldms (expected: %ld, seen: %ld)\n", i, timeout, expected, thunderc->received_pkts_on_link[i]);
     if (ctx->verbose > 1) {
       fprintf(stderr, "    [algo_thunder] Set timeout on link %d of %ld ms (packets expected: %ld, seen: %ld)\n",
               i, timeout, expected, thunderc->received_pkts_on_link[i]);

From 01f8e831484d6f3c3cc37246c35a75019c62bc1f Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Fri, 30 Aug 2019 12:16:34 +0200
Subject: [PATCH 34/35] Jitter at 100ms

---
 src/algo_thunder.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/algo_thunder.c b/src/algo_thunder.c
index c483bb8..3b30208 100644
--- a/src/algo_thunder.c
+++ b/src/algo_thunder.c
@@ -7,7 +7,7 @@
 
 // A Tor cell size is 512 bytes but handle only 498 bytes of data
 #define TOR_CELL_SIZE 498
-#define ALLOWED_JITTER_MS 200
+#define ALLOWED_JITTER_MS 100
 #define MAX_LINKS 64
 
 struct thunder_ctx {

From a80afbb737cfa9c780fd1c4873830332f549316c Mon Sep 17 00:00:00 2001
From: Quentin Dufour <quentin@deuxfleurs.fr>
Date: Mon, 2 Sep 2019 09:59:51 +0200
Subject: [PATCH 35/35] Remove runner run-2

---
 scripts/run-2 | 88 ---------------------------------------------------
 1 file changed, 88 deletions(-)
 delete mode 100755 scripts/run-2

diff --git a/scripts/run-2 b/scripts/run-2
deleted file mode 100755
index 9c3f4b3..0000000
--- a/scripts/run-2
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/bin/bash
-WAITFOR=2280 # 38min
-
-echo "Create output folder..."
-docker run \
-  --rm \
-  --user root \
-  -v `pwd`/out:/home/donar \
-  registry.gitlab.inria.fr/qdufour/donar \
-  chown -R 1000:1000 /home/donar
-
-for i in {1..10}; do
-echo "Spawning container $i..."
-docker run \
-  --rm \
-  -d \
-  --name "donarxp_server_${i}" \
-  -e HOME='/tmp' \
-  -v `pwd`/out:/home/donar \
-  registry.gitlab.inria.fr/qdufour/donar \
-  tor -f /etc/torrc
-
-docker run \
-  --rm \
-  -d \
-  --name "donarxp_client_${i}" \
-  -e HOME='/tmp' \
-  -v `pwd`/out:/home/donar \
-  registry.gitlab.inria.fr/qdufour/donar \
-  tor -f /etc/torrc
-
-done
-
-sleep 10
-
-for j in {1..100}; do
-echo "Run xp $j..."
-run_fold=`mktemp -up . XXXXXXXXXXXXXXXX`
-
-echo "Reset containers..."
-for i in {1..10}; do
-  docker exec donarxp_client_${i} sh -c 'killall -9 bash; killall -9 donar; killall -9 measlat; killall -9 udpecho'
-  docker exec donarxp_server_${i} sh -c 'killall -9 bash; killall -9 donar; killall -9 measlat; killall -9 udpecho'
-done
-
-echo "Launch servers..."
-docker exec donarxp_server_2 rrhr-server ${run_fold}-rrhr-2 &
-docker exec donarxp_server_3 dup2-server ${run_fold}-dup2-3 &
-docker exec donarxp_server_4 rrhr-server ${run_fold}-rrhr-4 &
-docker exec donarxp_server_5 dup2-server ${run_fold}-dup2-5 &
-docker exec donarxp_server_6 rrhr-server ${run_fold}-rrhr-6 &
-docker exec donarxp_server_7 dup2-server ${run_fold}-dup2-7 &
-docker exec donarxp_server_8 orig-server ${run_fold}-orig-8 &
-docker exec donarxp_server_9 orig-server ${run_fold}-orig-9 &
-docker exec donarxp_server_10 orig-server ${run_fold}-orig-10 &
-
-sleep 10
-echo "Launch measures..."
-timeout $WAITFOR bash <<EOF
-  docker exec donarxp_client_1 witness2-client 6000 100 100 ${run_fold}-witness2-1 &
-  sleep 10
-  docker exec donarxp_client_2 rrhr-client 6000 100 100 ${run_fold}-rrhr-2 &
-  sleep 10
-  docker exec donarxp_client_3 dup2-client 6000 100 100 ${run_fold}-dup2-3 &
-  sleep 10
-  docker exec donarxp_client_4 rrhr-client 6000 100 100 ${run_fold}-rrhr-4 &
-  sleep 10
-  docker exec donarxp_client_5 dup2-client 6000 100 100 ${run_fold}-dup2-5 &
-  sleep 10
-  docker exec donarxp_client_6 rrhr-client 6000 100 100 ${run_fold}-rrhr-6 &
-  sleep 10
-  docker exec donarxp_client_7 dup2-client 6000 100 100 ${run_fold}-dup2-7 &
-  sleep 10
-  docker exec donarxp_client_8 orig-client 6000 100 100 ${run_fold}-orig-8 &
-  sleep 10
-  docker exec donarxp_client_9 orig-client 6000 100 100 ${run_fold}-orig-9 &
-  sleep 10
-  docker exec donarxp_client_10 orig-client 6000 100 100 ${run_fold}-orig-10 &
-  sleep 10
-  wait;
-EOF
-done
-
-echo "Unspawn containers..."
-for i in {1..10}; do
-  docker kill "donarxp_server_${i}"
-  docker kill "donarxp_client_${i}"
-done